nodejieba-plus 3.5.8

package/submodules/cppjieba/test/unittest/textrank_test.cpp

@@ -0,0 +1,86 @@
+#include "cppjieba/TextRankExtractor.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+TEST(TextRankExtractorTest, Test1) {
+  TextRankExtractor Extractor(
+    "../test/testdata/extra_dict/jieba.dict.small.utf8",
+    "../dict/hmm_model.utf8", 
+    "../dict/stop_words.utf8");
+  {
+    string s("你好世界世界而且而且");
+    string res;
+    size_t topN = 5;
+
+    {
+      vector<string> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[\"世界\", \"你好\"]");
+    }
+
+    {
+      vector<pair<string, double> > words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[世界:1, 你好:0.519787]");
+    }
+
+    {
+      vector<TextRankExtractor::Word> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[{\"word\": \"世界\", \"offset\": [6, 12], \"weight\": 1}, {\"word\": \"你好\", \"offset\": [0], \"weight\": 0.519787}]");
+    }
+  }
+
+  { 
+    string s("\xe6\x88\x91\xe6\x98\xaf\xe6\x8b\x96\xe6\x8b\x89\xe6\x9c\xba\xe5\xad\xa6\xe9\x99\xa2\xe6\x89\x8b\xe6\x89\xb6\xe6\x8b\x96\xe6\x8b\x89\xe6\x9c\xba\xe4\xb8\x93\xe4\xb8\x9a\xe7\x9a\x84\xe3\x80\x82\xe4\xb8\x8d\xe7\x94\xa8\xe5\xa4\x9a\xe4\xb9\x85\xef\xbc\x8c\xe6\x88\x91\xe5\xb0\xb1\xe4\xbc\x9a\xe5\x8d\x87\xe8\x81\x8c\xe5\x8a\xa0\xe8\x96\xaa\xef\xbc\x8c\xe5\xbd\x93\xe4\xb8\x8a CEO\xef\xbc\x8c\xe8\xb5\xb0\xe4\xb8\x8a\xe4\xba\xba\xe7\x94\x9f\xe5\xb7\x85\xe5\xb3\xb0");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"当上\", \"offset\": [87], \"weight\": 1}, {\"word\": \"不用\", \"offset\": [48], \"weight\": 0.989848}, {\"word\": \"多久\", \"offset\": [54], \"weight\": 0.985126}, {\"word\": \"加薪\", \"offset\": [78], \"weight\": 0.983046}, {\"word\": \"升职\", \"offset\": [72], \"weight\": 0.980278}]");
+    //ASSERT_EQ(res, "[{\"word\": \"专业\", \"offset\": [36], \"weight\": 1}, {\"word\": \"CEO\", \"offset\": [94], \"weight\": 0.95375}, {\"word\": \"手扶拖拉机\", \"offset\": [21], \"weight\": 0.801701}, {\"word\": \"当上\", \"offset\": [87], \"weight\": 0.798968}, {\"word\": \"走上\", \"offset\": [100], \"weight\": 0.775505}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"一部\", \"offset\": [0], \"weight\": 1}, {\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 0.996126}]");
+  }
+}
+
+TEST(TextRankExtractorTest, Test2) {
+  TextRankExtractor Extractor(
+    "../test/testdata/extra_dict/jieba.dict.small.utf8",
+    "../dict/hmm_model.utf8",
+    "../dict/stop_words.utf8",
+    "../test/testdata/userdict.utf8");
+
+  {
+    string s("\xe8\x93\x9d\xe7\xbf\x94\xe4\xbc\x98\xe7\xa7\x80\xe6\xaf\x95\xe4\xb8\x9a\xe7\x94\x9f");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"蓝翔\", \"offset\": [0], \"weight\": 1}, {\"word\": \"毕业生\", \"offset\": [12], \"weight\": 0.996685}, {\"word\": \"优秀\", \"offset\": [6], \"weight\": 0.992994}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"一部\", \"offset\": [0], \"weight\": 1}, {\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 0.996126}]");
+  }
+}

package/submodules/cppjieba/test/unittest/trie_test.cpp

@@ -0,0 +1,177 @@
+#include "cppjieba/DictTrie.hpp"
+#include "cppjieba/MPSegment.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+static const char* const DICT_FILE = "../test/testdata/extra_dict/jieba.dict.small.utf8";
+
+TEST(TrieTest, Empty) {
+  vector<Unicode> keys;
+  vector<const DictUnit*> values;
+  Trie trie(keys, values);
+}
+
+TEST(TrieTest, Construct) {
+  vector<Unicode> keys;
+  vector<const DictUnit*> values;
+  keys.push_back(DecodeUTF8RunesInString("你"));
+  values.push_back((const DictUnit*)(NULL));
+  Trie trie(keys, values);
+}
+
+TEST(DictTrieTest, NewAndDelete) {
+  DictTrie * trie;
+  trie = new DictTrie(DICT_FILE);
+  delete trie;
+}
+
+TEST(DictTrieTest, Test1) {
+  string s1, s2;
+  DictTrie trie(DICT_FILE);
+  ASSERT_LT(trie.GetMinWeight() + 15.6479, 0.001);
+  string word("来到");
+  cppjieba::RuneStrArray uni;
+  ASSERT_TRUE(DecodeUTF8RunesInString(word, uni));
+  //DictUnit nodeInfo;
+  //nodeInfo.word = uni;
+  //nodeInfo.tag = "v";
+  //nodeInfo.weight = -8.87033;
+  //s1 << nodeInfo;
+  //s2 << (*trie.Find(uni.begin(), uni.end()));
+  const DictUnit* du = trie.Find(uni.begin(), uni.end());
+  ASSERT_TRUE(du != NULL);
+  ASSERT_EQ(2u, du->word.size());
+  ASSERT_EQ(26469u, du->word[0]);
+  ASSERT_EQ(21040u, du->word[1]);
+  ASSERT_EQ("v", du->tag);
+  ASSERT_NEAR(-8.870, du->weight, 0.001);
+
+  //EXPECT_EQ("[\"26469\", \"21040\"] v -8.870", s2);
+  word = "清华大学";
+  LocalVector<pair<size_t, const DictUnit*> > res;
+  const char * words[] = {"清", "清华", "清华大学"};
+  for (size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++) {
+    ASSERT_TRUE(DecodeUTF8RunesInString(words[i], uni));
+    res.push_back(make_pair(uni.size() - 1, trie.Find(uni.begin(), uni.end())));
+    //resMap[uni.size() - 1] = trie.Find(uni.begin(), uni.end());
+  }
+  vector<pair<size_t, const DictUnit*> > vec;
+  vector<struct Dag> dags;
+  ASSERT_TRUE(DecodeUTF8RunesInString(word, uni));
+  trie.Find(uni.begin(), uni.end(), dags);
+  ASSERT_EQ(dags.size(), uni.size());
+  ASSERT_NE(dags.size(), 0u);
+  s1 << res;
+  s2 << dags[0].nexts;
+  ASSERT_EQ(s1, s2);
+  
+}
+
+TEST(DictTrieTest, UserDict) {
+  DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
+  string word = "云计算";
+  cppjieba::RuneStrArray unicode;
+  ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+  const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit != NULL);
+  ASSERT_NEAR(unit->weight, -14.100, 0.001);
+
+  word = "蓝翔";
+  ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+  unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit != NULL);
+  ASSERT_EQ(unit->tag, "nz");
+  ASSERT_NEAR(unit->weight, -14.100, 0.001);
+
+  word = "区块链";
+  ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+  unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit != NULL);
+  ASSERT_EQ(unit->tag, "nz");
+  ASSERT_NEAR(unit->weight, -15.6478, 0.001);
+}
+
+TEST(DictTrieTest, UserDictWithMaxWeight) {
+  DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8", DictTrie::WordWeightMax);
+  string word = "云计算";
+  cppjieba::RuneStrArray unicode;
+  ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+  const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit);
+  ASSERT_NEAR(unit->weight, -2.975, 0.001);
+}
+
+TEST(DictTrieTest, Dag) {
+  DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
+
+  {
+    string word = "清华大学";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res);
+
+    size_t nexts_sizes[] = {3, 2, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "北京邮电大学";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res);
+
+    size_t nexts_sizes[] = {3, 1, 2, 2, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "长江大桥";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res);
+
+    size_t nexts_sizes[] = {3, 1, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "长江大桥";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res, 3);
+
+    size_t nexts_sizes[] = {2, 1, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "长江大桥";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeUTF8RunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res, 4);
+
+    size_t nexts_sizes[] = {3, 1, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+}

package/submodules/cppjieba/test/unittest/unicode_test.cpp

@@ -0,0 +1,43 @@
+#include "cppjieba/Unicode.hpp"
+#include "limonp/StdExtension.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+using namespace std;
+
+TEST(UnicodeTest, Test1) {
+  string s = "你好世界";
+  RuneStrArray runes;
+  ASSERT_TRUE(DecodeUTF8RunesInString(s, runes));
+  string actual;
+  string expected = "[\"{\"rune\": \"20320\", \"offset\": 0, \"len\": 3}\", \"{\"rune\": \"22909\", \"offset\": 3, \"len\": 3}\", \"{\"rune\": \"19990\", \"offset\": 6, \"len\": 3}\", \"{\"rune\": \"30028\", \"offset\": 9, \"len\": 3}\"]";
+  actual << runes;
+  ASSERT_EQ(expected, actual);
+}
+
+TEST(UnicodeTest, Illegal) {
+  string s = "123\x80";
+  RuneStrArray runes;
+  ASSERT_FALSE(DecodeUTF8RunesInString(s, runes));
+  string actual;
+  string expected = "[]";
+  actual << runes;
+  ASSERT_EQ(expected, actual);
+}
+
+TEST(UnicodeTest, Rand) {
+  const size_t ITERATION = 1024;
+  const size_t MAX_LEN = 256;
+  string s;
+  srand(time(NULL));
+
+  for (size_t i = 0; i < ITERATION; i++) {
+    size_t len = rand() % MAX_LEN;
+    s.resize(len);
+    for (size_t j = 0; j < len; j++) {
+      s[rand() % len] = rand();
+    }
+    RuneStrArray runes;
+    DecodeUTF8RunesInString(s, runes);
+  }
+}

package/test/load_dict_test.js

@@ -0,0 +1,14 @@
+var should = require("should");
+var nodejieba = require("../index.js");
+
+describe("nodejieba", function() {
+  var userDict = __dirname + '/testdata/userdict.utf8';
+  nodejieba.load({
+    userDict: userDict,
+  });
+  it('nodejieba.cut("红掌拨清波")', function() {
+    nodejieba.cut("红掌拨清波").should.eql([
+      '红掌拨清波',
+    ]);
+  });
+});

package/test/missing_binding_test.js

@@ -0,0 +1,42 @@
+var fs = require("fs");
+var os = require("os");
+var path = require("path");
+var should = require("should");
+
+describe("nodejieba missing binding", function() {
+  it("loads without the native binding until first use", function() {
+    var fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nodejieba-missing-binding-"));
+    try {
+      var fixtureIndex = path.join(fixtureDir, "index.js");
+
+      fs.copyFileSync(path.join(__dirname, "..", "index.js"), fixtureIndex);
+
+      var nodejieba = require(fixtureIndex);
+
+      should(nodejieba.DEFAULT_DICT).type("string");
+      should(nodejieba.cut).type("function");
+
+      var firstError;
+      var secondError;
+
+      try {
+        nodejieba.cut("南京市长江大桥");
+      } catch (err) {
+        firstError = err;
+      }
+
+      try {
+        nodejieba.cut("南京市长江大桥");
+      } catch (err) {
+        secondError = err;
+      }
+
+      should(firstError).be.an.Error();
+      firstError.message.should.match(/nodejieba native binding was not found/);
+      firstError.code.should.equal("BINDING_NOT_FOUND");
+      should(secondError).equal(firstError);
+    } finally {
+      fs.rmSync(fixtureDir, { recursive: true, force: true });
+    }
+  });
+});