cppjieba_rb 0.4.0 → 0.4.1

data/ext/cppjieba/test/unittest/trie_test.cpp

@@ -0,0 +1,177 @@
+#include "cppjieba/DictTrie.hpp"
+#include "cppjieba/MPSegment.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+static const char* const DICT_FILE = "../test/testdata/extra_dict/jieba.dict.small.utf8";
+
+TEST(TrieTest, Empty) {
+  vector<Unicode> keys;
+  vector<const DictUnit*> values;
+  Trie trie(keys, values);
+}
+
+TEST(TrieTest, Construct) {
+  vector<Unicode> keys;
+  vector<const DictUnit*> values;
+  keys.push_back(DecodeRunesInString("你"));
+  values.push_back((const DictUnit*)(NULL));
+  Trie trie(keys, values);
+}
+
+TEST(DictTrieTest, NewAndDelete) {
+  DictTrie * trie;
+  trie = new DictTrie(DICT_FILE);
+  delete trie;
+}
+
+TEST(DictTrieTest, Test1) {
+  string s1, s2;
+  DictTrie trie(DICT_FILE);
+  ASSERT_LT(trie.GetMinWeight() + 15.6479, 0.001);
+  string word("来到");
+  cppjieba::RuneStrArray uni;
+  ASSERT_TRUE(DecodeRunesInString(word, uni));
+  //DictUnit nodeInfo;
+  //nodeInfo.word = uni;
+  //nodeInfo.tag = "v";
+  //nodeInfo.weight = -8.87033;
+  //s1 << nodeInfo;
+  //s2 << (*trie.Find(uni.begin(), uni.end()));
+  const DictUnit* du = trie.Find(uni.begin(), uni.end());
+  ASSERT_TRUE(du != NULL);
+  ASSERT_EQ(2u, du->word.size());
+  ASSERT_EQ(26469u, du->word[0]);
+  ASSERT_EQ(21040u, du->word[1]);
+  ASSERT_EQ("v", du->tag);
+  ASSERT_NEAR(-8.870, du->weight, 0.001);
+
+  //EXPECT_EQ("[\"26469\", \"21040\"] v -8.870", s2);
+  word = "清华大学";
+  LocalVector<pair<size_t, const DictUnit*> > res;
+  const char * words[] = {"清", "清华", "清华大学"};
+  for (size_t i = 0; i < sizeof(words)/sizeof(words[0]); i++) {
+    ASSERT_TRUE(DecodeRunesInString(words[i], uni));
+    res.push_back(make_pair(uni.size() - 1, trie.Find(uni.begin(), uni.end())));
+    //resMap[uni.size() - 1] = trie.Find(uni.begin(), uni.end());
+  }
+  vector<pair<size_t, const DictUnit*> > vec;
+  vector<struct Dag> dags;
+  ASSERT_TRUE(DecodeRunesInString(word, uni));
+  trie.Find(uni.begin(), uni.end(), dags);
+  ASSERT_EQ(dags.size(), uni.size());
+  ASSERT_NE(dags.size(), 0u);
+  s1 << res;
+  s2 << dags[0].nexts;
+  ASSERT_EQ(s1, s2);
+  
+}
+
+TEST(DictTrieTest, UserDict) {
+  DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
+  string word = "云计算";
+  cppjieba::RuneStrArray unicode;
+  ASSERT_TRUE(DecodeRunesInString(word, unicode));
+  const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit != NULL);
+  ASSERT_NEAR(unit->weight, -14.100, 0.001);
+
+  word = "蓝翔";
+  ASSERT_TRUE(DecodeRunesInString(word, unicode));
+  unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit != NULL);
+  ASSERT_EQ(unit->tag, "nz");
+  ASSERT_NEAR(unit->weight, -14.100, 0.001);
+
+  word = "区块链";
+  ASSERT_TRUE(DecodeRunesInString(word, unicode));
+  unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit != NULL);
+  ASSERT_EQ(unit->tag, "nz");
+  ASSERT_NEAR(unit->weight, -15.6478, 0.001);
+}
+
+TEST(DictTrieTest, UserDictWithMaxWeight) {
+  DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8", DictTrie::WordWeightMax);
+  string word = "云计算";
+  cppjieba::RuneStrArray unicode;
+  ASSERT_TRUE(DecodeRunesInString(word, unicode));
+  const DictUnit * unit = trie.Find(unicode.begin(), unicode.end());
+  ASSERT_TRUE(unit);
+  ASSERT_NEAR(unit->weight, -2.975, 0.001);
+}
+
+TEST(DictTrieTest, Dag) {
+  DictTrie trie(DICT_FILE, "../test/testdata/userdict.utf8");
+
+  {
+    string word = "清华大学";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeRunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res);
+
+    size_t nexts_sizes[] = {3, 2, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "北京邮电大学";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeRunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res);
+
+    size_t nexts_sizes[] = {3, 1, 2, 2, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "长江大桥";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeRunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res);
+
+    size_t nexts_sizes[] = {3, 1, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "长江大桥";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeRunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res, 3);
+
+    size_t nexts_sizes[] = {2, 1, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+
+  {
+    string word = "长江大桥";
+    cppjieba::RuneStrArray unicode;
+    ASSERT_TRUE(DecodeRunesInString(word, unicode));
+    vector<struct Dag> res;
+    trie.Find(unicode.begin(), unicode.end(), res, 4);
+
+    size_t nexts_sizes[] = {3, 1, 2, 1};
+    ASSERT_EQ(res.size(), sizeof(nexts_sizes)/sizeof(nexts_sizes[0]));
+    for (size_t i = 0; i < res.size(); i++) {
+      ASSERT_EQ(res[i].nexts.size(), nexts_sizes[i]);
+    }
+  }
+}

data/ext/cppjieba/test/unittest/unicode_test.cpp

@@ -0,0 +1,43 @@
+#include "cppjieba/Unicode.hpp"
+#include "limonp/StdExtension.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+using namespace std;
+
+TEST(UnicodeTest, Test1) {
+  string s = "你好世界";
+  RuneStrArray runes;
+  ASSERT_TRUE(DecodeRunesInString(s, runes));
+  string actual;
+  string expected = "[\"{\"rune\": \"20320\", \"offset\": 0, \"len\": 3}\", \"{\"rune\": \"22909\", \"offset\": 3, \"len\": 3}\", \"{\"rune\": \"19990\", \"offset\": 6, \"len\": 3}\", \"{\"rune\": \"30028\", \"offset\": 9, \"len\": 3}\"]";
+  actual << runes;
+  ASSERT_EQ(expected, actual);
+}
+
+TEST(UnicodeTest, Illegal) {
+  string s = "123\x80";
+  RuneStrArray runes;
+  ASSERT_FALSE(DecodeRunesInString(s, runes));
+  string actual;
+  string expected = "[]";
+  actual << runes;
+  ASSERT_EQ(expected, actual);
+}
+
+TEST(UnicodeTest, Rand) {
+  const size_t ITERATION = 1024;
+  const size_t MAX_LEN = 256;
+  string s;
+  srand(time(NULL));
+
+  for (size_t i = 0; i < ITERATION; i++) {
+    size_t len = rand() % MAX_LEN;
+    s.resize(len);
+    for (size_t j = 0; j < len; j++) {
+      s[rand() % len] = rand();
+    }
+    RuneStrArray runes;
+    DecodeRunesInString(s, runes);
+  }
+}

data/lib/cppjieba_rb/version.rb

@@ -1,3 +1,3 @@
 module CppjiebaRb
-  VERSION = '0.4.0'
+  VERSION = '0.4.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cppjieba_rb
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Erick Guan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-02 00:00:00.000000000 Z
+date: 2021-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -88,6 +88,129 @@ files:
 - README.md
 - Rakefile
 - cppjieba_rb.gemspec
+- ext/cppjieba/.gitignore
+- ext/cppjieba/.travis.yml
+- ext/cppjieba/CMakeLists.txt
+- ext/cppjieba/ChangeLog.md
+- ext/cppjieba/README.md
+- ext/cppjieba/README_EN.md
+- ext/cppjieba/appveyor.yml
+- ext/cppjieba/deps/CMakeLists.txt
+- ext/cppjieba/deps/gtest/CMakeLists.txt
+- ext/cppjieba/deps/gtest/include/gtest/gtest-death-test.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest-message.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest-param-test.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest-param-test.h.pump
+- ext/cppjieba/deps/gtest/include/gtest/gtest-printers.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest-spi.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest-test-part.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest-typed-test.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest_pred_impl.h
+- ext/cppjieba/deps/gtest/include/gtest/gtest_prod.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-death-test-internal.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-filepath.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-internal.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-linked_ptr.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-param-util-generated.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-param-util-generated.h.pump
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-param-util.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-port.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-string.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-tuple.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-tuple.h.pump
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-type-util.h
+- ext/cppjieba/deps/gtest/include/gtest/internal/gtest-type-util.h.pump
+- ext/cppjieba/deps/gtest/src/.deps/.dirstamp
+- ext/cppjieba/deps/gtest/src/.deps/gtest-all.Plo
+- ext/cppjieba/deps/gtest/src/.deps/gtest_main.Plo
+- ext/cppjieba/deps/gtest/src/.dirstamp
+- ext/cppjieba/deps/gtest/src/gtest-all.cc
+- ext/cppjieba/deps/gtest/src/gtest-death-test.cc
+- ext/cppjieba/deps/gtest/src/gtest-filepath.cc
+- ext/cppjieba/deps/gtest/src/gtest-internal-inl.h
+- ext/cppjieba/deps/gtest/src/gtest-port.cc
+- ext/cppjieba/deps/gtest/src/gtest-printers.cc
+- ext/cppjieba/deps/gtest/src/gtest-test-part.cc
+- ext/cppjieba/deps/gtest/src/gtest-typed-test.cc
+- ext/cppjieba/deps/gtest/src/gtest.cc
+- ext/cppjieba/deps/gtest/src/gtest_main.cc
+- ext/cppjieba/deps/limonp/ArgvContext.hpp
+- ext/cppjieba/deps/limonp/BlockingQueue.hpp
+- ext/cppjieba/deps/limonp/BoundedBlockingQueue.hpp
+- ext/cppjieba/deps/limonp/BoundedQueue.hpp
+- ext/cppjieba/deps/limonp/Closure.hpp
+- ext/cppjieba/deps/limonp/Colors.hpp
+- ext/cppjieba/deps/limonp/Condition.hpp
+- ext/cppjieba/deps/limonp/Config.hpp
+- ext/cppjieba/deps/limonp/FileLock.hpp
+- ext/cppjieba/deps/limonp/ForcePublic.hpp
+- ext/cppjieba/deps/limonp/LocalVector.hpp
+- ext/cppjieba/deps/limonp/Logging.hpp
+- ext/cppjieba/deps/limonp/Md5.hpp
+- ext/cppjieba/deps/limonp/MutexLock.hpp
+- ext/cppjieba/deps/limonp/NonCopyable.hpp
+- ext/cppjieba/deps/limonp/StdExtension.hpp
+- ext/cppjieba/deps/limonp/StringUtil.hpp
+- ext/cppjieba/deps/limonp/Thread.hpp
+- ext/cppjieba/deps/limonp/ThreadPool.hpp
+- ext/cppjieba/dict/README.md
+- ext/cppjieba/dict/hmm_model.utf8
+- ext/cppjieba/dict/idf.utf8
+- ext/cppjieba/dict/jieba.dict.utf8
+- ext/cppjieba/dict/pos_dict/char_state_tab.utf8
+- ext/cppjieba/dict/pos_dict/prob_emit.utf8
+- ext/cppjieba/dict/pos_dict/prob_start.utf8
+- ext/cppjieba/dict/pos_dict/prob_trans.utf8
+- ext/cppjieba/dict/stop_words.utf8
+- ext/cppjieba/dict/user.dict.utf8
+- ext/cppjieba/include/cppjieba/DictTrie.hpp
+- ext/cppjieba/include/cppjieba/FullSegment.hpp
+- ext/cppjieba/include/cppjieba/HMMModel.hpp
+- ext/cppjieba/include/cppjieba/HMMSegment.hpp
+- ext/cppjieba/include/cppjieba/Jieba.hpp
+- ext/cppjieba/include/cppjieba/KeywordExtractor.hpp
+- ext/cppjieba/include/cppjieba/MPSegment.hpp
+- ext/cppjieba/include/cppjieba/MixSegment.hpp
+- ext/cppjieba/include/cppjieba/PosTagger.hpp
+- ext/cppjieba/include/cppjieba/PreFilter.hpp
+- ext/cppjieba/include/cppjieba/QuerySegment.hpp
+- ext/cppjieba/include/cppjieba/SegmentBase.hpp
+- ext/cppjieba/include/cppjieba/SegmentTagged.hpp
+- ext/cppjieba/include/cppjieba/TextRankExtractor.hpp
+- ext/cppjieba/include/cppjieba/Trie.hpp
+- ext/cppjieba/include/cppjieba/Unicode.hpp
+- ext/cppjieba/test/CMakeLists.txt
+- ext/cppjieba/test/demo.cpp
+- ext/cppjieba/test/load_test.cpp
+- ext/cppjieba/test/testdata/curl.res
+- ext/cppjieba/test/testdata/extra_dict/jieba.dict.small.utf8
+- ext/cppjieba/test/testdata/gbk_dict/hmm_model.gbk
+- ext/cppjieba/test/testdata/gbk_dict/jieba.dict.gbk
+- ext/cppjieba/test/testdata/jieba.dict.0.1.utf8
+- ext/cppjieba/test/testdata/jieba.dict.0.utf8
+- ext/cppjieba/test/testdata/jieba.dict.1.utf8
+- ext/cppjieba/test/testdata/jieba.dict.2.utf8
+- ext/cppjieba/test/testdata/load_test.urls
+- ext/cppjieba/test/testdata/review.100
+- ext/cppjieba/test/testdata/review.100.res
+- ext/cppjieba/test/testdata/server.conf
+- ext/cppjieba/test/testdata/testlines.gbk
+- ext/cppjieba/test/testdata/testlines.utf8
+- ext/cppjieba/test/testdata/userdict.2.utf8
+- ext/cppjieba/test/testdata/userdict.english
+- ext/cppjieba/test/testdata/userdict.utf8
+- ext/cppjieba/test/testdata/weicheng.utf8
+- ext/cppjieba/test/unittest/CMakeLists.txt
+- ext/cppjieba/test/unittest/gtest_main.cpp
+- ext/cppjieba/test/unittest/jieba_test.cpp
+- ext/cppjieba/test/unittest/keyword_extractor_test.cpp
+- ext/cppjieba/test/unittest/pos_tagger_test.cpp
+- ext/cppjieba/test/unittest/pre_filter_test.cpp
+- ext/cppjieba/test/unittest/segments_test.cpp
+- ext/cppjieba/test/unittest/textrank_test.cpp
+- ext/cppjieba/test/unittest/trie_test.cpp
+- ext/cppjieba/test/unittest/unicode_test.cpp
 - ext/cppjieba_rb/cppjieba_rb.c
 - ext/cppjieba_rb/extconf.rb
 - ext/cppjieba_rb/internal.cc