cppjieba_rb 0.4.1 → 0.4.4

data/ext/cppjieba/deps/limonp/test/unittest/TStringUtil.cpp

@@ -0,0 +1,183 @@
+#include "limonp/StringUtil.hpp"
+#include "gtest/gtest.h"
+using namespace limonp;
+
+TEST(StringUtilTest, Test1) {
+  vector<string> vec;
+  string s;
+  Split("\t1\t3\t4\t", vec, "\t");
+  ASSERT_EQ(s << vec, "[\"\", \"1\", \"3\", \"4\"]");
+  s = " \t\n ni hao ad \r\n";
+  ASSERT_EQ("ni hao ad", Trim(s));
+  ASSERT_EQ("select * from table1  limit 1;" ,StringFormat("select %s from %s  %s;", "*","table1","limit 1"));
+  s = StringFormat("select %s from %s  %s;", "*","table1","limit 1");
+  ASSERT_EQ("select * from table1  limit 1;" ,s);
+  vec.clear();
+  vec.push_back("1");
+  vec.push_back("2");
+  vec.push_back("3");
+  s.clear();
+  Join(vec.begin(), vec.end(), s,",");
+  ASSERT_EQ("1,2,3",s);
+  s = Join(vec.begin(), vec.end(), "..");
+  ASSERT_EQ("1..2..3", s);
+  const char* arr[] = {"2","3","5"};
+  ASSERT_EQ("2,3,5", Join(arr, arr + sizeof(arr)/sizeof(arr[0]), ","));
+  map<string , int> mp;
+  mp["key1"] =2;
+  ASSERT_EQ("{key1:2}", s <<  mp);
+  std::unordered_map<int,int> hmp;
+  hmp[1]=2;
+  ASSERT_EQ("{1:2}", s <<  hmp);
+}
+
+TEST(StringUtilTest, Test2) {
+  string s, gbks;
+  ifstream ifs("../test/testdata/dict.gbk");
+  ASSERT_TRUE(!!ifs);
+
+  vector<uint16_t> uni;
+  while(getline(ifs, s)) {
+    GBKTrans(s, uni);
+    GBKTrans(uni.begin(), uni.end(), gbks);
+    ASSERT_EQ(s, gbks);
+  }
+}
+
+TEST(StringUtilTest, Test3) {
+  string s, utf8;
+  ifstream ifs("../test/testdata/dict.utf8");
+  ASSERT_TRUE(!!ifs);
+
+  vector<uint16_t> uni;
+  while(getline(ifs, s)) {
+    ASSERT_TRUE(Utf8ToUnicode(s, uni));
+    UnicodeToUtf8(uni.begin(), uni.end(), utf8);
+    ASSERT_EQ(s, utf8);
+  }
+}
+
+TEST(StringUtilTest, Test4) {
+  //ASSERT_TRUE(StartsWith("--help",NULL));
+  ASSERT_TRUE(StartsWith("--help","--"));
+  ASSERT_TRUE(StartsWith("--help","-"));
+  ASSERT_FALSE(StartsWith("--help","he"));
+  ASSERT_TRUE(StartsWith("help","help"));
+  ASSERT_FALSE(StartsWith("","help"));
+  ASSERT_TRUE(StartsWith("hel",""));
+  ASSERT_TRUE(EndsWith("hel",""));
+  ASSERT_TRUE(EndsWith("hel","el"));
+}
+
+TEST(StringUtilTest, Test5) {
+  const char* str = "1,2,3,4";
+  vector<string> vec;
+  string res;
+  Split(str, vec, ",");
+  ASSERT_EQ("[\"1\", \"2\", \"3\", \"4\"]", res << vec);
+  Split("1,2,3,4,", vec, ",");
+  ASSERT_EQ("[\"1\", \"2\", \"3\", \"4\"]", res << vec);
+  Split(str, vec, ",", 3);
+  ASSERT_EQ("[\"1\", \"2\", \"3\", \"4\"]", res << vec);
+
+  Split("1", vec, ",");
+  ASSERT_EQ("[\"1\"]", res << vec);
+
+  Split(str, vec, ",", 1);
+  ASSERT_EQ("[\"1\", \"2,3,4\"]", res << vec);
+
+  Split("", vec, ",");
+  ASSERT_EQ("[]", res << vec);
+  
+  Split("1, 2", vec, ",");
+  ASSERT_EQ("[\"1\", \" 2\"]", res << vec);
+
+  Split("1==2", vec, "==");
+  ASSERT_EQ("[\"1\", \"\", \"2\"]", res << vec);
+
+  Split("1,", vec, ",");
+  ASSERT_EQ("[\"1\"]", res << vec);
+
+  Split(",1,", vec, ",");
+  ASSERT_EQ("[\"\", \"1\"]", res << vec);
+
+  Split("1, ", vec, ",");
+  ASSERT_EQ("[\"1\", \" \"]", res << vec);
+
+  res << Split("1|2,3", "|,");
+  ASSERT_EQ("[\"1\", \"2\", \"3\"]", res);
+}
+
+TEST(StringUtilTest, Trim) {
+  string s;
+  s = "xxxyyyxx";
+  ASSERT_EQ(RTrim(s, 'x'), "xxxyyy");
+  ASSERT_EQ(LTrim(s, 'x'), "yyy");
+  s = "xxxyyyxx";
+  ASSERT_EQ(Trim(s, 'x'), "yyy");
+
+  s = " x y ";
+  ASSERT_EQ(Trim(s), "x y");
+
+  // check if it core dump when using isalpha
+  wchar_t w = 1000024;
+  ASSERT_FALSE(IsSpace(w));
+  w = 0x20;
+  ASSERT_TRUE(IsSpace(w));
+}
+
+TEST(StringUtilTest, GetTime) {
+  string s;
+  GetTime("%Y-%m-%d %H:%M:%S", s);
+  //print(s);
+}
+
+TEST(StringUtilTest, PathJoin) {
+  const char * path1 = "/home/foo/dir";
+  const char * path2 = "file";
+  const char * path3 = "/home/foo/dir/";
+  const char * path4 = "file";
+  const char * answer = "/home/foo/dir/file";
+
+  ASSERT_EQ(answer, PathJoin(path1, path2));
+  ASSERT_EQ(answer, PathJoin(path3, path4));
+}
+
+TEST(StringUtilTest, JapaneseUnicode) {
+  // Japanese
+  const char* s = "がんば";
+  vector<uint16_t> unicode;
+  ASSERT_TRUE(Utf8ToUnicode(s, unicode));
+  ASSERT_EQ(3u, unicode.size());
+}
+
+TEST(StringUtilTest, RareChinese) {
+  //U+10000 – U+10FFFF
+  const char* s = "𪚥";
+  vector<uint16_t> unicode;
+  ASSERT_FALSE(Utf8ToUnicode(s, unicode));
+  ASSERT_EQ(0u, unicode.size());
+}
+
+TEST(StringUtilTest, RareChineseUnicode32) {
+  //U+10000 – U+10FFFF
+  const char* s = "𪚥";
+  vector<uint32_t> unicode;
+  ASSERT_TRUE(Utf8ToUnicode32(s, unicode));
+  ASSERT_EQ(1u, unicode.size());
+
+  string s2;
+  Unicode32ToUtf8(unicode.begin(), unicode.end(), s2);
+  ASSERT_EQ(s2, s);
+}
+
+TEST(StringUtilTest, Unicode32) {
+  const char* s = "1+1=2你好世界，。";
+  vector<uint32_t> unicode;
+  ASSERT_TRUE(Utf8ToUnicode32(s, unicode));
+  ASSERT_EQ(unicode.size(), 11u);
+
+  string s2;
+  Unicode32ToUtf8(unicode.begin(), unicode.end(), s2);
+  ASSERT_EQ(s2, s);
+}

data/ext/cppjieba/include/cppjieba/DictTrie.hpp

@@ -61,6 +61,15 @@ class DictTrie {
     return true;
   }
 
+  bool DeleteUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
+    DictUnit node_info;
+    if (!MakeNodeInfo(node_info, word, user_word_default_weight_, tag)) {
+      return false;
+    }
+    trie_->DeleteNode(node_info.word, &node_info);
+    return true;
+  }
+  
   const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
     return trie_->Find(begin, end);
   }

data/ext/cppjieba/include/cppjieba/Jieba.hpp

@@ -76,6 +76,10 @@ class Jieba {
     return dict_trie_.InsertUserWord(word,freq, tag);
   }
 
+  bool DeleteUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.DeleteUserWord(word, tag);
+  }
+  
   bool Find(const string& word)
   {
     return dict_trie_.Find(word);

data/ext/cppjieba/include/cppjieba/Trie.hpp

@@ -141,7 +141,33 @@ class Trie {
     assert(ptNode != NULL);
     ptNode->ptValue = ptValue;
   }
-
+  void DeleteNode(const Unicode& key, const DictUnit* ptValue) {
+      if (key.begin() == key.end()) {
+        return;
+      }
+      //定义一个NextMap迭代器
+      TrieNode::NextMap::const_iterator kmIter;
+      //定义一个指向root的TrieNode指针
+      TrieNode *ptNode = root_;
+      for (Unicode::const_iterator citer = key.begin(); citer != key.end(); ++citer) {
+        //链表不存在元素
+        if (NULL == ptNode->next) {
+          return;
+        }
+        kmIter = ptNode->next->find(*citer);
+        //如果map中不存在,跳出循环
+        if (ptNode->next->end() == kmIter) {
+              break;
+        }
+        //从unordered_map中擦除该项
+        ptNode->next->erase(*citer);
+        //删除该node
+        ptNode = kmIter->second;
+        delete ptNode;
+        break;
+      }
+      return;
+ }
  private:
   void CreateTrie(const vector<Unicode>& keys, const vector<const DictUnit*>& valuePointers) {
     if (valuePointers.empty() || keys.empty()) {

data/ext/cppjieba/test/CMakeLists.txt

@@ -1,5 +1,6 @@
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR})
 
-ADD_EXECUTABLE(demo demo.cpp)
-ADD_EXECUTABLE(load_test load_test.cpp)
-ADD_SUBDIRECTORY(unittest)
+if(NOT MSVC)
+	ADD_EXECUTABLE(load_test load_test.cpp)
+	ADD_SUBDIRECTORY(unittest)
+endif()

data/ext/cppjieba/test/unittest/CMakeLists.txt

@@ -1,8 +1,21 @@
+if (MSVC)
+	set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreadedDebugDLL")
+	set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
+endif()
+
+include(FetchContent)
+
+FetchContent_Declare(
+  googletest
+  GIT_REPOSITORY https://github.com/google/googletest.git
+  GIT_TAG release-1.11.0
+)
+FetchContent_MakeAvailable(googletest)
+
+
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/test)
 SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
 
-INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/deps/gtest/include)
-
 ADD_DEFINITIONS(-DLOGGING_LEVEL=LL_WARNING)
 
 ADD_EXECUTABLE(test.run 
@@ -17,8 +30,4 @@ ADD_EXECUTABLE(test.run
     textrank_test.cpp
 )
 
-if(MSVC)
-	TARGET_LINK_LIBRARIES(test.run gtest)
-else()
-	TARGET_LINK_LIBRARIES(test.run gtest pthread)
-endif()
+TARGET_LINK_LIBRARIES(test.run gtest)

data/ext/cppjieba_rb/extconf.rb

@@ -1,5 +1,7 @@
-require "mkmf"
-abs = File.expand_path File.dirname(__FILE__)
+# frozen_string_literal: true
+
+require 'mkmf'
+abs = __dir__
 
 LIBDIR      = RbConfig::CONFIG['libdir']
 INCLUDEDIR  = RbConfig::CONFIG['includedir']
@@ -7,17 +9,20 @@ INCLUDEDIR  = RbConfig::CONFIG['includedir']
 HEADER_DIRS = [
   INCLUDEDIR,
   "#{abs}/../cppjieba/include",
-  "#{abs}/../cppjieba/deps"
-]
+  "#{abs}/../cppjieba/deps/limonp/include"
+].freeze
 
 LIB_DIRS = [
   LIBDIR
-]
+].freeze
 
 dir_config('cppjieba_rb', HEADER_DIRS, LIB_DIRS)
 
-CONFIG["CXXFLAGS"] += " -std=c++11 -O3"
+# rubocop:disable Style/GlobalVars
+CONFIG['CXXFLAGS'] += ' -std=c++11 -O3'
 $CXXFLAGS = "#{$CXXFLAGS} -std=c++11 -O3"
+# rubocop:enable Style/GlobalVars
+
 create_makefile 'cppjieba_rb/cppjieba_rb'
 # respect header changes
 headers = Dir.glob('*.{hpp,h}').join ' '

data/ext/cppjieba_rb/internal.cc

@@ -163,7 +163,7 @@ void Init_internal()
     rb_sFull = rb_intern("full");
     u8_enc = rb_utf8_encoding();
 
-    rb_cCppjiebaRb_Internal = rb_define_class_under(rb_mCppjiebaRb, "Internal", rb_cData);
+    rb_cCppjiebaRb_Internal = rb_define_class_under(rb_mCppjiebaRb, "Internal", rb_cObject);
     rb_define_alloc_func(rb_cCppjiebaRb_Internal, internal_alloc);
     rb_define_method(rb_cCppjiebaRb_Internal, "initialize", (ruby_method*) &internal_initialize, 5);
     rb_define_method(rb_cCppjiebaRb_Internal, "extract_keyword", (ruby_method*) &internal_extract_keyword, 2);

data/lib/cppjieba_rb/segment.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 module CppjiebaRb
+  # Sentence segmentation
   class Segment
     VALID_MODES = %i[mix hmm mp query full].freeze
 
@@ -17,4 +20,4 @@ module CppjiebaRb
       CppjiebaRb.internal.segment(str, @mode, @max_word_length, @hmm)
     end
   end
-end
+end

data/lib/cppjieba_rb/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module CppjiebaRb
-  VERSION = '0.4.1'
+  VERSION = '0.4.4'
 end

data/lib/cppjieba_rb.rb

@@ -1,7 +1,14 @@
+# frozen_string_literal: true
+
 require 'cppjieba_rb/cppjieba_rb'
 require 'cppjieba_rb/version'
 require 'cppjieba_rb/segment'
 
+# CppjiebaRb segments a Chinese sentence into words.
+#
+# Available segmentation methods include HMM, MP, and mix mode.
+# Dictionaries takes a strong part in CppjiebaRb's accuracy.
+# Read more https://github.com/yanyiwu/cppjieba
 module CppjiebaRb
   EXT_BASE = File.join(File.dirname(__FILE__), '..', 'ext', 'cppjieba', 'dict')
   DICT_PATH = File.join(EXT_BASE, 'jieba.dict.utf8')
@@ -28,11 +35,11 @@ module CppjiebaRb
 
   class << self
     def internal
-      @backend ||= CppjiebaRb::Internal.new(DICT_PATH,
-                                            HMM_DICT_PATH,
-                                            USER_DICT,
-                                            IDF_PATH,
-                                            STOP_WORD_PATH)
+      @internal ||= CppjiebaRb::Internal.new(DICT_PATH,
+                                             HMM_DICT_PATH,
+                                             USER_DICT,
+                                             IDF_PATH,
+                                             STOP_WORD_PATH)
     end
   end
 end

data/test/test_keyword.rb

@@ -1,17 +1,17 @@
-# coding: utf-8
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'cppjieba_rb'
 
 class JiebaTest < Minitest::Test
   def test_keywords
-    results = CppjiebaRb.extract_keyword "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。", 5
+    results = CppjiebaRb.extract_keyword '我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。', 5
 
-    assert_equal [["CEO",
+    assert_equal [['CEO',
                    11.739204307083542],
-                  ["升职", 10.8561552143],
-                  ["加薪", 10.642581114],
-                  ["手扶拖拉机", 10.0088573539],
-                  ["巅峰", 9.49395840471]], results
-
+                  ['升职', 10.8561552143],
+                  ['加薪', 10.642581114],
+                  ['手扶拖拉机', 10.0088573539],
+                  ['巅峰', 9.49395840471]], results
   end
 end

data/test/test_segment.rb

@@ -1,24 +1,28 @@
-# coding: utf-8
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'cppjieba_rb'
 
 class JiebaTest < Minitest::Test
   def test_mix_segment
-    words = CppjiebaRb.segment "我来到南京市长江大桥"
-    assert_equal %w(我 来到 南京市 长江大桥), words
+    words = CppjiebaRb.segment '我来到南京市长江大桥'
+
+    assert_equal %w[我 来到 南京市 长江大桥], words
+
+    words = CppjiebaRb.segment '令狐冲是云计算行业的专家'
 
-    words = CppjiebaRb.segment "令狐冲是云计算行业的专家"
-    assert_equal %w(令狐冲 是 云计算 行业 的 专家), words
+    assert_equal %w[令狐冲 是 云计算 行业 的 专家], words
   end
 
   def test_hmm_segment
-    words = CppjiebaRb.segment "令狐冲是云计算行业的专家", mode: :hmm
-    assert_equal %w(令狐冲 是 云计算 行业 的 专家), words
+    words = CppjiebaRb.segment '令狐冲是云计算行业的专家', mode: :hmm
+
+    assert_equal %w[令狐冲 是 云计算 行业 的 专家], words
   end
 
   def test_max_prob_segment
-    words = CppjiebaRb.segment "令狐冲是云计算行业的专家", mode: :mp
-    assert_equal %w(令狐冲 是 云计算 行业 的 专家), words
-  end
+    words = CppjiebaRb.segment '令狐冲是云计算行业的专家', mode: :mp
 
+    assert_equal %w[令狐冲 是 云计算 行业 的 专家], words
+  end
 end

data/test/test_stop_word_filter.rb

@@ -1,10 +1,12 @@
-# coding: utf-8
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'cppjieba_rb'
 
 class JiebaTest < Minitest::Test
   def test_filter
-    words = CppjiebaRb.filter_stop_word %w(令狐冲 是 云计算 行业 的 专家)
-    assert_equal %w(令狐冲 云计算 行业 专家), words
+    words = CppjiebaRb.filter_stop_word %w[令狐冲 是 云计算 行业 的 专家]
+
+    assert_equal %w[令狐冲 云计算 行业 专家], words
   end
 end

data/test/test_tagging.rb

@@ -1,9 +1,12 @@
-# coding: utf-8
+# frozen_string_literal: true
+
 require 'minitest/autorun'
 require 'cppjieba_rb'
+
 class JiebaTest < Minitest::Test
   def test_tagging
     pairs = CppjiebaRb.segment_tag '我是蓝翔技工拖拉机学院手扶拖拉机专业的。'
+
     assert_equal({ '我' => 'r', '是' => 'v', '蓝翔' => 'nz', '技工' => 'n',
                    '拖拉机' => 'n', '学院' => 'n', '手扶拖拉机' => 'n', '专业' => 'n',
                    '的' => 'uj', '。' => 'x' }, pairs)
@@ -11,9 +14,9 @@ class JiebaTest < Minitest::Test
 
   def test_tagging_with_user_dict
     pairs = CppjiebaRb.segment_tag '我是蓝翔技工拖拉机学院手扶拖拉机专业的。'
+
     assert_equal({ '我' => 'r', '是' => 'v', '蓝翔' => 'nz', '技工' => 'n',
                    '拖拉机' => 'n', '学院' => 'n', '手扶拖拉机' => 'n', '专业' => 'n',
                    '的' => 'uj', '。' => 'x' }, pairs)
   end
-
 end