cppjieba_rb 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4ab9b4adf8e5ddccbe36570f3492b4ae438c7b6a
-  data.tar.gz: 72987214a678eaf3505f804bd7c9db86f94c6178
+  metadata.gz: 8c0c15dd47a0f3fe66f57e4f7d37fd5eca5c2959
+  data.tar.gz: 204f0898edcbe685ec297a4259c5a8bdac29eb3a
 SHA512:
-  metadata.gz: 1cd2f4cdd703464e0672ea034ae5bc875d4754ce3e45d859b66d2cf65759e32bbca499221ef4ca70239bcf3606eafeee13b9c6723d5a626a8aa2bf06fdfc970e
-  data.tar.gz: 796eacaca074e602a080a94a2532e4f051e2b9e874ee6412cf45b683522afc36f2ba219be6a375c728ce6f3eb9b225c7fff783643cac3732dd4992bf263a1e4a
+  metadata.gz: 9697b467d66427704cbb1b2944c0ac0f04b66153d704f3efa72cf0f6b226fc50c3510b688abf267b5aa4678d261ce18baf491713d39ebe6b2be9de6f2c33c3d5
+  data.tar.gz: 7bb56503a56bb7d305eecea1d4671365fc6c736204256e93d2113adf6f43344bab9cdf2b8122f80e0ef7bb42eb2ed01776070e235b81a902caabe89f341f869c

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
 rvm:
-  - 2.3.3
-  - 2.4.1
+  - 2.3
+  - 2.4
+  - 2.5
   - ruby-head
   - rbx-2
 

data/README.md

@@ -4,10 +4,12 @@
 
 [![Build Status](https://travis-ci.org/fantasticfears/cppjieba_rb.png?branch=master)](https://travis-ci.org/fantasticfears/cppjieba_rb)
 
-+[![Patreon](https://img.shields.io/badge/back_on-patreon-red.svg)](https://www.patreon.com/fantasticfears)
+[![Patreon](https://img.shields.io/badge/back_on-patreon-red.svg)](https://www.patreon.com/fantasticfears)
 
 Ruby bindings for [Cppjieba](https://github.com/yanyiwu/cppjieba). C++11 required. (gcc 4.8+)
 
+The TRIE tree has high memory usage. For default dict, it uses ~120 MB memory.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -28,7 +30,7 @@ Mix Segment mode (HMM with Max Prob, default):
 
 ```ruby
 require 'cppjieba_rb'
-seg = CppjiebaRb::Segment.new  # equivalent to "JiebaRb::Segment.new mode: :mix"
+seg = CppjiebaRb::Segment.new  # equivalent to "CppjiebaRb::Segment.new mode: :mix"
 words = seg.segment "令狐冲是云计算行业的专家"
 # 令狐冲 是 云 计算 行业 的 专家
 ```
@@ -72,7 +74,7 @@ CppjiebaRb.extract_keyword "我是拖拉机学院手扶拖拉机专业的。不
 
 ## Contributing
 
-1. Fork it ( http://github.com/<my-github-username>/cppjieba_rb/fork )
+1. Fork it ( http://github.com/fantasticfears/cppjieba_rb/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
@@ -81,4 +83,4 @@ CppjiebaRb.extract_keyword "我是拖拉机学院手扶拖拉机专业的。不
 ## TODO
 
 - including 367w dict and provide the option for it.
-- cppjieba implements trie tree, it's memory consuming
+- cppjieba implements trie tree, it's memory consuming

data/Rakefile

@@ -2,7 +2,7 @@ require "bundler/gem_tasks"
 require 'rake/testtask'
 require 'rake/extensiontask'
 
-gem = Gem::Specification.load(File.dirname(__FILE__) + '/cppjieba_rb.gemspec')
+gem = Gem::Specification.load(File.dirname(__FILE__) + '/cppjieba-rb.gemspec.gemspec')
 Rake::ExtensionTask.new('cppjieba_rb', gem) do |ext|
   ext.lib_dir = File.join('lib', 'cppjieba_rb')
 end

data/ext/cppjieba/.travis.yml

@@ -1,7 +1,6 @@
 language: cpp
 before_install:
   - if [ $TRAVIS_OS_NAME == linux ]; then sudo apt-get install cmake; fi
-  - if [ $TRAVIS_OS_NAME == osx ]; then brew install cmake; fi
 script: 
   - mkdir build
   - cd build

data/ext/cppjieba/README.md

@@ -226,8 +226,11 @@ Query方法先使用Mix方法切词，对于切出来的较长的词再使用Ful
 + [gitbook-plugin-search-pro] 支持中文搜索的 gitbook 插件。
 + [ngx_http_cppjieba_module] Nginx 分词插件。
 + [cppjiebapy] 由 [jannson] 开发的供 python 模块调用的项目 [cppjiebapy], 相关讨论 [cppjiebapy_discussion] .
++ [cppjieba-py] 由 [bung87] 基于 pybind11 封装的 python 模块,使用体验上接近于原jieba。
 + [KeywordServer] 50行搭建一个中文关键词抽取服务。
 + [cppjieba-server] CppJieba HTTP 服务器。
++ [phpjieba] php版本的结巴分词扩展。
++ [perl5-jieba] Perl版本的结巴分词扩展。
 
 ## 线上演示
 
@@ -261,6 +264,8 @@ Query方法先使用Mix方法切词，对于切出来的较长的词再使用Ful
 [CppJieba]:https://github.com/yanyiwu/cppjieba
 [jannson]:https://github.com/jannson
 [cppjiebapy]:https://github.com/jannson/cppjiebapy
+[bung87]:https://github.com/bung87
+[cppjieba-py]:https://github.com/bung87/cppjieba-py
 [cppjiebapy_discussion]:https://github.com/yanyiwu/cppjieba/issues/1
 [NodeJieba]:https://github.com/yanyiwu/nodejieba
 [jiebaR]:https://github.com/qinwf/jiebaR
@@ -279,6 +284,8 @@ Query方法先使用Mix方法切词，对于切出来的较长的词再使用Ful
 [pg_jieba]:https://github.com/jaiminpan/pg_jieba
 [gitbook-plugin-search-pro]:https://plugins.gitbook.com/plugin/search-pro
 [cppjieba-server]:https://github.com/yanyiwu/cppjieba-server
+[phpjieba]:https://github.com/jonnywang/phpjieba
+[perl5-jieba]:https://metacpan.org/pod/distribution/Lingua-ZH-Jieba/lib/Lingua/ZH/Jieba.pod
 
 
 [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/yanyiwu/cppjieba/trend.png)](https://bitdeli.com/free "Bitdeli Badge")

data/ext/cppjieba/README_EN.md

@@ -86,6 +86,7 @@ Please see details in `test/demo.cpp`.
 + [ngx_http_cppjieba_module]
 + [gitbook-plugin-search-pro]
 + [cppjieba-server]
++ [perl5-jieba]
 
 ## Contact
 
@@ -109,3 +110,4 @@ Please see details in `test/demo.cpp`.
 [pg_jieba]:https://github.com/jaiminpan/pg_jieba
 [gitbook-plugin-search-pro]:https://plugins.gitbook.com/plugin/search-pro
 [cppjieba-server]:https://github.com/yanyiwu/cppjieba-server
+[perl5-jieba]:https://metacpan.org/pod/distribution/Lingua-ZH-Jieba/lib/Lingua/ZH/Jieba.pod

data/ext/cppjieba/include/cppjieba/DictTrie.hpp

@@ -50,6 +50,17 @@ class DictTrie {
     return true;
   }
 
+  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
+    DictUnit node_info;
+    double weight = freq ? log(1.0 * freq / freq_sum_) : user_word_default_weight_ ;
+    if (!MakeNodeInfo(node_info, word, weight , tag)) {
+      return false;
+    }
+    active_node_infos_.push_back(node_info);
+    trie_->InsertNode(node_info.word, &active_node_infos_.back());
+    return true;
+  }
+
   const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
     return trie_->Find(begin, end);
   }
@@ -61,6 +72,25 @@ class DictTrie {
     trie_->Find(begin, end, res, max_word_len);
   }
 
+  bool Find(const string& word)
+  {
+    const DictUnit *tmp = NULL;
+    RuneStrArray runes;
+    if (!DecodeRunesInString(word, runes))
+    {
+      XLOG(ERROR) << "Decode failed.";
+    }
+    tmp = Find(runes.begin(), runes.end());
+    if (tmp == NULL)
+    {
+      return false;
+    }
+    else
+    {
+      return true;
+    }
+  }
+
   bool IsUserDictSingleChineseWord(const Rune& word) const {
     return IsIn(user_dict_single_chinese_word_, word);
   }
@@ -69,6 +99,63 @@ class DictTrie {
     return min_weight_;
   }
 
+  void InserUserDictNode(const string& line) {
+    vector<string> buf;
+    DictUnit node_info;
+    Split(line, buf, " ");
+    if(buf.size() == 1){
+          MakeNodeInfo(node_info, 
+                buf[0], 
+                user_word_default_weight_,
+                UNKNOWN_TAG);
+        } else if (buf.size() == 2) {
+          MakeNodeInfo(node_info, 
+                buf[0], 
+                user_word_default_weight_,
+                buf[1]);
+        } else if (buf.size() == 3) {
+          int freq = atoi(buf[1].c_str());
+          assert(freq_sum_ > 0.0);
+          double weight = log(1.0 * freq / freq_sum_);
+          MakeNodeInfo(node_info, buf[0], weight, buf[2]);
+        }
+        static_node_infos_.push_back(node_info);
+        if (node_info.word.size() == 1) {
+          user_dict_single_chinese_word_.insert(node_info.word[0]);
+        }
+  }
+  
+  void LoadUserDict(const vector<string>& buf) {
+    for (size_t i = 0; i < buf.size(); i++) {
+      InserUserDictNode(buf[i]);
+    }
+  }
+
+   void LoadUserDict(const set<string>& buf) {
+    std::set<string>::const_iterator iter;
+    for (iter = buf.begin(); iter != buf.end(); iter++){
+      InserUserDictNode(*iter);
+    }
+  }
+
+  void LoadUserDict(const string& filePaths) {
+    vector<string> files = limonp::Split(filePaths, "|;");
+    size_t lineno = 0;
+    for (size_t i = 0; i < files.size(); i++) {
+      ifstream ifs(files[i].c_str());
+      XCHECK(ifs.is_open()) << "open " << files[i] << " failed"; 
+      string line;
+      
+      for (; getline(ifs, line); lineno++) {
+        if (line.size() == 0) {
+          continue;
+        }
+        InserUserDictNode(line);
+      }
+    }
+  }
+
+
  private:
   void Init(const string& dict_path, const string& user_dict_paths, UserWordWeightOption user_word_weight_opt) {
     LoadDict(dict_path);
@@ -95,45 +182,8 @@ class DictTrie {
     trie_ = new Trie(words, valuePointers);
   }
 
-  void LoadUserDict(const string& filePaths) {
-    vector<string> files = limonp::Split(filePaths, "|;");
-    size_t lineno = 0;
-    for (size_t i = 0; i < files.size(); i++) {
-      ifstream ifs(files[i].c_str());
-      XCHECK(ifs.is_open()) << "open " << files[i] << " failed"; 
-      string line;
-      DictUnit node_info;
-      vector<string> buf;
-      for (; getline(ifs, line); lineno++) {
-        if (line.size() == 0) {
-          continue;
-        }
-        buf.clear();
-        Split(line, buf, " ");
-        DictUnit node_info;
-        if(buf.size() == 1){
-          MakeNodeInfo(node_info, 
-                buf[0], 
-                user_word_default_weight_,
-                UNKNOWN_TAG);
-        } else if (buf.size() == 2) {
-          MakeNodeInfo(node_info, 
-                buf[0], 
-                user_word_default_weight_,
-                buf[1]);
-        } else if (buf.size() == 3) {
-          int freq = atoi(buf[1].c_str());
-          assert(freq_sum_ > 0.0);
-          double weight = log(1.0 * freq / freq_sum_);
-          MakeNodeInfo(node_info, buf[0], weight, buf[2]);
-        }
-        static_node_infos_.push_back(node_info);
-        if (node_info.word.size() == 1) {
-          user_dict_single_chinese_word_.insert(node_info.word[0]);
-        }
-      }
-    }
-  }
+  
+
 
   bool MakeNodeInfo(DictUnit& node_info,
         const string& word, 

data/ext/cppjieba/include/cppjieba/FullSegment.hpp

@@ -48,17 +48,17 @@ class FullSegment: public SegmentBase {
   void Cut(RuneStrArray::const_iterator begin, 
         RuneStrArray::const_iterator end, 
         vector<WordRange>& res) const {
-    //resut of searching in trie tree
+    // resut of searching in trie tree
     LocalVector<pair<size_t, const DictUnit*> > tRes;
 
-    //max index of res's words
-    int maxIdx = 0;
+    // max index of res's words
+    size_t maxIdx = 0;
 
     // always equals to (uItr - begin)
-    int uIdx = 0;
+    size_t uIdx = 0;
 
-    //tmp variables
-    int wordLen = 0;
+    // tmp variables
+    size_t wordLen = 0;
     assert(dictTrie_);
     vector<struct Dag> dags;
     dictTrie_->Find(begin, end, dags);

data/ext/cppjieba/include/cppjieba/Jieba.hpp

@@ -72,6 +72,15 @@ class Jieba {
     return dict_trie_.InsertUserWord(word, tag);
   }
 
+  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.InsertUserWord(word,freq, tag);
+  }
+
+  bool Find(const string& word)
+  {
+    return dict_trie_.Find(word);
+  }
+
   void ResetSeparators(const string& s) {
     //TODO
     mp_seg_.ResetSeparators(s);
@@ -84,10 +93,23 @@ class Jieba {
   const DictTrie* GetDictTrie() const {
     return &dict_trie_;
   } 
+  
   const HMMModel* GetHMMModel() const {
     return &model_;
   }
 
+  void LoadUserDict(const vector<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const set<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const string& path)  {
+    dict_trie_.LoadUserDict(path);
+  }
+
  private:
   DictTrie dict_trie_;
   HMMModel model_;

data/ext/cppjieba/include/cppjieba/Unicode.hpp

@@ -18,9 +18,14 @@ typedef uint32_t Rune;
 struct Word {
   string word;
   uint32_t offset;
+  uint32_t unicode_offset;
+  uint32_t unicode_length;
   Word(const string& w, uint32_t o)
    : word(w), offset(o) {
   }
+  Word(const string& w, uint32_t o, uint32_t unicode_offset, uint32_t unicode_length)
+          : word(w), offset(o), unicode_offset(unicode_offset), unicode_length(unicode_length) {
+  }
 }; // struct Word
 
 inline std::ostream& operator << (std::ostream& os, const Word& w) {
@@ -31,11 +36,16 @@ struct RuneStr {
   Rune rune;
   uint32_t offset;
   uint32_t len;
+  uint32_t unicode_offset;
+  uint32_t unicode_length;
   RuneStr(): rune(0), offset(0), len(0) {
   }
   RuneStr(Rune r, uint32_t o, uint32_t l)
     : rune(r), offset(o), len(l) {
   }
+  RuneStr(Rune r, uint32_t o, uint32_t l, uint32_t unicode_offset, uint32_t unicode_length)
+          : rune(r), offset(o), len(l), unicode_offset(unicode_offset), unicode_length(unicode_length) {
+  }
 }; // struct RuneStr
 
 inline std::ostream& operator << (std::ostream& os, const RuneStr& r) {
@@ -132,15 +142,16 @@ inline RuneStrLite DecodeRuneInString(const char* str, size_t len) {
 inline bool DecodeRunesInString(const char* s, size_t len, RuneStrArray& runes) {
   runes.clear();
   runes.reserve(len / 2);
-  for (size_t i = 0; i < len;) {
+  for (uint32_t i = 0, j = 0; i < len;) {
     RuneStrLite rp = DecodeRuneInString(s + i, len - i);
     if (rp.len == 0) {
       runes.clear();
       return false;
     }
-    RuneStr x(rp.rune, i, rp.len);
+    RuneStr x(rp.rune, i, rp.len, j, 1);
     runes.push_back(x);
     i += rp.len;
+    ++j;
   }
   return true;
 }
@@ -182,7 +193,8 @@ inline Unicode DecodeRunesInString(const string& s) {
 inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
   assert(right->offset >= left->offset);
   uint32_t len = right->offset - left->offset + right->len;
-  return Word(s.substr(left->offset, len), left->offset);
+  uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length;
+  return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length);
 }
 
 inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {

data/ext/cppjieba/test/demo.cpp

@@ -68,7 +68,7 @@ int main(int argc, char** argv) {
   s = "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。";
   jieba.Tag(s, tagres);
   cout << s << endl;
-  cout << tagres << endl;;
+  cout << tagres << endl;
 
   cout << "[demo] Keyword Extraction" << endl;
   const size_t topk = 5;

data/ext/cppjieba_rb/internal.cc

@@ -75,7 +75,8 @@ VALUE internal_initialize(VALUE self,
     while (getline(ifs, line)) {
         data->stop_words->insert(line);
     }
-    assert(data->stop_words->size());
+    assert(data->stop_words->size() != 0);
+    return self;
 }
 
 VALUE internal_extract_keyword(VALUE self, VALUE text_rbs, VALUE topN)
@@ -171,4 +172,4 @@ void Init_internal()
     rb_define_method(rb_cCppjiebaRb_Internal, "stop_word?", (ruby_method*) &internal_stop_word, 1);
 }
 
-}
+}

data/lib/cppjieba_rb/version.rb

@@ -1,3 +1,3 @@
 module CppjiebaRb
-  VERSION = '0.3.0'
+  VERSION = '0.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cppjieba_rb
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Erick Guan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-26 00:00:00.000000000 Z
+date: 2018-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -235,7 +235,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: cppjieba binding for ruby