cppjieba_rb 0.2.3 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9761a257960334fdb5e3cad3dd80be9f669b7f65
-  data.tar.gz: e62980a6590c4e2d141b5583af34737152dc4343
+SHA256:
+  metadata.gz: 74faf109bce12cd386acb5d2ad03ab4fde55a68cdb07a723a5af1f5b2528f3d5
+  data.tar.gz: c4a58470ef8e352cad688b9814080c37b332e46a93987e536387a94304ad1383
 SHA512:
-  metadata.gz: b631d51e9ad4f79ef34baed3c2149a872b0a5e4a512c4d2bd8269c1df6d86d1a870e9824483855c881a2844250096bd2017ec34068af91b8629acc4910aa347d
-  data.tar.gz: 29f1619beb0cca01ded498afd16d6fd081ca308d1d371b01998557fddf57ef870c31c5b51c0cda609f8e28a083a0ab945d08c98906153b11119f881946064acb
+  metadata.gz: 925ba793289f09a922451f6717b18a769080be4ab132a23edb236e4b91760251b67237dc2b6505c1aa39988d60294ff1ac07638c40a46916f6064be2cd71a425
+  data.tar.gz: e5f9dfcd2d341dd880bfc9aceee265abf6cf1e1d1a3428f807b1cd810691bab8e24946c974fef1260f89aebf03e82e85e518fd961905fd1150eac3ad6108c7ca

data/.travis.yml

@@ -1,7 +1,11 @@
 language: ruby
 rvm:
-  - 2.3.3
-  - 2.4.1
+  - 2.3
+  - 2.4
+  - 2.5
+  - 2.6
+  - 2.7
+  - 3.0
   - ruby-head
   - rbx-2
 

data/README.md

@@ -2,10 +2,14 @@
 
 [![Gem Version](https://badge.fury.io/rb/cppjieba_rb.svg)](http://badge.fury.io/rb/cppjieba_rb)
 
-[![Build Status](https://travis-ci.org/fantasticfears/cppjieba_rb.png?branch=master)](https://travis-ci.org/fantasticfears/cppjieba_rb)
+[![Build Status](https://travis-ci.com/erickguan/cppjieba_rb.svg?branch=master)](https://travis-ci.com/erickguan/cppjieba_rb)
+
+[![Patreon](https://img.shields.io/badge/back_on-patreon-red.svg)](https://www.patreon.com/fantasticfears)
 
 Ruby bindings for [Cppjieba](https://github.com/yanyiwu/cppjieba). C++11 required. (gcc 4.8+)
 
+The TRIE tree has high memory usage. For default dict, it uses ~120 MB memory.
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -26,7 +30,7 @@ Mix Segment mode (HMM with Max Prob, default):
 
 ```ruby
 require 'cppjieba_rb'
-seg = CppjiebaRb::Segment.new  # equivalent to "JiebaRb::Segment.new mode: :mix"
+seg = CppjiebaRb::Segment.new  # equivalent to "CppjiebaRb::Segment.new mode: :mix"
 words = seg.segment "令狐冲是云计算行业的专家"
 # 令狐冲 是 云 计算 行业 的 专家
 ```
@@ -70,7 +74,7 @@ CppjiebaRb.extract_keyword "我是拖拉机学院手扶拖拉机专业的。不
 
 ## Contributing
 
-1. Fork it ( http://github.com/<my-github-username>/cppjieba_rb/fork )
+1. Fork it ( http://github.com/fantasticfears/cppjieba_rb/fork )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
@@ -79,4 +83,4 @@ CppjiebaRb.extract_keyword "我是拖拉机学院手扶拖拉机专业的。不
 ## TODO
 
 - including 367w dict and provide the option for it.
-- cppjieba implements trie tree, it's memory consuming
+- cppjieba implements trie tree, it's memory consuming

data/Rakefile

@@ -3,7 +3,7 @@ require 'rake/testtask'
 require 'rake/extensiontask'
 
 gem = Gem::Specification.load(File.dirname(__FILE__) + '/cppjieba_rb.gemspec')
-Rake::ExtensionTask.new('cppjieba_rb', gem) do |ext|
+Rake::ExtensionTask.new("cppjieba_rb", gem) do |ext|
   ext.lib_dir = File.join('lib', 'cppjieba_rb')
 end
 

data/cppjieba_rb.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ['Erick Guan']
   spec.email         = ['fantasticfears@gmail.com']
   spec.summary       = 'cppjieba binding for ruby'
-  spec.description   = 'cppjieba binding for ruby'
+  spec.description   = 'cppjieba binding for ruby. Mainly used by Discourse.'
   spec.homepage = 'https://github.com/fantasticfears/cppjieba_rb'
   spec.required_ruby_version = '>=2.3.0'
   spec.license       = 'MIT'
@@ -43,8 +43,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.5'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rake-compiler'
-  spec.add_development_dependency 'minitest'
+  spec.add_development_dependency 'bundler', '~> 2.2', '>= 2.2.10'
+  spec.add_development_dependency 'rake', '~> 13'
+  spec.add_development_dependency 'rake-compiler', '~> 1.1'
+  spec.add_development_dependency 'minitest', '~> 5.14'
 end

data/ext/cppjieba/.travis.yml

@@ -1,7 +1,6 @@
 language: cpp
 before_install:
   - if [ $TRAVIS_OS_NAME == linux ]; then sudo apt-get install cmake; fi
-  - if [ $TRAVIS_OS_NAME == osx ]; then brew install cmake; fi
 script: 
   - mkdir build
   - cd build

data/ext/cppjieba/ChangeLog.md

@@ -1,5 +1,18 @@
 # CppJieba ChangeLog
 
+## v5.0.3
+
++ Upgrade [limonp](https://github.com/yanyiwu/limonp) -> v0.6.3
+
+## v5.0.2
+
++ Upgrade [limonp](https://github.com/yanyiwu/limonp) -> v0.6.1
+
+## v5.0.1
+
++ Make Compiler Happier.
++ Add PHP, DLang Links.
+
 ## v5.0.0
 
 + Notice(**api changed**) : Jieba class 3 arguments -> 5 arguments, and use KeywordExtractor in Jieba

data/ext/cppjieba/README.md

@@ -2,12 +2,15 @@
 
 [![Build Status](https://travis-ci.org/yanyiwu/cppjieba.png?branch=master)](https://travis-ci.org/yanyiwu/cppjieba) 
 [![Author](https://img.shields.io/badge/author-@yanyiwu-blue.svg?style=flat)](http://yanyiwu.com/) 
+[![Donate](https://img.shields.io/badge/donate-eos_gitdeveloper-orange.svg)](https://eosflare.io/account/gitdeveloper)
 [![Platform](https://img.shields.io/badge/platform-Linux,%20OS%20X,%20Windows-green.svg?style=flat)](https://github.com/yanyiwu/cppjieba)
 [![Performance](https://img.shields.io/badge/performance-excellent-brightgreen.svg?style=flat)](http://yanyiwu.com/work/2015/06/14/jieba-series-performance-test.html) 
+[![Tag](https://img.shields.io/github/v/tag/yanyiwu/cppjieba.svg)](https://github.com/yanyiwu/cppjieba/releases)
 [![License](https://img.shields.io/badge/license-MIT-yellow.svg?style=flat)](http://yanyiwu.mit-license.org)
 [![Build status](https://ci.appveyor.com/api/projects/status/wl30fjnm2rhft6ta/branch/master?svg=true)](https://ci.appveyor.com/project/yanyiwu/cppjieba/branch/master)
 
-[![logo](http://7viirv.com1.z0.glb.clouddn.com/CppJiebaLogo-v1.png)](https://github.com/yanyiwu/cppjieba)
+
+[![logo](http://images.yanyiwu.com/CppJiebaLogo-v1.png)](https://github.com/yanyiwu/cppjieba)
 
 ## 简介
 
@@ -223,11 +226,16 @@ Query方法先使用Mix方法切词，对于切出来的较长的词再使用Ful
 + [iosjieba] iOS 版本的结巴分词。
 + [SqlJieba] MySQL 全文索引的结巴中文分词插件。
 + [pg_jieba] PostgreSQL 数据库的分词插件。
++ [simple] SQLite3 FTS5 数据库的分词插件。
 + [gitbook-plugin-search-pro] 支持中文搜索的 gitbook 插件。
 + [ngx_http_cppjieba_module] Nginx 分词插件。
 + [cppjiebapy] 由 [jannson] 开发的供 python 模块调用的项目 [cppjiebapy], 相关讨论 [cppjiebapy_discussion] .
++ [cppjieba-py] 由 [bung87] 基于 pybind11 封装的 python 模块,使用体验上接近于原jieba。
 + [KeywordServer] 50行搭建一个中文关键词抽取服务。
 + [cppjieba-server] CppJieba HTTP 服务器。
++ [phpjieba] php版本的结巴分词扩展。
++ [perl5-jieba] Perl版本的结巴分词扩展。
++ [jieba-dlang] D 语言的结巴分词 Deimos Bindings。
 
 ## 线上演示
 
@@ -238,29 +246,23 @@ Query方法先使用Mix方法切词，对于切出来的较长的词再使用Ful
 
 [Jieba中文分词系列性能评测]
 
-## 客服
-
-+ Email: `i@yanyiwu.com`
-+ QQ: 64162451
-+ WeChat: ![image](http://7viirv.com1.z0.glb.clouddn.com/5a7d1b5c0d_yanyiwu_personal_qrcodes.jpg)
-
-## 鸣谢
+## Sponsorship
 
-"结巴"中文分词作者: [SunJunyi](https://github.com/fxsjy)
+[![sponsorship](http://images.gitads.io/cppjieba)](https://tracking.gitads.io/?campaign=gitads&repo=cppjieba&redirect=gitads.io)
 
-## 许可证
+## Contributors
 
-[MIT](http://yanyiwu.mit-license.org)
+### Code Contributors
 
-## 作者
-
-- [yanyiwu](yanyiwu.com)
-- [aholic](https://github.com/aholic)
+This project exists thanks to all the people who contribute.
+<a href="https://github.com/yanyiwu/cppjieba/graphs/contributors"><img src="https://opencollective.com/cppjieba/contributors.svg?width=890&button=false" /></a>
 
 [GoJieba]:https://github.com/yanyiwu/gojieba
 [CppJieba]:https://github.com/yanyiwu/cppjieba
 [jannson]:https://github.com/jannson
 [cppjiebapy]:https://github.com/jannson/cppjiebapy
+[bung87]:https://github.com/bung87
+[cppjieba-py]:https://github.com/bung87/cppjieba-py
 [cppjiebapy_discussion]:https://github.com/yanyiwu/cppjieba/issues/1
 [NodeJieba]:https://github.com/yanyiwu/nodejieba
 [jiebaR]:https://github.com/qinwf/jiebaR
@@ -279,7 +281,9 @@ Query方法先使用Mix方法切词，对于切出来的较长的词再使用Ful
 [pg_jieba]:https://github.com/jaiminpan/pg_jieba
 [gitbook-plugin-search-pro]:https://plugins.gitbook.com/plugin/search-pro
 [cppjieba-server]:https://github.com/yanyiwu/cppjieba-server
+[phpjieba]:https://github.com/jonnywang/phpjieba
+[perl5-jieba]:https://metacpan.org/pod/distribution/Lingua-ZH-Jieba/lib/Lingua/ZH/Jieba.pod
+[jieba-dlang]:https://github.com/shove70/jieba
+[simple]:https://github.com/wangfenjin/simple
 
 
-[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/yanyiwu/cppjieba/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
-

data/ext/cppjieba/README_EN.md

@@ -86,6 +86,8 @@ Please see details in `test/demo.cpp`.
 + [ngx_http_cppjieba_module]
 + [gitbook-plugin-search-pro]
 + [cppjieba-server]
++ [perl5-jieba]
++ [jieba-dlang]
 
 ## Contact
 
@@ -109,3 +111,5 @@ Please see details in `test/demo.cpp`.
 [pg_jieba]:https://github.com/jaiminpan/pg_jieba
 [gitbook-plugin-search-pro]:https://plugins.gitbook.com/plugin/search-pro
 [cppjieba-server]:https://github.com/yanyiwu/cppjieba-server
+[perl5-jieba]:https://metacpan.org/pod/distribution/Lingua-ZH-Jieba/lib/Lingua/ZH/Jieba.pod
+[jieba-dlang]:https://github.com/shove70/jieba

data/ext/cppjieba/deps/limonp/StdExtension.hpp

@@ -6,7 +6,7 @@
 #ifdef __APPLE__
 #include <unordered_map>
 #include <unordered_set>
-#elif(__cplusplus == 201103L)
+#elif(__cplusplus >= 201103L)
 #include <unordered_map>
 #include <unordered_set>
 #elif defined _MSC_VER
@@ -29,8 +29,6 @@ using std::tr1::unordered_set;
 #include <fstream>
 #include <sstream>
 
-#define print(x) std::cout << x << std::endl
-
 namespace std {
 
 template<typename T>

data/ext/cppjieba/deps/limonp/StringUtil.hpp

@@ -80,7 +80,7 @@ inline string& Lower(string& str) {
 
 inline bool IsSpace(unsigned c) {
   // when passing large int as the argument of isspace, it core dump, so here need a type cast.
-  return c > 0xff ? false : std::isspace(c & 0xff);
+  return c > 0xff ? false : std::isspace(c & 0xff) != 0;
 }
 
 inline std::string& LTrim(std::string &s) {

data/ext/cppjieba/include/cppjieba/DictTrie.hpp

@@ -50,6 +50,17 @@ class DictTrie {
     return true;
   }
 
+  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
+    DictUnit node_info;
+    double weight = freq ? log(1.0 * freq / freq_sum_) : user_word_default_weight_ ;
+    if (!MakeNodeInfo(node_info, word, weight , tag)) {
+      return false;
+    }
+    active_node_infos_.push_back(node_info);
+    trie_->InsertNode(node_info.word, &active_node_infos_.back());
+    return true;
+  }
+
   const DictUnit* Find(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
     return trie_->Find(begin, end);
   }
@@ -61,6 +72,25 @@ class DictTrie {
     trie_->Find(begin, end, res, max_word_len);
   }
 
+  bool Find(const string& word)
+  {
+    const DictUnit *tmp = NULL;
+    RuneStrArray runes;
+    if (!DecodeRunesInString(word, runes))
+    {
+      XLOG(ERROR) << "Decode failed.";
+    }
+    tmp = Find(runes.begin(), runes.end());
+    if (tmp == NULL)
+    {
+      return false;
+    }
+    else
+    {
+      return true;
+    }
+  }
+
   bool IsUserDictSingleChineseWord(const Rune& word) const {
     return IsIn(user_dict_single_chinese_word_, word);
   }
@@ -69,6 +99,63 @@ class DictTrie {
     return min_weight_;
   }
 
+  void InserUserDictNode(const string& line) {
+    vector<string> buf;
+    DictUnit node_info;
+    Split(line, buf, " ");
+    if(buf.size() == 1){
+          MakeNodeInfo(node_info, 
+                buf[0], 
+                user_word_default_weight_,
+                UNKNOWN_TAG);
+        } else if (buf.size() == 2) {
+          MakeNodeInfo(node_info, 
+                buf[0], 
+                user_word_default_weight_,
+                buf[1]);
+        } else if (buf.size() == 3) {
+          int freq = atoi(buf[1].c_str());
+          assert(freq_sum_ > 0.0);
+          double weight = log(1.0 * freq / freq_sum_);
+          MakeNodeInfo(node_info, buf[0], weight, buf[2]);
+        }
+        static_node_infos_.push_back(node_info);
+        if (node_info.word.size() == 1) {
+          user_dict_single_chinese_word_.insert(node_info.word[0]);
+        }
+  }
+  
+  void LoadUserDict(const vector<string>& buf) {
+    for (size_t i = 0; i < buf.size(); i++) {
+      InserUserDictNode(buf[i]);
+    }
+  }
+
+   void LoadUserDict(const set<string>& buf) {
+    std::set<string>::const_iterator iter;
+    for (iter = buf.begin(); iter != buf.end(); iter++){
+      InserUserDictNode(*iter);
+    }
+  }
+
+  void LoadUserDict(const string& filePaths) {
+    vector<string> files = limonp::Split(filePaths, "|;");
+    size_t lineno = 0;
+    for (size_t i = 0; i < files.size(); i++) {
+      ifstream ifs(files[i].c_str());
+      XCHECK(ifs.is_open()) << "open " << files[i] << " failed"; 
+      string line;
+      
+      for (; getline(ifs, line); lineno++) {
+        if (line.size() == 0) {
+          continue;
+        }
+        InserUserDictNode(line);
+      }
+    }
+  }
+
+
  private:
   void Init(const string& dict_path, const string& user_dict_paths, UserWordWeightOption user_word_weight_opt) {
     LoadDict(dict_path);
@@ -95,45 +182,8 @@ class DictTrie {
     trie_ = new Trie(words, valuePointers);
   }
 
-  void LoadUserDict(const string& filePaths) {
-    vector<string> files = limonp::Split(filePaths, "|;");
-    size_t lineno = 0;
-    for (size_t i = 0; i < files.size(); i++) {
-      ifstream ifs(files[i].c_str());
-      XCHECK(ifs.is_open()) << "open " << files[i] << " failed"; 
-      string line;
-      DictUnit node_info;
-      vector<string> buf;
-      for (; getline(ifs, line); lineno++) {
-        if (line.size() == 0) {
-          continue;
-        }
-        buf.clear();
-        Split(line, buf, " ");
-        DictUnit node_info;
-        if(buf.size() == 1){
-          MakeNodeInfo(node_info, 
-                buf[0], 
-                user_word_default_weight_,
-                UNKNOWN_TAG);
-        } else if (buf.size() == 2) {
-          MakeNodeInfo(node_info, 
-                buf[0], 
-                user_word_default_weight_,
-                buf[1]);
-        } else if (buf.size() == 3) {
-          int freq = atoi(buf[1].c_str());
-          assert(freq_sum_ > 0.0);
-          double weight = log(1.0 * freq / freq_sum_);
-          MakeNodeInfo(node_info, buf[0], weight, buf[2]);
-        }
-        static_node_infos_.push_back(node_info);
-        if (node_info.word.size() == 1) {
-          user_dict_single_chinese_word_.insert(node_info.word[0]);
-        }
-      }
-    }
-  }
+  
+
 
   bool MakeNodeInfo(DictUnit& node_info,
         const string& word, 

data/ext/cppjieba/include/cppjieba/FullSegment.hpp

@@ -48,17 +48,17 @@ class FullSegment: public SegmentBase {
   void Cut(RuneStrArray::const_iterator begin, 
         RuneStrArray::const_iterator end, 
         vector<WordRange>& res) const {
-    //resut of searching in trie tree
+    // result of searching in trie tree
     LocalVector<pair<size_t, const DictUnit*> > tRes;
 
-    //max index of res's words
-    int maxIdx = 0;
+    // max index of res's words
+    size_t maxIdx = 0;
 
     // always equals to (uItr - begin)
-    int uIdx = 0;
+    size_t uIdx = 0;
 
-    //tmp variables
-    int wordLen = 0;
+    // tmp variables
+    size_t wordLen = 0;
     assert(dictTrie_);
     vector<struct Dag> dags;
     dictTrie_->Find(begin, end, dags);

data/ext/cppjieba/include/cppjieba/Jieba.hpp

@@ -72,6 +72,15 @@ class Jieba {
     return dict_trie_.InsertUserWord(word, tag);
   }
 
+  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
+    return dict_trie_.InsertUserWord(word,freq, tag);
+  }
+
+  bool Find(const string& word)
+  {
+    return dict_trie_.Find(word);
+  }
+
   void ResetSeparators(const string& s) {
     //TODO
     mp_seg_.ResetSeparators(s);
@@ -84,10 +93,23 @@ class Jieba {
   const DictTrie* GetDictTrie() const {
     return &dict_trie_;
   } 
+  
   const HMMModel* GetHMMModel() const {
     return &model_;
   }
 
+  void LoadUserDict(const vector<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const set<string>& buf)  {
+    dict_trie_.LoadUserDict(buf);
+  }
+
+  void LoadUserDict(const string& path)  {
+    dict_trie_.LoadUserDict(path);
+  }
+
  private:
   DictTrie dict_trie_;
   HMMModel model_;

data/ext/cppjieba/include/cppjieba/QuerySegment.hpp

@@ -10,7 +10,6 @@
 #include "FullSegment.hpp"
 #include "MixSegment.hpp"
 #include "Unicode.hpp"
-#include "DictTrie.hpp"
 
 namespace cppjieba {
 class QuerySegment: public SegmentBase {

data/ext/cppjieba/include/cppjieba/Unicode.hpp

@@ -18,9 +18,14 @@ typedef uint32_t Rune;
 struct Word {
   string word;
   uint32_t offset;
+  uint32_t unicode_offset;
+  uint32_t unicode_length;
   Word(const string& w, uint32_t o)
    : word(w), offset(o) {
   }
+  Word(const string& w, uint32_t o, uint32_t unicode_offset, uint32_t unicode_length)
+          : word(w), offset(o), unicode_offset(unicode_offset), unicode_length(unicode_length) {
+  }
 }; // struct Word
 
 inline std::ostream& operator << (std::ostream& os, const Word& w) {
@@ -31,10 +36,15 @@ struct RuneStr {
   Rune rune;
   uint32_t offset;
   uint32_t len;
-  RuneStr(): rune(0), offset(0), len(0) {
+  uint32_t unicode_offset;
+  uint32_t unicode_length;
+  RuneStr(): rune(0), offset(0), len(0), unicode_offset(0), unicode_length(0) {
   }
   RuneStr(Rune r, uint32_t o, uint32_t l)
-    : rune(r), offset(o), len(l) {
+    : rune(r), offset(o), len(l), unicode_offset(0), unicode_length(0) {
+  }
+  RuneStr(Rune r, uint32_t o, uint32_t l, uint32_t unicode_offset, uint32_t unicode_length)
+          : rune(r), offset(o), len(l), unicode_offset(unicode_offset), unicode_length(unicode_length) {
   }
 }; // struct RuneStr
 
@@ -132,15 +142,16 @@ inline RuneStrLite DecodeRuneInString(const char* str, size_t len) {
 inline bool DecodeRunesInString(const char* s, size_t len, RuneStrArray& runes) {
   runes.clear();
   runes.reserve(len / 2);
-  for (size_t i = 0; i < len;) {
+  for (uint32_t i = 0, j = 0; i < len;) {
     RuneStrLite rp = DecodeRuneInString(s + i, len - i);
     if (rp.len == 0) {
       runes.clear();
       return false;
     }
-    RuneStr x(rp.rune, i, rp.len);
+    RuneStr x(rp.rune, i, rp.len, j, 1);
     runes.push_back(x);
     i += rp.len;
+    ++j;
   }
   return true;
 }
@@ -182,7 +193,8 @@ inline Unicode DecodeRunesInString(const string& s) {
 inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
   assert(right->offset >= left->offset);
   uint32_t len = right->offset - left->offset + right->len;
-  return Word(s.substr(left->offset, len), left->offset);
+  uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length;
+  return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length);
 }
 
 inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {

data/ext/cppjieba/test/demo.cpp

@@ -68,7 +68,7 @@ int main(int argc, char** argv) {
   s = "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。";
   jieba.Tag(s, tagres);
   cout << s << endl;
-  cout << tagres << endl;;
+  cout << tagres << endl;
 
   cout << "[demo] Keyword Extraction" << endl;
   const size_t topk = 5;

data/ext/cppjieba_rb/cppjieba_rb.c

@@ -1,5 +1,6 @@
 #include <ruby.h>
 
+void Init_internal();
 VALUE rb_mCppjiebaRb;
 
 void Init_cppjieba_rb()

data/ext/cppjieba_rb/internal.cc

@@ -1,5 +1,10 @@
 #include <ruby.h>
 #include <ruby/encoding.h>
+
+#include <string>
+#include <iostream>
+#include <unordered_set>
+
 #include "cppjieba/Jieba.hpp"
 
 #define GET_CPPJIEBA(_data) jieba_cpp_data* _data; \
@@ -7,6 +12,7 @@
 
 typedef struct {
     cppjieba::Jieba* jieba;
+    std::unordered_set<std::string>* stop_words;
 } jieba_cpp_data;
 
 // make compiler happy
@@ -26,6 +32,8 @@ static void jieba_cpp_free(void* _this)
     jieba_cpp_data* data = static_cast<jieba_cpp_data*>(_this);
     delete data->jieba;
     data->jieba = nullptr;
+    delete data->stop_words;
+    data->stop_words = nullptr;
 }
 
 static size_t jieba_cpp_memsize(const void* _)
@@ -61,6 +69,14 @@ VALUE internal_initialize(VALUE self,
                                       StringValueCStr(user_dict_path),
                                       StringValueCStr(idf_path),
                                       StringValueCStr(stop_word_path));
+    data->stop_words = new std::unordered_set<std::string>();
+    std::ifstream ifs(StringValueCStr(stop_word_path));
+    std::string line;
+    while (getline(ifs, line)) {
+        data->stop_words->insert(line);
+    }
+    assert(data->stop_words->size() != 0);
+    return self;
 }
 
 VALUE internal_extract_keyword(VALUE self, VALUE text_rbs, VALUE topN)
@@ -127,6 +143,16 @@ static VALUE internal_segment_tag(VALUE self, VALUE text_rbs)
     return result;
 }
 
+static VALUE internal_stop_word(VALUE self, VALUE word)
+{
+    std::string test(StringValueCStr(word));
+    GET_CPPJIEBA(data);
+    if (data->stop_words->find(test) != data->stop_words->end()) {
+        return Qtrue;
+    } else {
+        return Qfalse;
+    }
+}
 
 void Init_internal()
 {
@@ -143,6 +169,7 @@ void Init_internal()
     rb_define_method(rb_cCppjiebaRb_Internal, "extract_keyword", (ruby_method*) &internal_extract_keyword, 2);
     rb_define_method(rb_cCppjiebaRb_Internal, "segment", (ruby_method*) &internal_segment, 4);
     rb_define_method(rb_cCppjiebaRb_Internal, "segment_tag", (ruby_method*) &internal_segment_tag, 1);
+    rb_define_method(rb_cCppjiebaRb_Internal, "stop_word?", (ruby_method*) &internal_stop_word, 1);
 }
 
-}
+}

data/lib/cppjieba_rb.rb

@@ -22,6 +22,10 @@ module CppjiebaRb
     internal.segment_tag(str)
   end
 
+  def self.filter_stop_word(arr)
+    arr.reject { |w| internal.stop_word?(w) }
+  end
+
   class << self
     def internal
       @backend ||= CppjiebaRb::Internal.new(DICT_PATH,

data/lib/cppjieba_rb/version.rb

@@ -1,3 +1,3 @@
 module CppjiebaRb
-  VERSION = '0.2.3'
+  VERSION = '0.4.1'
 end

data/test/test_stop_word_filter.rb

@@ -0,0 +1,10 @@
+# coding: utf-8
+require 'minitest/autorun'
+require 'cppjieba_rb'
+
+class JiebaTest < Minitest::Test
+  def test_filter
+    words = CppjiebaRb.filter_stop_word %w(令狐冲 是 云计算 行业 的 专家)
+    assert_equal %w(令狐冲 云计算 行业 专家), words
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cppjieba_rb
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.4.1
 platform: ruby
 authors:
 - Erick Guan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-26 00:00:00.000000000 Z
+date: 2021-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,57 +16,63 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '2.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.10
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '2.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.2.10
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-description: cppjieba binding for ruby
+        version: '5.14'
+description: cppjieba binding for ruby. Mainly used by Discourse.
 email:
 - fantasticfears@gmail.com
 executables: []
@@ -213,6 +219,7 @@ files:
 - lib/cppjieba_rb/version.rb
 - test/test_keyword.rb
 - test/test_segment.rb
+- test/test_stop_word_filter.rb
 - test/test_tagging.rb
 homepage: https://github.com/fantasticfears/cppjieba_rb
 licenses:
@@ -233,12 +240,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.12
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: cppjieba binding for ruby
 test_files:
 - test/test_keyword.rb
 - test/test_segment.rb
+- test/test_stop_word_filter.rb
 - test/test_tagging.rb