cppjieba_rb 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9761a257960334fdb5e3cad3dd80be9f669b7f65
-  data.tar.gz: e62980a6590c4e2d141b5583af34737152dc4343
+  metadata.gz: 4ab9b4adf8e5ddccbe36570f3492b4ae438c7b6a
+  data.tar.gz: 72987214a678eaf3505f804bd7c9db86f94c6178
 SHA512:
-  metadata.gz: b631d51e9ad4f79ef34baed3c2149a872b0a5e4a512c4d2bd8269c1df6d86d1a870e9824483855c881a2844250096bd2017ec34068af91b8629acc4910aa347d
-  data.tar.gz: 29f1619beb0cca01ded498afd16d6fd081ca308d1d371b01998557fddf57ef870c31c5b51c0cda609f8e28a083a0ab945d08c98906153b11119f881946064acb
+  metadata.gz: 1cd2f4cdd703464e0672ea034ae5bc875d4754ce3e45d859b66d2cf65759e32bbca499221ef4ca70239bcf3606eafeee13b9c6723d5a626a8aa2bf06fdfc970e
+  data.tar.gz: 796eacaca074e602a080a94a2532e4f051e2b9e874ee6412cf45b683522afc36f2ba219be6a375c728ce6f3eb9b225c7fff783643cac3732dd4992bf263a1e4a

data/README.md

@@ -4,6 +4,8 @@
 
 [![Build Status](https://travis-ci.org/fantasticfears/cppjieba_rb.png?branch=master)](https://travis-ci.org/fantasticfears/cppjieba_rb)
 
++[![Patreon](https://img.shields.io/badge/back_on-patreon-red.svg)](https://www.patreon.com/fantasticfears)
+
 Ruby bindings for [Cppjieba](https://github.com/yanyiwu/cppjieba). C++11 required. (gcc 4.8+)
 
 ## Installation

data/ext/cppjieba_rb/internal.cc

@@ -1,5 +1,10 @@
 #include <ruby.h>
 #include <ruby/encoding.h>
+
+#include <string>
+#include <iostream>
+#include <unordered_set>
+
 #include "cppjieba/Jieba.hpp"
 
 #define GET_CPPJIEBA(_data) jieba_cpp_data* _data; \
@@ -7,6 +12,7 @@
 
 typedef struct {
     cppjieba::Jieba* jieba;
+    std::unordered_set<std::string>* stop_words;
 } jieba_cpp_data;
 
 // make compiler happy
@@ -26,6 +32,8 @@ static void jieba_cpp_free(void* _this)
     jieba_cpp_data* data = static_cast<jieba_cpp_data*>(_this);
     delete data->jieba;
     data->jieba = nullptr;
+    delete data->stop_words;
+    data->stop_words = nullptr;
 }
 
 static size_t jieba_cpp_memsize(const void* _)
@@ -61,6 +69,13 @@ VALUE internal_initialize(VALUE self,
                                       StringValueCStr(user_dict_path),
                                       StringValueCStr(idf_path),
                                       StringValueCStr(stop_word_path));
+    data->stop_words = new std::unordered_set<std::string>();
+    std::ifstream ifs(StringValueCStr(stop_word_path));
+    std::string line;
+    while (getline(ifs, line)) {
+        data->stop_words->insert(line);
+    }
+    assert(data->stop_words->size());
 }
 
 VALUE internal_extract_keyword(VALUE self, VALUE text_rbs, VALUE topN)
@@ -127,6 +142,16 @@ static VALUE internal_segment_tag(VALUE self, VALUE text_rbs)
     return result;
 }
 
+static VALUE internal_stop_word(VALUE self, VALUE word)
+{
+    std::string test(StringValueCStr(word));
+    GET_CPPJIEBA(data);
+    if (data->stop_words->find(test) != data->stop_words->end()) {
+        return Qtrue;
+    } else {
+        return Qfalse;
+    }
+}
 
 void Init_internal()
 {
@@ -143,6 +168,7 @@ void Init_internal()
     rb_define_method(rb_cCppjiebaRb_Internal, "extract_keyword", (ruby_method*) &internal_extract_keyword, 2);
     rb_define_method(rb_cCppjiebaRb_Internal, "segment", (ruby_method*) &internal_segment, 4);
     rb_define_method(rb_cCppjiebaRb_Internal, "segment_tag", (ruby_method*) &internal_segment_tag, 1);
+    rb_define_method(rb_cCppjiebaRb_Internal, "stop_word?", (ruby_method*) &internal_stop_word, 1);
 }
 
 }

data/lib/cppjieba_rb.rb

@@ -22,6 +22,10 @@ module CppjiebaRb
     internal.segment_tag(str)
   end
 
+  def self.filter_stop_word(arr)
+    arr.reject { |w| internal.stop_word?(w) }
+  end
+
   class << self
     def internal
       @backend ||= CppjiebaRb::Internal.new(DICT_PATH,

data/lib/cppjieba_rb/version.rb

@@ -1,3 +1,3 @@
 module CppjiebaRb
-  VERSION = '0.2.3'
+  VERSION = '0.3.0'
 end

data/test/test_stop_word_filter.rb

@@ -0,0 +1,10 @@
+# coding: utf-8
+require 'minitest/autorun'
+require 'cppjieba_rb'
+
+class JiebaTest < Minitest::Test
+  def test_filter
+    words = CppjiebaRb.filter_stop_word %w(令狐冲 是 云计算 行业 的 专家)
+    assert_equal %w(令狐冲 云计算 行业 专家), words
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cppjieba_rb
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Erick Guan
@@ -213,6 +213,7 @@ files:
 - lib/cppjieba_rb/version.rb
 - test/test_keyword.rb
 - test/test_segment.rb
+- test/test_stop_word_filter.rb
 - test/test_tagging.rb
 homepage: https://github.com/fantasticfears/cppjieba_rb
 licenses:
@@ -241,4 +242,5 @@ summary: cppjieba binding for ruby
 test_files:
 - test/test_keyword.rb
 - test/test_segment.rb
+- test/test_stop_word_filter.rb
 - test/test_tagging.rb