jieba_rb 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 122f7b0e8353ea96f9eecc620c894d8dc27b3402
-  data.tar.gz: 8ceb3c44c9957f8a080653f091580d6ea24a7e48
+  metadata.gz: cf9ad1c59145d326f4edd1c639066e107b52c13a
+  data.tar.gz: 4732514648268711284a8198a4a324889636112d
 SHA512:
-  metadata.gz: 39f49d5d34221863aab6864465a8671a662e4d94fdb93ffe0439fd2f89d7b63d983002c031abb54e651a5a2f99d220f4ec27b24778774f93497dd9ecf9e788df
-  data.tar.gz: 9232b90f160726d296d64c95bbb39545ebb1ed362fef0463fdd30a9d7777dedfe37cbca162cdf31618cf3cc1502c7db5de815866fba80956dc97325bbe881372
+  metadata.gz: 8fdc938f12a9506eb46baef952fd6bd5a261803934673273ba33f8e57cccd614ea31725dc58262b1e6c24828db9f8787027f8e95f4e614504eb943cf75dadf04
+  data.tar.gz: cd5d7df0f5f46cf4a7f8553d9f09cf05d0aeceb2f7b211a9238b9596aa43c3bd395425d2d278fc971bc3f224ac764b55cdfe88506c1b77a0ab8bcb927b206b48

data/.travis.yml

@@ -2,5 +2,13 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.2
+  - 2.1.5
+  - ruby-head
+  - rbx-2
+
 before_script: rake compile
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+    - rvm: rbx-2

data/README.md

@@ -23,24 +23,42 @@ Or install it yourself as:
 
 ## Word segment Usage
 
-Mix Segment (HMM with Max Prob, default):
+Mix Segment mode (HMM with Max Prob, default):
 
 	require 'jieba_rb'
-    seg = JiebaRb::Segment.new type: :mix
+    seg = JiebaRb::Segment.new  # equivalent to "JiebaRb::Segment.new mode: :mix"
     words = seg.cut "令狐冲是云计算行业的专家"
     # 令狐冲 是 云 计算 行业 的 专家
 
-Mix Segment with user-defined dictionary:
+Mix Segment mode with user-defined dictionary:
 
-    seg = JiebaRb::Segment.new type: :mix, user_dict: "ext/cppjieba/dict/user.dict.utf8"
+    seg = JiebaRb::Segment.new mode: :mix, user_dict: "ext/cppjieba/dict/user.dict.utf8"
     words = seg.cut "令狐冲是云计算行业的专家"
     # 令狐冲 是 云计算 行业 的 专家
 
-HMM or Max probability (mp) Segment: 
+HMM or Max probability (mp) Segment mode:
 
-    seg = JiebaRb::Segment.new type: :hmm # or  type: :mp 
+    seg = JiebaRb::Segment.new mode: :hmm # or  mode: :mp
     words = seg.cut "令狐冲是云计算行业的专家"
 
+## Keyword Extractor Usage
+
+* only support TF-IDF currently
+
+```
+    keyword = JiebaRb::Keyword.new
+    keywords_weights = keyword.extract "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。", 5
+
+                  [
+                  ["CEO", 11.739204307083542],
+                  ["升职", 10.8561552143],
+                  ["加薪", 10.642581114],
+                  ["手扶拖拉机", 10.0088573539],
+                  ["巅峰", 9.49395840471]
+                  ]
+```
+
+
 
 ## Contributing
 

data/Rakefile

@@ -7,5 +7,9 @@ Rake::ExtensionTask.new "jieba"
 Rake::TestTask.new do |t|
   t.libs << 'test'
 end
-
+desc "clean compile files"
+task :clean_compile do
+  system "rm -r tmp"
+  system "rm lib/*.bundle"
+end
 task :default => :test

data/ext/jieba/jieba.c

@@ -6,4 +6,5 @@ void Init_jieba()
     mJieba = rb_define_module("JiebaRb");
 
     Init_segment();
+    Init_keyword();
 }

data/ext/jieba/jieba.h

@@ -3,6 +3,7 @@
 
 #include <ruby.h>
 #include <segment.h>
+#include <keyword.h>
 
 extern VALUE mJieba;
 

data/ext/jieba/keyword.cc

@@ -0,0 +1,98 @@
+#include "segment.h"
+#include <ruby/encoding.h>
+#include <KeywordExtractor.hpp>
+
+static rb_encoding* u8_enc;
+
+struct Keyword{
+    CppJieba::KeywordExtractor * p;
+};
+
+static void keyword_free(void *p){
+    delete ((Keyword*) p) -> p;
+    delete (Keyword*)p;
+}
+
+static VALUE allocate(VALUE klass)
+{
+    Keyword * keyword = new Keyword();
+    return Data_Wrap_Struct(klass, NULL, keyword_free, keyword);
+}
+
+static void init(VALUE self,
+                 VALUE mode_rb_sym,
+                 VALUE jieba_dict_rbs,
+                 VALUE hmm_dict_rbs,
+                 VALUE idf_rbs,
+                 VALUE stop_words_rbs,
+                 VALUE user_dict_rbs)
+{
+    Keyword * keyword;
+    Data_Get_Struct(self, Keyword, keyword);
+
+    Check_Type(jieba_dict_rbs, T_STRING);
+    Check_Type(hmm_dict_rbs, T_STRING);
+    Check_Type(user_dict_rbs, T_STRING);
+    Check_Type(idf_rbs, T_STRING);
+    Check_Type(stop_words_rbs, T_STRING);
+
+    std::string jieba_dict = StringValueCStr(jieba_dict_rbs);
+    std::string hmm_dict = StringValueCStr(hmm_dict_rbs);
+    std::string idf = StringValueCStr(idf_rbs);
+    std::string stop_words = StringValueCStr(stop_words_rbs);
+    std::string user_dict = StringValueCStr(user_dict_rbs);
+
+    ID mode = SYM2ID(mode_rb_sym);
+    if ( mode == rb_intern("tf_idf") )
+    {
+        keyword->p = new CppJieba::KeywordExtractor(jieba_dict, hmm_dict, idf, stop_words);
+    }
+}
+
+static VALUE extract(VALUE self, VALUE text_rbs, VALUE topN)
+{
+    Check_Type(text_rbs, T_STRING);
+    std::string text = StringValueCStr(text_rbs);
+
+    Check_Type(topN, T_FIXNUM);
+    int top_n = NUM2INT(topN);
+
+    Keyword * keyword;
+    Data_Get_Struct(self, Keyword, keyword);
+
+    std::vector<std::pair<std::string, double> > top_words;
+
+    if (keyword->p->extract(text, top_words, top_n))
+    {
+        volatile VALUE arr = rb_ary_new();
+        for(size_t i = 0; i < top_words.size(); i++)
+        {
+            volatile VALUE inner_arr = rb_ary_new();
+            std::string & word = top_words[i].first;
+            rb_ary_push(inner_arr, rb_enc_str_new(word.c_str(), word.length(), u8_enc));
+            rb_ary_push(inner_arr, rb_float_new(top_words[i].second));
+
+            rb_ary_push(arr, inner_arr);
+
+        }
+        return arr;
+    }
+    else
+    {
+        return Qfalse;
+    }
+}
+
+#define DEF(k,n,f,c) rb_define_method(k,n,RUBY_METHOD_FUNC(f),c)
+
+extern "C" {
+    void Init_keyword()
+    {
+        VALUE cKeyword = rb_define_class_under(mJieba, "Keyword", rb_cObject);
+        u8_enc = rb_utf8_encoding();
+        rb_define_alloc_func(cKeyword, allocate);
+        DEF(cKeyword, "_init", init, 6);
+        DEF(cKeyword, "extract",extract,2);
+    }
+
+}

data/ext/jieba/keyword.h

@@ -0,0 +1,17 @@
+#ifndef RUBY_JIEBA_KEYWORD
+#define RUBY_JIEBA_KEYWORD
+
+#include <jieba.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    void Init_keyword();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

data/lib/jieba_rb/version.rb

@@ -1,3 +1,3 @@
 module JiebaRb
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/jieba_rb.rb

@@ -1,28 +1,55 @@
 require "jieba_rb/version"
 require "jieba"
 module JiebaRb
+  abs = File.expand_path File.dirname(__FILE__)
+  EXT_BASE = "#{abs}/../ext/cppjieba/"
+  DEFAULT_JIEBA_DICT = EXT_BASE + "dict/jieba.dict.utf8";
+  DEFAULT_HMM_DICT = EXT_BASE + "dict/hmm_model.utf8";
+  DEFAULT_USER_DICT = EXT_BASE + "dict/user.dict.utf8";
+
   class Segment
-    abs = File.expand_path File.dirname(__FILE__)
-    EXT_BASE = "#{abs}/../ext/cppjieba/"
-    JIEBA_DICT_FILE = EXT_BASE + "dict/jieba.dict.utf8";
-    HMM_DICT_FILE = EXT_BASE + "dict/hmm_model.utf8";
-    USER_DICT_FILE = EXT_BASE + "dict/user.dict.utf8";
+    private :_init
+    def initialize opts = {}
+      valid_modes = [:mix, :hmm, :mp]
+      if mode = opts[:mode]
+        raise "Mode must be one of :mix :hmm :mp" unless valid_modes.include? mode
+      else
+        mode = :mix #default
+      end
+
+      jieba_dict = opts[:jieba_dict] || DEFAULT_JIEBA_DICT
+      hmm_dict = opts[:hmm_dict] || DEFAULT_HMM_DICT
+      user_dict = opts[:user_dict] || ""
+      user_dict = DEFAULT_USER_DICT if user_dict == :default
+
+      _init mode, jieba_dict, hmm_dict, user_dict
+    end
+  end
+
+  class Keyword
+    DEFAULT_IDF = EXT_BASE + "dict/idf.utf8"
+    DEFAULT_STOP_WORDS = EXT_BASE + "dict/stop_words.utf8"
 
     private :_init
+
     def initialize opts = {}
-      valid_seg_types = [:mix, :hmm, :mp]
-      if type = opts[:type]
-        raise "Type must be one of :mix :hmm :mp" unless valid_seg_types.include? type
+      valid_modes = [:tf_idf]
+      if mode = opts[:mode]
+        raise "Mode must be one of :tf_idf" unless valid_modes.include? mode
       else
-        type = :mix #default
+        mode = :tf_idf #default
       end
 
-      jieba_dict = opts[:jieba_dict] || JIEBA_DICT_FILE
-      hmm_dict = opts[:hmm_dict] || HMM_DICT_FILE
+      jieba_dict = opts[:jieba_dict] || DEFAULT_JIEBA_DICT
+      hmm_dict = opts[:hmm_dict] || DEFAULT_HMM_DICT
+      idf_path = opts[:idf] || DEFAULT_IDF
+      stop_words_path = opts[:stop_words] || DEFAULT_STOP_WORDS
+
       user_dict = opts[:user_dict] || ""
       user_dict = USER_DICT_FILE if user_dict == :default
 
-      _init type, jieba_dict, hmm_dict, user_dict
+      _init mode, jieba_dict, hmm_dict, idf_path, stop_words_path, user_dict
     end
   end
+
 end

data/test/test_keyword.rb

@@ -0,0 +1,17 @@
+# coding: utf-8
+require 'minitest/autorun'
+require 'jieba_rb'
+class JiebaTest < Minitest::Test
+  def test_keywords
+    keyword = JiebaRb::Keyword.new
+    keywords_weights = keyword.extract "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。", 5
+
+    assert_equal [["CEO",
+                   11.739204307083542],
+                  ["升职", 10.8561552143],
+                  ["加薪", 10.642581114],
+                  ["手扶拖拉机", 10.0088573539],
+                  ["巅峰", 9.49395840471]], keywords_weights
+
+  end
+end

data/test/test_segment.rb

@@ -18,13 +18,13 @@ class JiebaTest < Minitest::Test
   end
 
   def test_hmm_segment
-    seg = JiebaRb::Segment.new type: :hmm
+    seg = JiebaRb::Segment.new mode: :hmm
     words = seg.cut "令狐冲是云计算行业的专家"
     assert_equal %w(令狐冲 是 云计算 行业 的 专家), words
   end
 
   def test_max_prob_segment
-    seg = JiebaRb::Segment.new type: :mp
+    seg = JiebaRb::Segment.new mode: :mp
     words = seg.cut "令狐冲是云计算行业的专家"
     assert_equal %w(令狐冲 是 云 计算 行业 的 专家), words
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: jieba_rb
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Chris Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-21 00:00:00.000000000 Z
+date: 2014-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,15 +81,6 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- ext/jieba/extconf.rb
-- ext/jieba/jieba.c
-- ext/jieba/jieba.h
-- ext/jieba/segment.cc
-- ext/jieba/segment.h
-- jieba_rb.gemspec
-- lib/jieba_rb.rb
-- lib/jieba_rb/version.rb
-- test/test_segment.rb
 - ext/cppjieba/.gitignore
 - ext/cppjieba/.travis.yml
 - ext/cppjieba/CMakeLists.txt
@@ -217,6 +208,18 @@ files:
 - ext/cppjieba/test/unittest/gtest-1.6.0/src/gtest.cc
 - ext/cppjieba/test/unittest/gtest-1.6.0/src/gtest_main.cc
 - ext/cppjieba/test/unittest/gtest_main.cpp
+- ext/jieba/extconf.rb
+- ext/jieba/jieba.c
+- ext/jieba/jieba.h
+- ext/jieba/keyword.cc
+- ext/jieba/keyword.h
+- ext/jieba/segment.cc
+- ext/jieba/segment.h
+- jieba_rb.gemspec
+- lib/jieba_rb.rb
+- lib/jieba_rb/version.rb
+- test/test_keyword.rb
+- test/test_segment.rb
 homepage: https://github.com/altkatz/jieba_rb
 licenses:
 - MIT
@@ -237,10 +240,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.11
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: cppjieba binding for ruby
 test_files:
+- test/test_keyword.rb
 - test/test_segment.rb
 has_rdoc: