jieba-rb 5.0.0

data/ext/jieba/extconf.rb

@@ -0,0 +1,28 @@
+require "mkmf"
+abs = File.expand_path File.dirname(__FILE__)
+
+LIBDIR      = RbConfig::CONFIG['libdir']
+INCLUDEDIR  = RbConfig::CONFIG['includedir']
+
+HEADER_DIRS = [
+  INCLUDEDIR,
+  "#{abs}/../cppjieba/src",
+  "#{abs}/../cppjieba/include/cppjieba",
+  "#{abs}/../cppjieba/deps"
+]
+
+LIB_DIRS = [
+
+  LIBDIR
+]
+
+dir_config('cppjieba_src', HEADER_DIRS, LIB_DIRS)
+
+CONFIG["CXXFLAGS"] += " -std=c++0x -O3"
+$CXXFLAGS = "#{$CXXFLAGS} -std=c++0x -O3"
+create_makefile 'jieba'
+# respect header changes
+headers = Dir.glob('*.{hpp,h}').join ' '
+File.open 'Makefile', 'a' do |f|
+  f.puts "\n$(OBJS): #{headers}"
+end

data/ext/jieba/jieba.c

@@ -0,0 +1,11 @@
+#include <jieba.h>
+VALUE mJieba;
+
+void Init_jieba()
+{
+    mJieba = rb_define_module("JiebaRb");
+
+    Init_segment();
+    Init_keyword();
+    Init_tagging();
+}

data/ext/jieba/jieba.h

@@ -0,0 +1,11 @@
+#ifndef RUBY_JIEBA
+#define RUBY_JIEBA
+
+#include <ruby.h>
+#include <segment.h>
+#include <keyword.h>
+#include <tagging.h>
+
+extern VALUE mJieba;
+
+#endif

data/ext/jieba/keyword.cc

@@ -0,0 +1,92 @@
+#include "segment.h"
+#include <ruby/encoding.h>
+#include <KeywordExtractor.hpp>
+
+static rb_encoding* u8_enc;
+
+struct Keyword{
+    cppjieba::KeywordExtractor * p;
+};
+
+static void keyword_free(void *p){
+    delete ((Keyword*) p) -> p;
+    delete (Keyword*)p;
+}
+
+static VALUE allocate(VALUE klass)
+{
+    Keyword * keyword = new Keyword();
+    return Data_Wrap_Struct(klass, NULL, keyword_free, keyword);
+}
+
+static void init(VALUE self,
+                 VALUE mode_rb_sym,
+                 VALUE jieba_dict_rbs,
+                 VALUE hmm_dict_rbs,
+                 VALUE idf_rbs,
+                 VALUE stop_words_rbs,
+                 VALUE user_dict_rbs)
+{
+    Keyword * keyword;
+    Data_Get_Struct(self, Keyword, keyword);
+
+    Check_Type(jieba_dict_rbs, T_STRING);
+    Check_Type(hmm_dict_rbs, T_STRING);
+    Check_Type(user_dict_rbs, T_STRING);
+    Check_Type(idf_rbs, T_STRING);
+    Check_Type(stop_words_rbs, T_STRING);
+
+    std::string jieba_dict = StringValueCStr(jieba_dict_rbs);
+    std::string hmm_dict = StringValueCStr(hmm_dict_rbs);
+    std::string idf = StringValueCStr(idf_rbs);
+    std::string stop_words = StringValueCStr(stop_words_rbs);
+    std::string user_dict = StringValueCStr(user_dict_rbs);
+
+    ID mode = SYM2ID(mode_rb_sym);
+    if ( mode == rb_intern("tf_idf") )
+    {
+        keyword->p = new cppjieba::KeywordExtractor(jieba_dict, hmm_dict, idf, stop_words, user_dict);
+    }
+}
+
+static VALUE extract(VALUE self, VALUE text_rbs, VALUE topN)
+{
+    Check_Type(text_rbs, T_STRING);
+    std::string text = StringValueCStr(text_rbs);
+
+    Check_Type(topN, T_FIXNUM);
+    int top_n = NUM2INT(topN);
+
+    Keyword * keyword;
+    Data_Get_Struct(self, Keyword, keyword);
+
+    std::vector<std::pair<std::string, double> > top_words;
+
+    keyword->p->Extract(text, top_words, top_n);
+    volatile VALUE arr = rb_ary_new();
+    for(size_t i = 0; i < top_words.size(); i++)
+    {
+        volatile VALUE inner_arr = rb_ary_new();
+        std::string & word = top_words[i].first;
+        rb_ary_push(inner_arr, rb_enc_str_new(word.c_str(), word.length(), u8_enc));
+        rb_ary_push(inner_arr, rb_float_new(top_words[i].second));
+
+        rb_ary_push(arr, inner_arr);
+
+    }
+    return arr;
+}
+
+#define DEF(k,n,f,c) rb_define_method(k,n,RUBY_METHOD_FUNC(f),c)
+
+extern "C" {
+    void Init_keyword()
+    {
+        VALUE cKeyword = rb_define_class_under(mJieba, "Keyword", rb_cObject);
+        u8_enc = rb_utf8_encoding();
+        rb_define_alloc_func(cKeyword, allocate);
+        DEF(cKeyword, "_init", init, 6);
+        DEF(cKeyword, "extract",extract,2);
+    }
+
+}

data/ext/jieba/keyword.h

@@ -0,0 +1,17 @@
+#ifndef RUBY_JIEBA_KEYWORD
+#define RUBY_JIEBA_KEYWORD
+
+#include <jieba.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    void Init_keyword();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

data/ext/jieba/segment.cc

@@ -0,0 +1,107 @@
+#include "segment.h"
+#include <ruby/encoding.h>
+#include <MPSegment.hpp>
+#include <HMMSegment.hpp>
+#include <MixSegment.hpp>
+
+static rb_encoding* u8_enc;
+
+struct SegWrapper{
+    cppjieba::MixSegment *mixp;
+    cppjieba::HMMSegment *hmmp;
+    cppjieba::MPSegment *mpsp;
+    SegWrapper(): mixp(nullptr), hmmp(nullptr), mpsp(nullptr) {}
+};
+
+static void seg_free(void *p){
+    auto seg = reinterpret_cast<SegWrapper *>(p);
+    if (seg->mixp)
+        delete seg->mixp;
+    else if (seg->hmmp)
+        delete seg->hmmp;
+    else
+        delete seg->mpsp;
+    delete seg;
+}
+
+static VALUE allocate(VALUE klass)
+{
+    SegWrapper* seg_wrapper = new SegWrapper();
+    return Data_Wrap_Struct(klass, NULL, seg_free, seg_wrapper);
+}
+
+static void seg_init(VALUE self,
+                     VALUE type_rb_sym,
+                     VALUE jieba_dict_rbs,
+                     VALUE hmm_dict_rbs,
+                     VALUE user_dict_rbs)
+{
+    SegWrapper* seg_wrapper;
+    Data_Get_Struct(self, SegWrapper, seg_wrapper);
+
+    Check_Type(jieba_dict_rbs, T_STRING);
+    Check_Type(hmm_dict_rbs, T_STRING);
+    Check_Type(user_dict_rbs, T_STRING);
+
+    std::string jieba_dict = StringValueCStr(jieba_dict_rbs);
+    std::string hmm_dict = StringValueCStr(hmm_dict_rbs);
+    std::string user_dict = StringValueCStr(user_dict_rbs);
+
+    ID type = SYM2ID(type_rb_sym);
+    if ( type == rb_intern("mix") )
+    {
+        seg_wrapper->mixp = new cppjieba::MixSegment(jieba_dict, hmm_dict, user_dict);
+    }
+    else if ( type == rb_intern("hmm") )
+    {
+        seg_wrapper->hmmp = new cppjieba::HMMSegment(hmm_dict);
+    }
+    else if ( type == rb_intern("mp"))
+    {
+        seg_wrapper->mpsp = new cppjieba::MPSegment(jieba_dict);
+    }
+}
+
+static VALUE seg_cut(VALUE self, VALUE text_rbs)
+{
+    Check_Type(text_rbs, T_STRING);
+    std::string text = StringValueCStr(text_rbs);
+
+    SegWrapper* seg_wrapper;
+    Data_Get_Struct(self, SegWrapper, seg_wrapper);
+
+    std::vector<std::string> words;
+
+    if (seg_wrapper->mixp) {
+        seg_wrapper->mixp->Cut(text, words);
+    }
+    else if (seg_wrapper->hmmp) {
+        seg_wrapper->hmmp->Cut(text, words);
+    }
+    else {
+        seg_wrapper->mpsp->Cut(text, words);
+    }
+
+    volatile VALUE arr = rb_ary_new();
+    for (std::vector<std::string>::const_iterator j = words.begin(); j != words.end(); j++)
+    {
+
+        rb_ary_push(arr, rb_enc_str_new((*j).c_str(), (*j).length(), u8_enc));
+
+    }
+    return arr;
+}
+
+#define DEF(k,n,f,c) rb_define_method(k,n,RUBY_METHOD_FUNC(f),c)
+
+extern "C" {
+    void Init_segment()
+    {
+        VALUE cSegment = rb_define_class_under(mJieba, "Segment", rb_cObject);
+        u8_enc = rb_utf8_encoding();
+        rb_define_alloc_func(cSegment, allocate);
+        DEF(cSegment, "_init",seg_init,4);
+        DEF(cSegment, "cut",seg_cut,1);
+    }
+
+}

data/ext/jieba/segment.h

@@ -0,0 +1,17 @@
+#ifndef RUBY_JIEBA_SEGMENT
+#define RUBY_JIEBA_SEGMENT
+
+#include <jieba.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    void Init_segment();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

data/ext/jieba/tagging.cc

@@ -0,0 +1,76 @@
+#include "tagging.h"
+#include <ruby/encoding.h>
+#include <PosTagger.hpp>
+#include <MixSegment.hpp>
+
+static rb_encoding* u8_enc;
+
+struct Tagging{
+    cppjieba::MixSegment *p;
+};
+
+static void tagger_free(void *p){
+    delete reinterpret_cast<Tagging *>(p)->p;
+    delete reinterpret_cast<Tagging *>(p);
+}
+
+static VALUE alloc(VALUE klass)
+{
+    Tagging * tagging = new Tagging();
+    return Data_Wrap_Struct(klass, NULL, tagger_free, tagging);
+}
+
+static void init(VALUE self,
+                 VALUE jieba_dict_rbs,
+                 VALUE hmm_dict_rbs,
+                 VALUE user_dict_rbs)
+{
+    Tagging *tagging;
+    Data_Get_Struct(self, Tagging, tagging);
+
+    Check_Type(jieba_dict_rbs, T_STRING);
+    Check_Type(hmm_dict_rbs, T_STRING);
+    Check_Type(user_dict_rbs, T_STRING);
+
+    std::string jieba_dict = StringValueCStr(jieba_dict_rbs);
+    std::string hmm_dict = StringValueCStr(hmm_dict_rbs);
+    std::string user_dict = StringValueCStr(user_dict_rbs);
+
+    tagging->p = new cppjieba::MixSegment(jieba_dict, hmm_dict, user_dict);
+}
+
+static VALUE tag(VALUE self, VALUE text_rbs)
+{
+    Check_Type(text_rbs, T_STRING);
+    std::string text = StringValueCStr(text_rbs);
+
+    Tagging *tagging;
+    Data_Get_Struct(self, Tagging, tagging);
+
+    std::vector<std::pair<std::string, std::string>> pairs;
+    tagging->p->Tag(text, pairs);
+
+    volatile VALUE arr = rb_ary_new();
+    for (std::vector<std::pair<std::string, std::string>>::const_iterator j = pairs.begin(); j != pairs.end(); j++)
+    {
+        VALUE pair = rb_hash_new();
+        rb_hash_aset(pair, rb_enc_str_new(std::get<0>(*j).c_str(), std::get<0>(*j).length(), u8_enc), rb_enc_str_new(std::get<1>(*j).c_str(), std::get<1>(*j).length(), u8_enc));
+        rb_ary_push(arr, pair);
+
+    }
+    return arr;
+}
+
+#define DEF(k,n,f,c) rb_define_method(k,n,RUBY_METHOD_FUNC(f),c)
+
+extern "C" {
+    void Init_tagging()
+    {
+        VALUE cTagging = rb_define_class_under(mJieba, "Tagging", rb_cObject);
+        u8_enc = rb_utf8_encoding();
+        rb_define_alloc_func(cTagging, alloc);
+        DEF(cTagging, "_init",init,3);
+        DEF(cTagging, "tag",tag,1);
+    }
+
+}

data/ext/jieba/tagging.h

@@ -0,0 +1,17 @@
+#ifndef RUBY_JIEBA_TAGGING
+#define RUBY_JIEBA_TAGGING
+
+#include <jieba.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+    void Init_tagging();
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif

data/jieba_rb.gemspec

@@ -0,0 +1,51 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'jieba_rb/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "jieba-rb"
+  spec.version       = JiebaRb::VERSION
+  spec.authors       = ["Chris Li"]
+  spec.email         = ["liqi8822@gmail.com"]
+  spec.summary       = "cppjieba binding for ruby"
+  spec.description   = "cppjieba binding for ruby"
+  spec.homepage = "https://github.com/Xu-Zhiqing/jieba_rb"
+  spec.required_ruby_version = ">=1.9.2"
+  spec.license       = "MIT"
+  spec.extensions = ["ext/jieba/extconf.rb"]
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  relative_path = File.expand_path("../", __FILE__) + '/'
+  `git submodule --quiet foreach pwd`.split($\).each do |submodule_path|
+    if (ENV['OS'] == 'Windows_NT') && submodule_path[0] == '/'
+      # Detect if cygwin path is being used by git
+      submodule_path = submodule_path[1..-1]
+      submodule_path.insert(1, ':')
+    end
+    # for each submodule, change working directory to that submodule
+    Dir.chdir(submodule_path) do
+      # Make the submodule path relative
+      submodule_path = submodule_path.gsub(/#{relative_path}/i, '')
+      # issue git ls-files in submodule's directory
+      submodule_files = `git ls-files`.split($\).reject { |i| i.start_with? 'test/' }
+
+      # prepend the submodule path to create relative file paths
+      submodule_files_paths = submodule_files.map do |filename|
+        File.join(submodule_path, filename)
+      end
+
+      # add relative paths to gem.files
+      spec.files += submodule_files_paths
+    end
+  end
+
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rake-compiler"
+  spec.add_development_dependency "minitest"
+end

data/lib/jieba-rb.rb

@@ -0,0 +1,66 @@
+require "jieba_rb/version"
+require "jieba"
+module JiebaRb
+  abs = File.expand_path File.dirname(__FILE__)
+  EXT_BASE = "#{abs}/../ext/cppjieba/"
+  DEFAULT_JIEBA_DICT = EXT_BASE + "dict/jieba.dict.utf8";
+  DEFAULT_HMM_DICT = EXT_BASE + "dict/hmm_model.utf8";
+  DEFAULT_USER_DICT = EXT_BASE + "dict/user.dict.utf8";
+
+  class Segment
+    private :_init
+    def initialize opts = {}
+      valid_modes = [:mix, :hmm, :mp]
+      if mode = opts[:mode]
+        raise "Mode must be one of :mix :hmm :mp" unless valid_modes.include? mode
+      else
+        mode = :mix #default
+      end
+
+      jieba_dict = opts[:jieba_dict] || DEFAULT_JIEBA_DICT
+      hmm_dict = opts[:hmm_dict] || DEFAULT_HMM_DICT
+      user_dict = opts[:user_dict] || ""
+      user_dict = DEFAULT_USER_DICT if user_dict == :default
+
+      _init mode, jieba_dict, hmm_dict, user_dict
+    end
+  end
+
+  class Keyword
+    DEFAULT_IDF = EXT_BASE + "dict/idf.utf8"
+    DEFAULT_STOP_WORDS = EXT_BASE + "dict/stop_words.utf8"
+
+    private :_init
+
+    def initialize opts = {}
+      valid_modes = [:tf_idf]
+      if mode = opts[:mode]
+        raise "Mode must be one of :tf_idf" unless valid_modes.include? mode
+      else
+        mode = :tf_idf #default
+      end
+
+      jieba_dict = opts[:jieba_dict] || DEFAULT_JIEBA_DICT
+      hmm_dict = opts[:hmm_dict] || DEFAULT_HMM_DICT
+      idf_path = opts[:idf] || DEFAULT_IDF
+      stop_words_path = opts[:stop_words] || DEFAULT_STOP_WORDS
+
+      user_dict = opts[:user_dict] || ""
+      user_dict = DEFAULT_USER_DICT if user_dict == :default
+
+      _init mode, jieba_dict, hmm_dict, idf_path, stop_words_path, user_dict
+    end
+  end
+
+  class Tagging
+    private :_init
+    def initialize opts = {}
+      jieba_dict = opts[:jieba_dict] || DEFAULT_JIEBA_DICT
+      hmm_dict = opts[:hmm_dict] || DEFAULT_HMM_DICT
+      user_dict = opts[:user_dict] || ""
+      user_dict = DEFAULT_USER_DICT if user_dict == :default
+
+      _init jieba_dict, hmm_dict, user_dict
+    end
+  end
+end