pluskid-rmmseg-cpp 0.2.0

data/ext/rmmseg/rmmseg.cpp

@@ -0,0 +1,174 @@
+#include <ruby.h>
+#include <cstdio>               // for debug
+
+#include "token.h"
+#include "dict.h"
+#include "algor.h"
+
+using namespace std;
+
+extern "C" {
+
+    /*****************************************
+     * 
+     * Normal interface
+     *
+     *****************************************/
+
+    /*********************
+     * RMMSeg module
+     *********************/
+    static VALUE mRMMSeg;
+
+
+    /*********************
+     * Dictionary module
+     *********************/
+    static VALUE mDictionary;
+    
+    static VALUE dic_load_chars(VALUE mod, VALUE path)
+    {
+        if (rmmseg::dict::load_chars(RSTRING(path)->ptr))
+            return Qtrue;
+        return Qfalse;
+    }
+    static VALUE dic_load_words(VALUE mod, VALUE path)
+    {
+        if (rmmseg::dict::load_words(RSTRING(path)->ptr))
+            return Qtrue;
+        return Qfalse;
+    }
+    static VALUE dic_add(VALUE mod, VALUE word, VALUE len, VALUE freq)
+    {
+        const char *str = RSTRING(word)->ptr;
+        int nbytes = RSTRING(word)->len;
+        rmmseg::Word *w = rmmseg::make_word(str, FIX2INT(len), FIX2INT(freq), nbytes);
+        rmmseg::dict::add(w);
+        return Qnil;
+    }
+    static VALUE dic_has_word(VALUE mod, VALUE word)
+    {
+        const char *str = RSTRING(word)->ptr;
+        int nbytes = RSTRING(word)->len;
+        if (rmmseg::dict::get(str, nbytes) != NULL)
+            return Qtrue;
+        return Qfalse;
+    }
+
+
+    /**********************
+     * Token Class
+     **********************/
+    struct Token
+    {
+        VALUE text;
+        VALUE start;
+        VALUE end;
+    };
+
+    static void tk_mark(Token *t)
+    {
+        // start and end are Fixnums, no need to mark them
+        rb_gc_mark(t->text);
+    }
+    static void tk_free(Token *t)
+    {
+        free(t);
+    }
+
+    static VALUE tk_text(VALUE self)
+    {
+        Token *tk = (Token *)DATA_PTR(self);
+        return tk->text;
+    }
+    static VALUE tk_start(VALUE self)
+    {
+        Token *tk = (Token *)DATA_PTR(self);
+        return tk->start;
+    }
+    static VALUE tk_end(VALUE self)
+    {
+        Token *tk = (Token *)DATA_PTR(self);
+        return tk->end;
+    }
+
+    static VALUE cToken;
+    static VALUE tk_create(const char* base, const rmmseg::Token &t)
+    {
+        Token *tk = (Token *)malloc(sizeof(Token));
+        int start = t.text-base;
+        tk->text = rb_str_new(t.text, t.length);
+        tk->start = INT2FIX(start);
+        tk->end = INT2FIX(start + t.length);
+        return Data_Wrap_Struct(cToken,
+                                (RUBY_DATA_FUNC)tk_mark,
+                                (RUBY_DATA_FUNC)tk_free,
+                                tk);
+    }
+
+    /*********************
+     * Algorithm Class
+     *********************/
+    struct Algorithm
+    {
+        VALUE text;             // hold to avoid being garbage collected
+        rmmseg::Algorithm *algor;
+    };
+
+    static void algor_mark(Algorithm *a)
+    {
+        rb_gc_mark(a->text);
+    }
+    static void algor_free(Algorithm *a)
+    {
+        free(a->algor);
+    }
+
+    static VALUE cAlgorithm;
+    static VALUE algor_create(VALUE klass, VALUE text)
+    {
+        Algorithm *algor = (Algorithm *)malloc(sizeof(Algorithm));
+        void *mem;
+        algor->text = text;
+        mem = malloc(sizeof(rmmseg::Algorithm));
+        algor->algor = new(mem) rmmseg::Algorithm(RSTRING(text)->ptr,
+                                                  RSTRING(text)->len);
+        
+        return Data_Wrap_Struct(klass,
+                                (RUBY_DATA_FUNC)algor_mark,
+                                (RUBY_DATA_FUNC)algor_free,
+                                algor);
+    }
+    static VALUE algor_next_token(VALUE self)
+    {
+        Algorithm *algor = (Algorithm *)DATA_PTR(self);
+        rmmseg::Token tk = algor->algor->next_token();
+
+        if (tk.length == 0)
+            return Qnil;
+        return tk_create(RSTRING(algor->text)->ptr, tk);
+    }
+
+
+    void Init_rmmseg()
+    {
+        typedef VALUE (*RUBY_METHOD) (...);
+        mRMMSeg = rb_define_module("RMMSeg");
+
+        mDictionary = rb_define_module_under(mRMMSeg, "Dictionary");
+        rb_define_singleton_method(mDictionary, "load_chars", (RUBY_METHOD)dic_load_chars, 1);
+        rb_define_singleton_method(mDictionary, "load_words", (RUBY_METHOD)dic_load_words, 1);
+        rb_define_singleton_method(mDictionary, "load_add", (RUBY_METHOD)dic_add, 3);
+        rb_define_singleton_method(mDictionary, "has_word?", (RUBY_METHOD)dic_has_word, 1);
+
+        cToken = rb_define_class_under(mRMMSeg, "Token", rb_cObject);
+        rb_undef_method(rb_singleton_class(cToken), "new");
+        rb_define_method(cToken, "text", (RUBY_METHOD)tk_text, 0);
+        rb_define_method(cToken, "start", (RUBY_METHOD)tk_start, 0);
+        rb_define_method(cToken, "end", (RUBY_METHOD)tk_end, 0);
+
+        cAlgorithm = rb_define_class_under(mRMMSeg, "Algorithm", rb_cObject);
+        rb_define_singleton_method(cAlgorithm, "new", (RUBY_METHOD)algor_create, 1);
+        rb_define_method(cAlgorithm, "next_token", (RUBY_METHOD)algor_next_token, 0);
+    }
+}

data/ext/rmmseg/rules.h

@@ -0,0 +1,87 @@
+#ifndef _RULES_H_
+#define _RULES_H_
+
+#include <vector>
+#include <algorithm>
+
+#include "chunk.h"
+
+namespace rmmseg
+{
+    template <typename Cmp>
+    void take_highest(std::vector<Chunk> &chunks, Cmp &cmp)
+    {
+        int i = 1, j;
+        Chunk& max = chunks[0];
+
+        for (j = 1; j < chunks.size(); ++j)
+        {
+            int rlt = cmp(chunks[j], max);
+            if (rlt > 0)
+                i = 0;
+            if (rlt >= 0)
+                std::swap(chunks[i++], chunks[j]);
+        }
+        chunks.erase(chunks.begin()+i, chunks.end());
+    }
+
+    struct MMCmp_t
+    {
+        int operator()(Chunk &a, Chunk &b)
+        {
+            return a.total_length() - b.total_length();
+        }
+    } MMCmp;
+    void mm_filter(std::vector<Chunk> &chunks)
+    {
+        take_highest(chunks, MMCmp);
+    }
+
+    struct LAWLCmp_t
+    {
+        int operator()(Chunk &a, Chunk &b)
+        {
+            double rlt = a.average_length() - b.average_length();
+            if (rlt == 0)
+                return 0;
+            if (rlt > 0)
+                return 1;
+            return -1;
+        }
+    } LAWLCmp;
+    void lawl_filter(std::vector<Chunk> &chunks)
+    {
+        take_highest(chunks, LAWLCmp);
+    }
+
+    struct SVWLCmp_t
+    {
+        int operator()(Chunk &a, Chunk& b)
+        {
+            double rlt = a.variance() - b.variance();
+            if (rlt == 0)
+                return 0;
+            if (rlt < 0)
+                return 1;
+            return -1;
+        }
+    } SVWLCmp;
+    void svwl_filter(std::vector<Chunk> &chunks)
+    {
+        take_highest(chunks, SVWLCmp);
+    }
+
+    struct LSDMFOCWCmp_t
+    {
+        int operator()(Chunk &a, Chunk& b)
+        {
+            return a.degree_of_morphemic_freedom() - b.degree_of_morphemic_freedom();
+        }
+    } LSDMFOCWCmp;
+    void lsdmfocw_filter(std::vector<Chunk> &chunks)
+    {
+        take_highest(chunks, LSDMFOCWCmp);
+    }
+}
+
+#endif /* _RULES_H_ */

data/ext/rmmseg/token.h

@@ -0,0 +1,19 @@
+#ifndef _TOKEN_H_
+#define _TOKEN_H_
+
+namespace rmmseg
+{
+    struct Token
+    {
+        Token(const char *txt, int len)
+            :text(txt), length(len) { }
+        // `text' may or may not be nul-terminated, its length
+        // should be stored in the `length' field.
+        //
+        // if length is 0, this is an empty token
+        const char *text;
+        int length;
+    };
+}
+
+#endif /* _TOKEN_H_ */

data/ext/rmmseg/word.h

@@ -0,0 +1,44 @@
+#ifndef _WORD_H_
+#define _WORD_H_
+
+#include <climits>
+#include <cstring>
+
+#include "memory.h"
+
+namespace rmmseg
+{
+    const int word_embed_len = 4; /* at least 1 char (3 bytes+'\0') */
+    struct Word
+    {
+        unsigned char   nbytes;   /* number of bytes */
+        char            length;   /* number of characters */
+        unsigned short  freq;
+        char            text[word_embed_len];
+    };
+
+    /**
+     * text: the text of the word.
+     * length: number of characters (not bytes).
+     * freq: the frequency of the word.
+     */
+    inline Word *make_word(const char *text, int length=1,
+                           int freq=0, int nbytes=-1)
+    {
+        if (freq > USHRT_MAX)
+            freq = USHRT_MAX;   /* avoid overflow */
+        if (nbytes == -1)
+            nbytes = strlen(text);
+        Word *w = static_cast<Word *>(pool_alloc(sizeof(Word)
+                                                 + nbytes+1
+                                                 - word_embed_len));
+        w->nbytes = std::strlen(text);
+        w->length = length;
+        w->freq = freq;
+        std::strncpy(w->text, text, nbytes);
+        w->text[nbytes] = '\0';
+        return w;
+    }
+}
+
+#endif /* _WORD_H_ */

data/lib/rmmseg/dictionary.rb

@@ -0,0 +1,54 @@
+module RMMSeg
+  module Dictionary
+    @dictionaries = [
+                     [:chars, File.join(File.dirname(__FILE__),
+                                        "..", "..", "data", "chars.dic")],
+                     [:words, File.join(File.dirname(__FILE__),
+                                        "..", "..", "data", "words.dic")]
+                    ]
+
+    class << self
+      #
+      # An array of dictionaries used by RMMSeg. Each entry is of the
+      # following form:
+      #
+      #   [type, path]
+      #
+      # where +type+ can either <tt>:chars</tt> or <tt>:words</tt>. +path+ is the path
+      # to the dictionary file.
+      #
+      # The format of <tt>:chars</tt> dictionary is a collection of lines of the
+      # following form:
+      #
+      #   freq char
+      #
+      # Where +frequency+ is a number <b>less than 65535</b>. +char+ is the
+      # character. They are spearated by <b>exactly one space</b>.
+      #
+      # The format of <tt>:words</tt> dictionary is similar:
+      #
+      #   length word
+      #
+      # except the first number is not the frequency, but the number of
+      # characters (not number of bytes) in the word.
+      # 
+      attr_accessor :dictionaries
+
+      # Add a user defined dictionary, +type+ can be
+      # +:chars+ or <tt>:words</tt>. See doc of dictionaries.
+      def add_dictionary(path, type)
+        @dictionaries << [type, path]
+      end
+
+      def load_dictionaries
+        @dictionaries.each do |type, path|
+          if type == :chars
+            load_chars(path)
+          elsif type == :words
+            load_words(path)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rmmseg/ferret.rb

@@ -0,0 +1,112 @@
+require 'singleton'
+require 'rubygems'
+require 'rmmseg'
+require 'ferret'
+
+module RMMSeg
+  module Ferret
+        # The Analyzer class can be used with Ferret .
+    class Analyzer < ::Ferret::Analysis::Analyzer
+      
+      # Construct an Analyzer. Optional block can be used to
+      # add more +TokenFilter+s. e.g.
+      #
+      #   analyzer = RMMSeg::Ferret::Analyzer.new { |tokenizer|
+      #     Ferret::Analysis::LowerCaseFilter.new(tokenizer)
+      #   }
+      #
+      def initialize(&brk)
+        @brk = brk
+      end
+      
+      def token_stream(field, text)
+        t = PunctuationFilter.new(Tokenizer.new(text))
+        if @brk
+          @brk.call(t)
+        else
+          t
+        end
+      end
+    end
+
+    # The Tokenizer tokenize text with RMMSeg::Algorithm.
+    class Tokenizer < ::Ferret::Analysis::TokenStream
+      # Create a new Tokenizer to tokenize +text+
+      def initialize(str)
+        self.text = str
+      end
+
+      # Get next token
+      def next
+        tok = @algor.next_token
+        if tok.nil?
+          return nil
+        else
+          return ::Ferret::Analysis::Token.new(tok.text, tok.start, tok.end)
+        end
+      end
+      
+      # Get the text being tokenized
+      def text
+        @text
+      end
+
+      # Set the text to be tokenized
+      def text=(str)
+        @text = str
+        @algor = Algorithm.new(@text)
+      end
+    end
+
+    # PunctuationFilter filter out the stand alone Chinese
+    # punctuation tokens.
+    class PunctuationFilter < ::Ferret::Analysis::TokenStream
+      # The punctuation dictionary.
+      class Dictionary
+        include Singleton
+
+        DIC_FILE = File.join(File.dirname(__FILE__),
+                             "..",
+                             "..",
+                             "data",
+                             "punctuation.dic")
+        def initialize
+          @dic = Hash.new
+          File.open(DIC_FILE, "r") do |f|
+            f.each_line { |line|
+              @dic[line.chomp.freeze] = nil
+            }
+          end
+        end
+
+        def include?(str)
+          @dic.has_key?(str)
+        end
+      end
+      
+      def initialize(stream)
+        @stream = stream
+      end
+
+      # Get next token, skip stand alone Chinese punctuations.
+      def next
+        token = @stream.next
+        dic = Dictionary.instance
+
+        until token.nil? || !(dic.include? token.text)
+          token = @stream.next
+        end
+
+        token
+      end
+
+      def text
+        @stream.text
+      end
+
+      def text=(str)
+        @stream.text = str
+      end
+    end
+  end
+end

data/lib/rmmseg.rb

@@ -0,0 +1,3 @@
+require 'rmmseg/dictionary'
+require File.join(File.dirname(__FILE__), '..',
+                  'ext', 'rmmseg', 'rmmseg')

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: pluskid-rmmseg-cpp
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- pluskid
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-05-14 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: pluskid@gmail.com
+executables: []
+
+extensions: 
+- ext/rmmseg/extconf.rb
+extra_rdoc_files: 
+- README
+files: 
+- lib/rmmseg/dictionary.rb
+- lib/rmmseg/ferret.rb
+- lib/rmmseg.rb
+- ext/rmmseg/dict.cpp
+- ext/rmmseg/memory.cpp
+- ext/rmmseg/rmmseg.cpp
+- ext/rmmseg/algor.cpp
+- ext/rmmseg/rules.h
+- ext/rmmseg/dict.h
+- ext/rmmseg/token.h
+- ext/rmmseg/algor.h
+- ext/rmmseg/word.h
+- ext/rmmseg/chunk.h
+- ext/rmmseg/memory.h
+- data/punctuation.dic
+- data/words.dic
+- data/chars.dic
+- README
+- ext/rmmseg/extconf.rb
+has_rdoc: true
+homepage: 
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --main
+- README
+- --line-numbers
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: A high performance package for doing Chinese word segmentation.
+test_files: []
+