plexus-rmmseg 0.1.6

data/lib/rmmseg/ferret.rb

@@ -0,0 +1,109 @@
+# This file integrate RMMSeg with Ferret
+require 'singleton'
+require 'rubygems'
+require 'ferret'
+require 'rmmseg'
+
+module RMMSeg
+  module Ferret
+    # The Analyzer class can be used with Ferret .
+    class Analyzer < ::Ferret::Analysis::Analyzer
+      
+      # Construct an Analyzer. Optional block can be used to
+      # add more +TokenFilter+s. e.g.
+      #
+      #   analyzer = RMMSeg::Ferret::Analyzer.new { |tokenizer|
+      #     Ferret::Analysis::LowerCaseFilter.new(tokenizer)
+      #   }
+      #
+      def initialize(&brk)
+        @brk = brk
+      end
+      
+      def token_stream(field, text)
+        t = PunctuationFilter.new(Tokenizer.new(text))
+        if @brk
+          @brk.call(t)
+        else
+          t
+        end
+      end
+    end
+
+    # The Tokenizer tokenize text with RMMSeg::Algorithm.
+    class Tokenizer < ::Ferret::Analysis::TokenStream
+      # Create a new Tokenizer to tokenize +text+
+      def initialize(str)
+        self.text = str
+      end
+
+      # Get next token
+      def next
+        @algor.next_token
+      end
+      
+      # Get the text being tokenized
+      def text
+        @text
+      end
+
+      # Set the text to be tokenized
+      def text=(str)
+        @text = str
+        @algor = RMMSeg::Config.algorithm_instance(@text,
+                                                   ::Ferret::Analysis::Token)
+      end
+    end
+
+    # PunctuationFilter filter out the stand alone Chinese
+    # punctuation tokens.
+    class PunctuationFilter < ::Ferret::Analysis::TokenStream
+      # The punctuation dictionary.
+      class Dictionary
+        include Singleton
+
+        DIC_FILE = File.join(File.dirname(__FILE__),
+                             "..",
+                             "..",
+                             "data",
+                             "punctuation.dic")
+        def initialize
+          @dic = Hash.new
+          File.open(DIC_FILE, "r") do |f|
+            f.each_line { |line|
+              @dic[line.chomp.freeze] = nil
+            }
+          end
+        end
+
+        def include?(str)
+          @dic.has_key?(str)
+        end
+      end
+      
+      def initialize(stream)
+        @stream = stream
+      end
+
+      # Get next token, skip stand alone Chinese punctuations.
+      def next
+        token = @stream.next
+        dic = Dictionary.instance
+
+        until token.nil? || !(dic.include? token.text)
+          token = @stream.next
+        end
+
+        token
+      end
+
+      def text
+        @stream.text
+      end
+
+      def text=(str)
+        @stream.text = str
+      end
+    end
+  end
+end

data/lib/rmmseg/lawl_rule.rb

@@ -0,0 +1,12 @@
+require 'rmmseg/rule_helper'
+
+module RMMSeg
+  # Largest average word length rule.
+  class LAWLRule
+    def self.filter(chunks)
+      chunks.take_highest { |a, b|
+        Chunk::average_length(a) <=> Chunk::average_length(b)
+      }
+    end
+  end
+end

data/lib/rmmseg/lsdmfocw_rule.rb

@@ -0,0 +1,13 @@
+require 'rmmseg/rule_helper'
+
+module RMMSeg
+  # Largest sum of degree of morphemic freedom of one-character
+  # words rule.
+  class LSDMFOCWRule
+    def self.filter(chunks)
+      chunks.take_highest { |a, b|
+        Chunk::degree_of_morphemic_freedom(a) <=> Chunk::degree_of_morphemic_freedom(b)
+      }
+    end
+  end
+end

data/lib/rmmseg/mm_rule.rb

@@ -0,0 +1,13 @@
+require 'rmmseg/rule_helper'
+
+module RMMSeg
+  # Maximum matching rule, select the chunks with the
+  # maximum length.
+  class MMRule
+    def self.filter(chunks)
+      chunks.take_highest { |a, b|
+        Chunk::total_length(a) <=> Chunk::total_length(b)
+      }
+    end
+  end
+end

data/lib/rmmseg/rule_helper.rb

@@ -0,0 +1,28 @@
+class Array
+  # Take the elements with the highest value. Value are compared
+  # through the block. e.g
+  #
+  #   ["aaaa", "bb", "cccc"].take_highest { |a, b|
+  #     a.length <=> b.length
+  #   }
+  #   # => ["aaaa", "cccc"]
+  #
+  def take_highest
+    return [] if empty?
+    
+    rlt = [self.first]
+    max = self.first
+
+    for i in 1...length
+      cmp = yield(self[i], max)
+      if cmp == 0
+        rlt << self[i]
+      elsif cmp > 0
+        max = self[i]
+        rlt = [max]
+      end
+    end
+
+    rlt
+  end
+end

data/lib/rmmseg/simple_algorithm.rb

@@ -0,0 +1,37 @@
+require 'rmmseg/algorithm'
+require 'rmmseg/mm_rule'
+
+module RMMSeg
+  class SimpleAlgorithm
+    include Algorithm
+
+    # Create a new SimpleAlgorithm . The only rule used by this
+    # algorithm is MMRule .
+    def initialize(text, token=Token)
+      super
+    end
+
+    # Get the most proper CJK word.
+    def get_cjk_word
+      dic = Dictionary.instance
+      i = Config.max_word_length
+      if i + @index > @chars.length
+        i = @chars.length - @index
+      end
+      chars = @chars[@index, i]
+      word = chars.join
+
+      while i > 1 && !dic.has_word?(word)
+        i -= 1
+        word.slice!(-chars[i].size,chars[i].size) # truncate last char
+      end
+
+      token = @token.new(word, @byte_index, @byte_index+word.size)
+
+      @index += i
+      @byte_index += word.size
+
+      return token
+    end
+  end
+end

data/lib/rmmseg/svwl_rule.rb

@@ -0,0 +1,12 @@
+require 'rmmseg/rule_helper'
+
+module RMMSeg
+  # Smallest variance of word length rule.
+  class SVWLRule
+    def self.filter(chunks)
+      chunks.take_highest { |a, b|
+        Chunk::variance(b) <=> Chunk::variance(a)
+      }
+    end
+  end
+end

data/lib/rmmseg/token.rb

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+module RMMSeg
+  # A Token consists of a term's text and the start and end offset
+  # of the term.
+  class Token
+    # The text of the token
+    attr_accessor :text
+    
+    # The start position of the token. This is *byte* index instead of
+    # character.
+    attr_accessor :start
+
+    # The one greater than the position of the last byte of the
+    # token. This is *byte* index instead of character.
+    attr_accessor :end
+
+    # +text+ is the ref to the whole text. In other words:
+    # +text[start_pos...end_pos]+ should be the string held by this
+    # token.
+    def initialize(text, start_pos, end_pos)
+      @text  = text
+      @start = start_pos
+      @end   = end_pos
+    end
+
+    def to_s
+      @text.dup
+    end
+  end
+end

data/lib/rmmseg/version.rb

@@ -0,0 +1,3 @@
+module RMMSeg
+  VERSION = '0.1.6'
+end

data/lib/rmmseg/word.rb

@@ -0,0 +1,38 @@
+module RMMSeg
+  # An object representing a CJK word.
+  class Word
+    TYPES = {
+      :unrecognized => :unrecognized,
+      :basic_latin_word => :basic_latin_word,
+      :cjk_word => :cjk_word
+    }.freeze
+
+    # The content text of the word.
+    attr_reader :text
+
+    # The type of the word, may be one of the key of TYPES .
+    attr_reader :type
+
+    # The frequency of the word. This value is meaningful only
+    # when this is a one-character word.
+    attr_reader :frequency
+
+    # Initialize a Word object.
+    def initialize(text, type=TYPES[:unrecognized], frequency=nil)
+      @text = text
+      @type = type
+      @frequency = frequency
+      @length = @text.length
+    end
+
+    # The number of characters in the word. *Not* number of bytes.
+    def length
+      @length
+    end
+
+    # The number of bytes in the word.
+    def byte_size
+      @text.each_byte.to_a.length
+    end
+  end
+end

data/misc/ferret_example.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'rmmseg'
+require 'rmmseg/ferret'
+
+analyzer = RMMSeg::Ferret::Analyzer.new { |tokenizer|
+  Ferret::Analysis::LowerCaseFilter.new(tokenizer)
+}
+
+$index = Ferret::Index::Index.new(:analyzer => analyzer)
+
+$index << {
+  :title => "分词",
+  :content => "中文分词比较困难，不像英文那样，直接在空格和标点符号的地方断开就可以了。"
+}
+$index << {
+  :title => "RMMSeg",
+  :content => "RMMSeg 我近日做的一个 Ruby 中文分词实现，下一步是和 Ferret 进行集成。"
+}
+$index << {
+  :title => "Ruby 1.9",
+  :content => "Ruby 1.9.0 已经发布了，1.9 的一个重大改进就是对 Unicode 的支持。"
+}
+$index << {
+  :title => "Ferret",
+  :content => <<END
+Ferret is a high-performance, full-featured text search engine library
+written for Ruby. It is inspired by Apache Lucene Java project. With
+the introduction of Ferret, Ruby users now have one of the fastest and
+most flexible search libraries available. And it is surprisingly easy
+to use.
+END
+}
+
+def highlight_search(key)
+  $index.search_each(%Q!content:"#{key}"!) do |id, score|
+    puts "*** Document \"#{$index[id][:title]}\" found with a score of #{score}"
+    puts "-"*40
+    highlights = $index.highlight("content:#{key}", id,
+                                  :field => :content,
+                                  :pre_tag => "\033[36m",
+                                  :post_tag => "\033[m")
+    puts "#{highlights}"
+    puts ""
+  end
+end
+
+ARGV.each { |key|
+  puts "\033[33mSearching for #{key}...\033[m"
+  puts ""
+  highlight_search(key)
+}
+
+# Local Variables:
+# coding: utf-8
+# End:

data/misc/homepage.erb

@@ -0,0 +1,170 @@
+<%# -*- mode: text; coding: utf-8 -*- %>
+<%
+  $title = "RMMSeg Homepage"
+  $authors = { 'pluskid' => 'http://pluskid.lifegoo.com' }
+
+  $unindent = '  '
+%>
+
+<% chapter "Introduction" do %> 
+
+  RMMSeg is an implementation of
+  "MMSEG":http://technology.chtsai.org/mmseg/ Chinese word
+  segmentation algorithm. It is based on two variants of maximum
+  matching algorithms. Two algorithms are available for using:
+  
+  * simple algorithm that uses only forward maximum matching.
+  * complex algorithm that uses three-word chunk maximum matching and 3
+    aditonal rules to solve ambiguities.
+
+  For more information about the algorithm, please refer to the
+  following essays:
+
+  * http://technology.chtsai.org/mmseg/
+  * http://pluskid.lifegoo.com/?p=261
+
+  RMMSeg can be used as either a stand alone program or an Analyzer of
+  "Ferret":http://ferret.davebalmain.com/trac.
+
+<% end %>
+
+<% chapter "Setup" do %>
+  <% section "Requirements" do %>
+
+    Your system needs the following software to run RMMSeg.
+
+    |_. Software                  |_. Notes                   |
+    | "Ruby":http://ruby-lang.org | Version 1.8.x is required |
+    | "hoe":http://seattlerb.rubyforge.org/hoe/ | If you want to build the gem manually |
+    | "Rake":http://rake.rubyforge.org/ | If you want to build the gem manually |
+    | "rspec":http://rspec.rubyforge.org/ | If you want to run the testcases |
+
+  <% end %>
+
+  <% section "Installation" do %>
+    <% section "Using RubyGems" do %>
+      To install the gem remotely from "RubyForge":http://rubyforge.org :
+
+        sudo gem install rmmseg
+
+      Or you can download the gem file manually from "RubyForge":http://rubyforge.org/projects/rmmseg/ and install it locally:
+
+        sudo gem install --local rmmseg-x.y.z.gem
+
+    <% end %>
+
+    <% section "From Subversion" do %>
+      From subversion repository hosted at "RubyForge":http://rmmseg.rubyforge.org/svn/, you can always get the latest source code.
+      <% note "The latest code might be unstable" do %>
+        Some new features may only be available in the latest code in subversion, but the code might be broken in some cases. So it is recommended to use the released gem package for production.
+      <% end %>
+      To check out the code from Rubyforge, you need to install subversion, then:
+
+        svn checkout http://rmmseg.rubyforge.org/svn/trunk/ rmmseg
+
+      Then you can run
+
+        rake gem
+
+      to build the gem file.
+    <% end %>
+  <% end %>
+<% end %>
+
+<% chapter "Usage" do %>
+   
+  <% section "Stand Alone rmmseg" do %>
+    RMMSeg comes with a script @rmmseg@. To get the basic usage, just execute it with @-h@ option:
+      
+      rmmseg -h
+
+    It reads from STDIN and print result to STDOUT. Here is a real
+    example:
+
+      $ echo "我们都喜欢用 Ruby" | rmmseg
+      我们 都 喜欢 用 Ruby
+
+  <% end %>
+
+  <% section "Analyzer for Ferret" do %>
+    RMMSeg include an analyzer for Ferret. It is simply ready to
+    use. Just require it and pass it to Ferret. Here's a complete
+    example:
+
+    <code lang="ruby">
+    <%# include ferret_example.rb %>
+    </code>
+
+    execute it on the following key words:
+
+      $ ruby ferret_example.rb Ruby 中文
+
+    will generate the following results:
+    
+    <code lang="text">
+    Searching for Ruby...
+    
+    *** Document "RMMSeg" found with a score of 0.21875
+    ----------------------------------------
+    RMMSeg 我近日做的一个 Ruby 中文分词实现，下一步是和 Ferret 进行集成。
+    
+    *** Document "Ruby 1.9" found with a score of 0.21875
+    ----------------------------------------
+    Ruby 1.9.0 已经发布了，1.9 的一个重大改进就是对 Unicode 的支持。
+    
+    *** Document "Ferret" found with a score of 0.176776692271233
+    ----------------------------------------
+    Ferret is a high-performance, full-featured text search engine library
+    written for Ruby. It is inspired by Apache Lucene Java project. With
+    the introduction of Ferret, Ruby users now have one of the fastest and
+    most flexible search libraries available. And it's surprisingly easy
+    to use.
+    
+    Searching for 中文...
+    
+    *** Document "分词" found with a score of 0.281680464744568
+    ----------------------------------------
+    中文分词比较困难，不像英文那样，直接在空格和标点符号的地方断开就可以了。
+    
+    *** Document "RMMSeg" found with a score of 0.281680464744568
+    ----------------------------------------
+    RMMSeg 我近日做的一个 Ruby 中文分词实现，下一步是和 Ferret 进行集成。
+    </code>
+
+    And if you run the example in terminal, you'll see the result
+    highlighted as in <%= xref "Ferret Example Screenshot" %>.
+
+    <% figure "Ferret Example Screenshot" do %>
+      !http://pluskid.lifegoo.com/wp-content/uploads/2008/02/rmmseg.png!
+    <% end %>
+
+  <% end %>
+
+  <% section "Customization" do %>
+    RMMSeg can be customized through @RMMSeg::Config@. For example, to use your own dictionaries, just set it before starting to do segmentation:
+
+    <code lang="ruby">
+    RMMSeg::Config.dictionaries = [["dict1.dic", true],  # with frequency info
+                                   ["dict2.dic", false], # without
+                                   ["dict3.dic", false]]
+    RMMSeg::Config.max_word_length = 6
+    </code>
+
+    Or to use the simple algorithm for more efficient (and less accurate) segmenting:
+
+    <code>
+    RMMSeg::Config.algorithm = :simple
+    </code>
+
+    For more information on customization, please refer to the RDoc of "RMMSeg::Config":http://rmmseg.rubyforge.org/rmmseg/index.html.
+  <% end %>
+
+<% end %>
+
+<% chapter "Resources" do %>
+  * "Project Home":http://rubyforge.org/projects/rmmseg/: The Project page at RubyForge.
+  * "RDoc of RMMSeg":http://rmmseg.rubyforge.org/rmmseg/index.html: The auto generated rdoc of RMMSeg.
+  * "A Screencast":http://pluskid.lifegoo.com/?p=272: Demo of Ferret RMMSeg and acts_as_ferret.
+  * "Implementation Details":http://pluskid.lifegoo.com/?p=261: My blog post about the implementation details of RMMSeg (Chinese).
+  * "Author's Email":mailto:pluskid@gmail.com: Contact me if you have any problem.
+<% end %>