maxixe 0.0.1 → 0.1.0

data/.gitignore

@@ -2,3 +2,4 @@
 .bundle
 Gemfile.lock
 pkg/*
+*.sw?

data/Gemfile

@@ -2,3 +2,4 @@ source "http://rubygems.org"
 
 # Specify your gem's dependencies in maxixe.gemspec
 gemspec
+

data/README.md

@@ -0,0 +1,10 @@
+# Maxixe
+### A simple statistical segmenter for any language
+
+## About
+
+Maxixe is an implementation of the Tango algorithm describe in the paper "Mostly-unsupervised statistical segmentation of Japanese kanji sequences" by Ando and Lee. While the paper deals with Japanese characters, it should work on any unsegmented text given enough corpus data and a tuning of the algorithm paramenters.
+
+## How to use
+
+First, you need a hash that contains the count of all n-grams in a given corpus.

data/lib/maxixe.rb

@@ -1,3 +1,4 @@
+require "text"
 module Maxixe
   class Segmenter
     
@@ -98,28 +99,51 @@ module Maxixe
 
   class Trainer
 
-    def self.generate_and_dump(n, output, *files)
-      res = self.generate_training_data(n, *files)
-      File.open(output,"w") do |file|
-        Yajl::Encoder.encode res, file
+    def self.generate_corpus_from_io(n , io)
+      result = n.inject({}){|r, c_n| r[c_n.to_s] = Hash.new{0}; r} 
+      io.each_line do |line|
+        n.each do |c_n|
+          n_grams = line.each_char.each_cons(c_n).map(&:join).to_a
+          n_grams.each do |n_gram|
+            result[c_n.to_s][n_gram] += 1
+          end
+        end
       end
+      result
     end
 
-    def self.generate_training_data(n, *files)
-      result = n.inject({}){|r, c_n| r[c_n.to_s] = Hash.new{0}; r} 
+    def self.optimize(index, samples)
+      res = check_recognition(index, samples)
+      min = nil
+      res.each do |n, ts|
+        ts.each do |t, score|
+          if !min or score < min[1]
+            min = [[n,t],score]
+          end
+        end
+      end 
+      {:n => min[0][0], :t => min[0][1], :score => min[1]}
+    end
 
-      files.each do |file|
-        input = open(file)
-        input.each_line do |line|
-          n.each do |c_n|
-            n_grams = line.each_char.each_cons(c_n).map(&:join).to_a
-            n_grams.each do |n_gram|
-              result[c_n.to_s][n_gram] += 1
-            end
+    def self.check_recognition(index, samples)
+      # Get all subsets of N
+      ns = 1.upto(index.keys.size).map{|i| index.keys.combination(i).to_a}.flatten(1)
+      results = ns.inject({}) do |res, n|
+        n_index = index.select{|key, value| n.include? key}
+        m = Maxixe::Segmenter.new(n_index)
+
+        t_values = ((0.1)..(1.0)).step(0.1).inject({}) do |res, t|
+          difference = samples.inject(0) do |result, (not_split, split)|
+            temp = m.segment(not_split, t)
+            result += Text::Levenshtein.distance(temp, split) 
           end
+          res[t] = difference
+          res
         end
+        res[n] = t_values
+        res
       end
-      result
+      results
     end
   end
 end

data/lib/maxixe/version.rb

@@ -1,3 +1,3 @@
 module Maxixe
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
 end

data/maxixe.gemspec

@@ -10,11 +10,11 @@ Gem::Specification.new do |s|
   s.email       = ["maxixe@rogerbraun.net"]
   s.homepage    = "https://github.com/rogerbraun/Maxixe"
   s.summary     = %q{A small statistical segmenter for any language.}
-  s.description = %q{Maxixe is an implementation of the Tango algorithm describe in the paper "Mostly-unsupervised statistical segmentation of Japanese kanji sequences" by Ando and Lee. While the paper deals with Japanese characters, it should work on any unsegmented text given enough corpus data and a tuning of the algorithm paramenters.}
+  s.description = %q{Maxixe is an implementation of the Tango algorithm describe in the paper "Mostly-unsupervised statistical segmentation of Japanese kanji sequences" by Ando and Lee. While the paper deals with Japanese characters, it should work on any unsegmented text given enough corpus data and a tuning of the algorithm parameters.}
 
   s.rubyforge_project = "maxixe"
 
-  s.add_dependency "yajl-ruby"
+  s.add_dependency "text"
   s.add_development_dependency "rspec"
 
   s.files         = `git ls-files`.split("\n")

data/spec/segmenter/segmenter_spec.rb

@@ -56,12 +56,17 @@ describe Maxixe::Segmenter do
   end
 
   describe "Segmenting Text" do
-    before(:each) do 
-      @segmenter = Maxixe::Segmenter.new({"2"=>{"AB"=>2, "BC"=>2, "CD"=>1, "DE"=>1, "EF"=>1, "FG"=>1, "G\n"=>1, "CX"=>1, "XY"=>1, "YZ"=>1, "Z\n"=>1}, "3"=>{"ABC"=>2, "BCD"=>1, "CDE"=>1, "DEF"=>1, "EFG"=>1, "FG\n"=>1, "BCX"=>1, "CXY"=>1, "XYZ"=>1, "YZ\n"=>1}})
-    end
 
-    it "should be able to segment text" do
-      @segmenter.segment("ABCDE").should == "ABC DE"
+    it "should do some examples" do 
+      index = Maxixe::Trainer.generate_corpus_from_io([3], "ILIKEMYDOG
+  THISHOUSEISMYHOUSE
+  MYDOGISSONICE
+  INMYHOUSETHEREAREFOURDOGS
+  IWANTAHOUSEFORMYDOG")
+      
+      m = Maxixe::Segmenter.new(index,0.3)
+      m.segment("FOURNICEDOGS").should == "FOUR NICE DOGS"
+      m.segment("MYDOGISINTHEHOUSE").should == "MY DOG IS IN THE HOUSE"
     end
   end
 end

data/spec/trainer/trainer_spec.rb

@@ -2,12 +2,32 @@ require "spec_helper"
 
 describe Maxixe::Trainer do
   
-  it "should generate n-gram data from a set of files" do
+  it "should generate n-gram data from IOs" do
 
     pwd = File.dirname(__FILE__)
 
-    Maxixe::Trainer.generate_training_data([2,3], File.join(pwd, "first_file"), File.join(pwd,"second_file")).should == {"2"=>{"AB"=>2, "BC"=>2, "CD"=>1, "DE"=>1, "EF"=>1, "FG"=>1, "G\n"=>1, "CX"=>1, "XY"=>1, "YZ"=>1, "Z\n"=>1}, "3"=>{"ABC"=>2, "BCD"=>1, "CDE"=>1, "DEF"=>1, "EFG"=>1, "FG\n"=>1, "BCX"=>1, "CXY"=>1, "XYZ"=>1, "YZ\n"=>1}}
+    Maxixe::Trainer.generate_corpus_from_io([2,3], open(File.join(pwd, "first_file"))).should == {"2"=>{"AB"=>1, "BC"=>1, "CD"=>1, "DE"=>1, "EF"=>1, "FG"=>1, "G\n"=>1}, "3"=>{"ABC"=>1, "BCD"=>1, "CDE"=>1, "DEF"=>1, "EFG"=>1, "FG\n"=>1}}
 
   end
 
+  it "should be able to find the optimal threshold and n values" do
+    pre_segmented = [["MYDOGISINTHEHOUSE", "MY DOG IS IN THE HOUSE"],
+                     ["FOURNICEDOGS", "FOUR NICE DOGS"],
+                     ["MYCATLIKESMYDOG", "MY CAT LIKES MY DOG"]]
+    index = Maxixe::Trainer.generate_corpus_from_io([2,3,4,5], "ILIKEMYDOG
+THISHOUSEISMYHOUSE
+MYDOGISSONICE
+WHOLIKESDOGSANYWAY
+CATSANDDOGSUSUALLYFIGHT
+INMYHOUSETHEREAREFOURDOGS
+IWANTAHOUSEFORMYDOG")
+  
+    optimal = Maxixe::Trainer.optimize(index, pre_segmented)
+    optimal[:n].should == ["2","4"]
+    optimal[:score].should == 0
+    optimal[:t].should be_within(0.01).of(0.5)
+
+  end
+
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: maxixe
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-20 00:00:00.000000000 %:z
-default_executable: 
+date: 2011-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: yajl-ruby
-  requirement: &72352710 !ruby/object:Gem::Requirement
+  name: text
+  requirement: &81697730 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *72352710
+  version_requirements: *81697730
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &72352370 !ruby/object:Gem::Requirement
+  requirement: &81697520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,11 +32,11 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *72352370
+  version_requirements: *81697520
 description: Maxixe is an implementation of the Tango algorithm describe in the paper
   "Mostly-unsupervised statistical segmentation of Japanese kanji sequences" by Ando
   and Lee. While the paper deals with Japanese characters, it should work on any unsegmented
-  text given enough corpus data and a tuning of the algorithm paramenters.
+  text given enough corpus data and a tuning of the algorithm parameters.
 email:
 - maxixe@rogerbraun.net
 executables: []
@@ -47,21 +46,16 @@ files:
 - .gitignore
 - .rspec
 - Gemfile
+- README.md
 - Rakefile
-- lib/.maxixe.rb.swp
 - lib/maxixe.rb
 - lib/maxixe/version.rb
 - maxixe.gemspec
-- spec/segmenter/.segmenter_spec.rb.swp
 - spec/segmenter/segmenter_spec.rb
 - spec/spec_helper.rb
-- spec/trainer/.first_file.swp
-- spec/trainer/.second_file.swp
-- spec/trainer/.trainer_spec.rb.swp
 - spec/trainer/first_file
 - spec/trainer/second_file
 - spec/trainer/trainer_spec.rb
-has_rdoc: true
 homepage: https://github.com/rogerbraun/Maxixe
 licenses: []
 post_install_message: 
@@ -82,7 +76,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: maxixe
-rubygems_version: 1.6.1
+rubygems_version: 1.8.9
 signing_key: 
 specification_version: 3
 summary: A small statistical segmenter for any language.