maxixe 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in maxixe.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler'
+require "rspec/core/rake_task"
+
+Bundler::GemHelper.install_tasks
+
+desc "Run specs"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = "./spec/**/*_spec.rb"
+end

data/lib/maxixe.rb

@@ -0,0 +1,125 @@
+module Maxixe
+  class Segmenter
+    
+    attr_accessor :t
+    
+    def initialize(index, t = 0.5)
+      @index = index
+      @n = index.keys.map(&:to_i)
+      @t = t
+    end
+
+    def segment(str, t = nil)
+
+      n_grams = all_n_grams(str)
+      
+      votes_for_all = n_grams.map{|n| compute_votes(straddling_and_non_straddling(n,str), n.first.size)}
+
+      averaged = average_votes(votes_for_all)
+
+      split_with_votes(averaged, str, t)
+
+    end
+
+    def split_with_votes(votes, str, t = nil)
+      points = []
+      votes.each_with_index do |vote, i|
+        treshold = vote > (t || @t)
+        maximum = if i > 0 and i < (votes.size - 1)
+          vote > votes[i - 1] and vote > votes[i + 1]
+        else false end 
+
+        points << i if treshold or maximum
+      end 
+
+      res = str.dup
+      offset = 1
+      points.each do |p|
+        res.insert(p + offset, " ")
+        offset += 1
+      end
+
+      res 
+
+    end
+
+    def all_n_grams str
+      @n.map do |n| str.each_char.each_cons(n).to_a end
+    end
+
+    def token_count(n_gram)
+      @index[n_gram.length.to_s][n_gram] || 0
+    end
+
+    def straddling_and_non_straddling n_grams, str
+      (0..(str.length - 2)).map do |pos|
+        [non_straddling(n_grams, pos), straddling(n_grams, pos)]
+      end
+    end
+
+    def non_straddling n_grams, pos
+      res = []
+      n_grams.each_with_index do |n_gram, i|
+        res << n_gram if i == pos + 1 or i == pos - (n_gram.size - 1)
+      end
+      res.map(&:join)
+    end
+
+    def straddling n_grams, pos
+      res = []
+      n_grams.each_with_index do |n_gram, i|
+        res << n_gram if i <= pos and i > pos - (n_gram.size - 1)
+      end
+      res.map(&:join)
+    end
+
+    def compute_votes positions_with_ngrams, n
+      positions_with_ngrams.map do |(non_strad, strad)|
+        compute_vote(non_strad, strad, n)
+      end
+    end
+
+    def compute_vote(non_strad, strad, n)
+      res = non_strad.inject(0) do |res, s|
+        res + strad.inject(0) do |res_2, t|
+          res_2 + ((token_count(s) > token_count(t)) ? 1 : 0)
+        end
+      end 
+      res / (2.0 * (n - 1))
+    end
+
+    def average_votes(votes)
+      votes.transpose.map do |vote_array|
+        vote_array.inject(&:+).to_f / vote_array.size
+      end
+    end
+  end
+
+
+  class Trainer
+
+    def self.generate_and_dump(n, output, *files)
+      res = self.generate_training_data(n, *files)
+      File.open(output,"w") do |file|
+        Yajl::Encoder.encode res, file
+      end
+    end
+
+    def self.generate_training_data(n, *files)
+      result = n.inject({}){|r, c_n| r[c_n.to_s] = Hash.new{0}; r} 
+
+      files.each do |file|
+        input = open(file)
+        input.each_line do |line|
+          n.each do |c_n|
+            n_grams = line.each_char.each_cons(c_n).map(&:join).to_a
+            n_grams.each do |n_gram|
+              result[c_n.to_s][n_gram] += 1
+            end
+          end
+        end
+      end
+      result
+    end
+  end
+end

data/lib/maxixe/version.rb

@@ -0,0 +1,3 @@
+module Maxixe
+  VERSION = "0.0.1"
+end

data/maxixe.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "maxixe/version"
+
+Gem::Specification.new do |s|
+  s.name        = "maxixe"
+  s.version     = Maxixe::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Roger Braun"]
+  s.email       = ["maxixe@rogerbraun.net"]
+  s.homepage    = "https://github.com/rogerbraun/Maxixe"
+  s.summary     = %q{A small statistical segmenter for any language.}
+  s.description = %q{Maxixe is an implementation of the Tango algorithm describe in the paper "Mostly-unsupervised statistical segmentation of Japanese kanji sequences" by Ando and Lee. While the paper deals with Japanese characters, it should work on any unsegmented text given enough corpus data and a tuning of the algorithm paramenters.}
+
+  s.rubyforge_project = "maxixe"
+
+  s.add_dependency "yajl-ruby"
+  s.add_development_dependency "rspec"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/spec/segmenter/segmenter_spec.rb

@@ -0,0 +1,67 @@
+require "spec_helper"
+
+describe Maxixe::Segmenter do
+  describe "internal functions" do
+
+    before(:each) do 
+      @sentence = "1234567"
+      @two_grams = @sentence.each_char.each_cons(2).to_a
+      @three_grams = @sentence.each_char.each_cons(3).to_a
+      @segmenter = Maxixe::Segmenter.new({})
+    end
+
+    it "should give all non_straddling n_grams for a given position" do
+
+      # only right segment exists
+      @segmenter.non_straddling(@two_grams, 0).should == ["23"] 
+      @segmenter.non_straddling(@three_grams, 0).should == ["234"] 
+
+      # only left segment exists
+      @segmenter.non_straddling(@two_grams, 5).should == ["56"]
+      @segmenter.non_straddling(@three_grams, 5).should == ["456"]
+
+      # both segments exists
+      @segmenter.non_straddling(@two_grams, 1).should == ["12","34"]
+      @segmenter.non_straddling(@three_grams, 2).should == ["123", "456"]
+
+    end
+
+    it "should give all straddling n_grams for a given position" do
+
+      @segmenter.straddling(@two_grams, 1).should == ["23"]
+      @segmenter.straddling(@three_grams, 1).should == ["123", "234"]
+      @segmenter.straddling(@three_grams, 0).should == ["123"]
+
+    end
+
+    it "should give all straddling and non straddling n-grams for a given string and all positions" do
+      
+      res = @segmenter.straddling_and_non_straddling(@two_grams, @sentence)
+
+      res.size.should == @sentence.size - 1
+      
+      res[0].should == [["23"],["12"]]
+      res[1].should == [["12","34"],["23"]]
+
+      res = @segmenter.straddling_and_non_straddling(@three_grams, @sentence)
+      res[0].should == [["234"],["123"]]
+      res[1].should == [["345"],["123","234"]]
+
+    end
+
+    it "should average votes" do
+      votes = [[1,0,1,0],[0,1,0,1]]
+      @segmenter.average_votes(votes).should == [0.5, 0.5, 0.5, 0.5]
+    end
+  end
+
+  describe "Segmenting Text" do
+    before(:each) do 
+      @segmenter = Maxixe::Segmenter.new({"2"=>{"AB"=>2, "BC"=>2, "CD"=>1, "DE"=>1, "EF"=>1, "FG"=>1, "G\n"=>1, "CX"=>1, "XY"=>1, "YZ"=>1, "Z\n"=>1}, "3"=>{"ABC"=>2, "BCD"=>1, "CDE"=>1, "DEF"=>1, "EFG"=>1, "FG\n"=>1, "BCX"=>1, "CXY"=>1, "XYZ"=>1, "YZ\n"=>1}})
+    end
+
+    it "should be able to segment text" do
+      @segmenter.segment("ABCDE").should == "ABC DE"
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'maxixe' # and any other gems you need
+

data/spec/trainer/first_file

@@ -0,0 +1 @@
+ABCDEFG

data/spec/trainer/second_file

@@ -0,0 +1 @@
+ABCXYZ

data/spec/trainer/trainer_spec.rb

@@ -0,0 +1,13 @@
+require "spec_helper"
+
+describe Maxixe::Trainer do
+  
+  it "should generate n-gram data from a set of files" do
+
+    pwd = File.dirname(__FILE__)
+
+    Maxixe::Trainer.generate_training_data([2,3], File.join(pwd, "first_file"), File.join(pwd,"second_file")).should == {"2"=>{"AB"=>2, "BC"=>2, "CD"=>1, "DE"=>1, "EF"=>1, "FG"=>1, "G\n"=>1, "CX"=>1, "XY"=>1, "YZ"=>1, "Z\n"=>1}, "3"=>{"ABC"=>2, "BCD"=>1, "CDE"=>1, "DEF"=>1, "EFG"=>1, "FG\n"=>1, "BCX"=>1, "CXY"=>1, "XYZ"=>1, "YZ\n"=>1}}
+
+  end
+
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: maxixe
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Roger Braun
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-08-20 00:00:00.000000000 %:z
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: &72352710 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *72352710
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &72352370 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *72352370
+description: Maxixe is an implementation of the Tango algorithm describe in the paper
+  "Mostly-unsupervised statistical segmentation of Japanese kanji sequences" by Ando
+  and Lee. While the paper deals with Japanese characters, it should work on any unsegmented
+  text given enough corpus data and a tuning of the algorithm paramenters.
+email:
+- maxixe@rogerbraun.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Rakefile
+- lib/.maxixe.rb.swp
+- lib/maxixe.rb
+- lib/maxixe/version.rb
+- maxixe.gemspec
+- spec/segmenter/.segmenter_spec.rb.swp
+- spec/segmenter/segmenter_spec.rb
+- spec/spec_helper.rb
+- spec/trainer/.first_file.swp
+- spec/trainer/.second_file.swp
+- spec/trainer/.trainer_spec.rb.swp
+- spec/trainer/first_file
+- spec/trainer/second_file
+- spec/trainer/trainer_spec.rb
+has_rdoc: true
+homepage: https://github.com/rogerbraun/Maxixe
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: maxixe
+rubygems_version: 1.6.1
+signing_key: 
+specification_version: 3
+summary: A small statistical segmenter for any language.
+test_files: []