chinese_sugar 0.0.1

data/lib/sugar/trie.rb

@@ -0,0 +1,110 @@
+require "sugar/version"
+
+module Sugar
+  class Trie
+    # FIXME: should be class method
+    attr_accessor :trie, :frequnces, :count
+
+    def initialize
+      dict = File.expand_path("../../dict.txt", __FILE__)
+      self.trie, self.frequnces, self.count = Sugar::Trie.build(dict)
+    end
+
+    def word?(word)
+      sfx = suffix(word)
+      !!sfx && sfx.has_key?('')
+    end
+
+    def suffix(word)
+      p = trie
+      word.each_char do |char|
+        return nil if p[char].nil?
+        p = p[char]
+      end
+      p
+    end
+
+    def frequnce(word)
+      frequnces[word]
+    end
+    
+    # construct a DAG of sentence
+    def DAG(sentence)
+      n = sentence.length
+      dag = Array.new(n){[]} # [[]]*n
+      0.upto(n-1) do |i|
+        sfx = suffix(sentence[i])
+        i.upto(n-1) do |j|
+          if sfx == nil
+            break
+          elsif sfx['']
+            dag[i].push(j)
+          end
+          sfx = sfx[sentence[j+1]]
+        end
+      end
+      dag
+    end
+
+    # Viterbi算法，递归过程
+    def viterbi_distance(graph, sentence, i, path)
+      if i < graph.size
+        max, node = -1, -1
+        graph[i].map.with_index do |j, index|
+          path[j+1] = {}
+          distance = viterbi_distance(graph, sentence, j+1, path[j+1])*possibility(sentence[i..j])
+          if max < distance # find the min distance
+            max, node = distance, index
+          end
+          distance
+        end.each_with_index do |distance, index|
+          path.delete(graph[i][index] + 1) if distance != max
+        end
+        max
+      else
+        1
+      end
+    end
+
+    def possibility(word)
+      1.0*frequnce(word)/count
+    end
+
+    # 
+    def best_segmentation(sentence)
+      dag = self.DAG(sentence)
+      path = {1 => {}}
+      min_distance = viterbi_distance(dag, sentence, 0, path)
+      return [min_distance, path]
+    end
+    
+    # load from dict.txt
+    def self.build(dict)
+      trie, frequnces, count = {}, {}, 0
+      File.read(dict).split("\n").each do |line|
+        word, freq, _ = line.rstrip.split(' ')
+        count += (frequnces[word] = freq.to_i)
+        p = trie # reference pointer
+        word.each_char do |char|
+          p[char] = {} if p[char].nil?
+          p = p[char]
+        end
+        p[''] = '' # label the end of word
+      end
+      [trie, frequnces, count]
+    end
+
+    # load from cache
+    def self.load(tempfile)
+      Marshal.load(tempfile.read)
+    end
+
+    def dump(tempfile)
+      tempfile.write(Marshal.dump(self))
+    end
+
+    def insepct
+      "#<trie: #{trie.keys[0..10].join(',')}..., count: #{count}>"
+    end
+  end
+end

data/lib/sugar/version.rb

@@ -0,0 +1,3 @@
+module Sugar
+  VERSION = "0.0.1"
+end

data/lib/sugar.rb

@@ -0,0 +1,5 @@
+require "sugar/version"
+
+module Sugar
+  
+end

data/sugar.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sugar/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "chinese_sugar"
+  spec.version       = Sugar::VERSION
+  spec.authors       = ["binz"]
+  spec.email         = ["xinkiang@gmail.com"]
+  spec.summary       = %q{Chinese text data mining.}
+  spec.description   = %q{1.Chinese words segmentation using Trie and Viterbi}
+  spec.homepage      = "https://github.com/slacken/sugar"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

data/sugar.rb

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
+
+require 'sugar/trie'
+require 'tempfile'
+
+
+t = Sugar::Trie.new
+str = '到底有没有自然灾害自然灾害到了什么程度我不是很清楚但我希望这个回答能终结知乎上对大饥荒死亡人数的疑问'
+puts str
+dag = t.DAG(str)
+
+puts dag.to_s
+
+puts t.best_segmentation(str).to_s

metadata

@@ -0,0 +1,83 @@
+--- !ruby/object:Gem::Specification
+name: chinese_sugar
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- binz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 1.Chinese words segmentation using Trie and Viterbi
+email:
+- xinkiang@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/dict.txt
+- lib/sugar.rb
+- lib/sugar/trie.rb
+- lib/sugar/version.rb
+- sugar.gemspec
+- sugar.rb
+homepage: https://github.com/slacken/sugar
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Chinese text data mining.
+test_files: []