hanzi 0.4.0 → 0.4.1

data/Gemfile

@@ -1,6 +1,9 @@
 source "http://rubygems.org"
 
+gem 'fast_trie'
+
 group :development do
   gem "rdoc", "~> 3.12"
   gem "jeweler", "~> 1.8.4"
+  gem 'ruby-prof'
 end

data/README.rdoc

@@ -41,6 +41,10 @@ To run tests:
 
   rake test
 
+There is a script for profiling:
+
+  ruby test/profile_speed.rb # generates output in /tmp
+
 == Contributing to hanzi
 
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.1

data/hanzi.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "hanzi"
-  s.version = "0.4.0"
+  s.version = "0.4.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Steve Jackson"]
-  s.date = "2013-01-23"
+  s.date = "2013-02-06"
   s.description = "Convert Hanzi to pinyin. Unlike other similar gems, this includes tones and can accurately translate common words."
   s.email = "steven.j.jackson@gmail.com"
   s.extra_rdoc_files = [
@@ -32,22 +32,28 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/stevejackson/hanzi"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.24"
+  s.rubygems_version = "1.8.23"
   s.summary = "Convert Hanzi to pinyin. Unlike other similar gems, this includes tones and can accurately translate common words."
 
   if s.respond_to? :specification_version then
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<fast_trie>, [">= 0"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
+      s.add_development_dependency(%q<ruby-prof>, [">= 0"])
     else
+      s.add_dependency(%q<fast_trie>, [">= 0"])
       s.add_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+      s.add_dependency(%q<ruby-prof>, [">= 0"])
     end
   else
+    s.add_dependency(%q<fast_trie>, [">= 0"])
     s.add_dependency(%q<rdoc>, ["~> 3.12"])
     s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
+    s.add_dependency(%q<ruby-prof>, [">= 0"])
   end
 end
 

data/lib/hanzi.rb

@@ -1,14 +1,18 @@
 # encoding: utf-8
+require 'trie'
 
 class Hanzi
   class << self
     attr_accessor :data
+    attr_accessor :data_trie
 
     def load_data
       return if @data
       @data = []
+      @data_trie = Trie.new
 
       file_path = File.expand_path('../../lib/data/cedict_ts.u8', __FILE__)
+      index = 0
       File.open(file_path).each_line do |line|
         next if line.start_with?('#')
         line = line.force_encoding('utf-8')
@@ -27,9 +31,26 @@ class Hanzi
         line = line[line.index('/'), line.rindex('/')]
         line_data[:english] = line[1, line.rindex('/') - 1]
 
+        existing_count_simplified = 0
+        if find_first_hanzi_match(line_data[:simplified])
+          existing_count_simplified = matching_entries(line_data[:simplified]).count
+        end
+        @data_trie.add(line_data[:simplified] + existing_count_simplified.to_s, index)
+
+        if line_data[:simplified] != line_data[:traditional]
+          existing_count_traditional = 0
+          if find_first_hanzi_match(line_data[:traditional])
+            existing_count_traditional = matching_entries(line_data[:traditional]).count
+          end
+
+          @data_trie.add(line_data[:traditional] + existing_count_traditional.to_s, index)
+        end
+
         @data << line_data
-      end
 
+
+        index += 1
+      end
     end
 
     def to_pinyin(text, options={})
@@ -52,7 +73,7 @@ class Hanzi
           match = nil
           match_length = 0
           4.downto(1) do |length|
-            match = find_hanzi_match(text[pos, length])
+            match = find_first_hanzi_match(text[pos, length])
             match_length = length
             break if match
           end
@@ -73,37 +94,44 @@ class Hanzi
     def to_english(text)
       load_data if @data.nil?
 
-      entry = find_hanzi_match(text)
+      entry = find_first_hanzi_match(text)
       entry[:english] if entry && entry[:english]
     end
 
     def to_simplified(text)
       load_data if @data.nil?
 
-      entry = find_hanzi_match(text)
+      entry = find_first_hanzi_match(text)
       entry[:simplified] if entry && entry[:simplified]
     end
 
     def to_traditional(text)
       load_data if @data.nil?
 
-      entry = find_hanzi_match(text)
+      entry = find_first_hanzi_match(text)
       entry[:traditional] if entry && entry[:traditional]
     end
 
     def matching_entries(text)
       load_data if @data.nil?
 
-      entries = @data.select do |word|
-        word[:simplified] == text || word[:traditional] == text
+      results = []
+      index = 0
+      loop do
+        id = @data_trie.get(text + index.to_s)
+        break if !id
+
+        results << @data[id]
+        index += 1
       end
+
+      results
     end
 
     private
-    def find_hanzi_match(text)
-      entry = @data.find do |word|
-        word[:simplified] == text || word[:traditional] == text
-      end
+    def find_first_hanzi_match(text)
+      id = @data_trie.get(text + "0")
+      @data[id] if id
     end
 
   end

data/test/test_hanzi.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require 'helper'
+require 'benchmark'
 
 class TestHanzi < Test::Unit::TestCase
 
@@ -71,4 +72,31 @@ class TestHanzi < Test::Unit::TestCase
     assert_equal nil, result
   end
 
+  def test_speed
+    entries = words = ["果断", "过度", "过渡", "过奖", "过滤", "过失", "过问", "过瘾", "过于", "嗨", "海拔",
+      "海滨", "含糊", "寒暄", "含义", "罕见", "捍卫", "航空", "行列", "航天", "航行", "豪迈", "毫米",
+      "毫无", "耗费", "好客", "号召", "呵", "和蔼", "合并", "合成", "合乎", "合伙", "和解", "和睦", "和气",
+      "合身", "合算", "和谐", "嘿", "痕迹", "狠心", "恨不得", "哼", "哄", "烘", "轰动", "红包", "宏观",
+      "洪水", "宏伟", "喉咙", "吼", "后代", "后顾之忧", "后勤", "候选", "忽略", "呼啸", "呼吁", "胡乱",
+      "湖泊", "互联网", "华丽", "华侨", "化肥", "划分", "画蛇添足", "化石", "话筒", "化验", "化妆", "怀孕",
+      "欢乐", "环节", "还原", "缓和", "患者", "荒凉", "慌忙", "荒谬", "荒唐", "黄昏", "恍然大悟", "辉煌",
+      "挥霍", "回报", "回避", "回顾", "回收", "悔恨", "毁灭", "汇报", "贿赂", "会晤", "昏迷", "浑身", "混合",
+      "混乱", "混淆", "混浊", "活该", "活力", "火箭", "火焰", "火药", "货币", "或许", "基地", "机动", "饥饿",
+      "激发", "机构", "机关", "基金", "激励", "机灵", "机密", "激情", "讥笑", "机械", "基因", "机遇", "机智",
+      "即便", "级别", "疾病", "嫉妒", "极端", "急功近利", "籍贯", "即将", "急剧", "急切", "集团", "极限",
+      "吉祥", "急于求成", "及早", "急躁", "给予", "继承", "季度", "忌讳", "计较", "寂静", "季军", "技能",
+      "技巧", "寄托", "继往开来", "迹象", "记性", "纪要", "记载", "家常", "加工", "家伙", "加剧", "家属",
+      "空虚", "孔", "恐吓", "恐惧", "空白", "空隙", "口气", "口腔", "口头", "口音", "枯竭", "枯燥", "苦尽甘来"]
+
+    Hanzi.load_data
+
+    time = Benchmark.realtime {
+      entries.each do |word|
+        Hanzi.matching_entries(word)
+      end
+    }
+
+    assert time < 0.01, "Lookups took #{time}s, should be less than 0.01s"
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hanzi
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-01-23 00:00:00.000000000 Z
+date: 2013-02-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fast_trie
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc
   requirement: !ruby/object:Gem::Requirement
@@ -43,6 +59,22 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.8.4
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Convert Hanzi to pinyin. Unlike other similar gems, this includes tones
   and can accurately translate common words.
 email: steven.j.jackson@gmail.com
@@ -78,7 +110,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3371334032433433338
+      hash: 2045449454900323420
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -87,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Convert Hanzi to pinyin. Unlike other similar gems, this includes tones and