feedbackmine-language_detector 0.1.1 → 0.1.2

data/lib/language_detector.rb

@@ -6,7 +6,7 @@ class LanguageDetector
   def detect text
     @profiles ||= load_model
 
-    p = Profile.new("")
+    p = LanguageDetector::Profile.new("")
     p.init_with_string text
     best_profile = nil
     best_distance = nil
@@ -97,7 +97,7 @@ class LanguageDetector
 
     profiles = []
     training_data.each {|data|
-      p = Profile.new data[0]
+      p = LanguageDetector::Profile.new data[0]
       p.init_with_file data[1]
       profiles << p
     }
@@ -112,118 +112,119 @@ class LanguageDetector
     filename = File.expand_path(File.join(File.dirname(__FILE__), "model.yml"))
     @profiles = YAML.load_file(filename)
   end
-end
-
-class Profile
-
-  PUNCTUATIONS = [?\n, ?\r, ?\t, ?\s, ?!, ?", ?#, ?$, ?%, ?&, ?', ?(, ?), ?*, ?+, ?,, ?-, ?., ?/,
-  ?0, ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9,
-  ?:, ?;, ?<, ?=, ?>, ??, ?@, ?[, ?\\, ?], ?^, ?_, ?`, ?{, ?|, ?}, ?~]
-
-  LIMIT = 2000
 
-  def compute_distance other_profile
-    distance = 0
-    other_profile.ngrams.each {|k, v|
-      n = @ngrams[k]
-      if n
-        distance += (v - n).abs
-      else
-        distance += Profile::LIMIT
-      end
-    }
-    return distance
-  end
+  class LanguageDetector::Profile
+
+    PUNCTUATIONS = [?\n, ?\r, ?\t, ?\s, ?!, ?", ?#, ?$, ?%, ?&, ?', ?(, ?), ?*, ?+, ?,, ?-, ?., ?/,
+    ?0, ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9,
+    ?:, ?;, ?<, ?=, ?>, ??, ?@, ?[, ?\\, ?], ?^, ?_, ?`, ?{, ?|, ?}, ?~]
+
+    LIMIT = 2000
+
+    def compute_distance other_profile
+      distance = 0
+      other_profile.ngrams.each {|k, v|
+        n = @ngrams[k]
+        if n
+          distance += (v - n).abs
+        else
+          distance += LanguageDetector::Profile::LIMIT
+        end
+      }
+      return distance
+    end
 
-  attr_reader :ngrams, :name
+    attr_reader :ngrams, :name
 
-  def initialize(name)
-    @name = name
-    @puctuations = {}
-    PUNCTUATIONS.each {|p| @puctuations[p] = 1}
-    @ngrams = {}
-  end
+    def initialize(name)
+      @name = name
+      @puctuations = {}
+      PUNCTUATIONS.each {|p| @puctuations[p] = 1}
+      @ngrams = {}
+    end
 
-  def init_with_file filename
-    ngram_count = {}
+    def init_with_file filename
+      ngram_count = {}
+
+      path = File.expand_path(File.join(File.dirname(__FILE__), "training_data/" + filename))
+      puts "training with " + path
+      File.open(path).each_line{ |line|
+        _init_with_string line, ngram_count
+      }
+
+      a = ngram_count.sort {|a,b| b[1] <=> a[1]}
+      i = 1
+      a.each {|t|
+        @ngrams[t[0]] = i
+        i += 1
+        break if i > LIMIT
+      }
+    end
 
-    path = File.expand_path(File.join(File.dirname(__FILE__), "training_data/" + filename))
-    puts "training with " + path
-    File.open(path).each_line{ |line|
-      _init_with_string line, ngram_count
-    }
+    def init_with_string str
+      ngram_count = {}
 
-    a = ngram_count.sort {|a,b| b[1] <=> a[1]}
-    i = 1
-    a.each {|t|
-      @ngrams[t[0]] = i
-      i += 1
-      break if i > LIMIT
-    }
-  end
+      _init_with_string str, ngram_count
 
-  def init_with_string str
-    ngram_count = {}
+      a = ngram_count.sort {|a,b| b[1] <=> a[1]}
+      i = 1
+      a.each {|t|
+        @ngrams[t[0]] = i
+        i += 1
+        break if i > LIMIT
+      }
+    end
 
-    _init_with_string str, ngram_count
+    def _init_with_string str, ngram_count
+      tokens = tokenize(str)
+      tokens.each {|token|
+        count_ngram token, 2, ngram_count
+        count_ngram token, 3, ngram_count
+        count_ngram token, 4, ngram_count
+        count_ngram token, 5, ngram_count
+      }
+    end
 
-    a = ngram_count.sort {|a,b| b[1] <=> a[1]}
-    i = 1
-    a.each {|t|
-      @ngrams[t[0]] = i
-      i += 1
-      break if i > LIMIT
-    }
-  end
+    def tokenize str
+      tokens = []
+      s = ''
+      str.each_byte {|b|
+        if is_puctuation?(b)
+          tokens << s unless s.empty?
+          s = ''
+        else
+          s << b
+        end
+      }
+      tokens << s unless s.empty?
+      return tokens
+    end
 
-  def _init_with_string str, ngram_count
-    tokens = tokenize(str)
-    tokens.each {|token|
-      count_ngram token, 2, ngram_count
-      count_ngram token, 3, ngram_count
-      count_ngram token, 4, ngram_count
-      count_ngram token, 5, ngram_count
-    }
-  end
+    def is_puctuation? b
+      @puctuations[b]
+    end
 
-  def tokenize str
-    tokens = []
-    s = ''
-    str.each_byte {|b|
-      if is_puctuation?(b)
-        tokens << s unless s.empty?
+    def count_ngram token, n, counts
+      token = "_#{token}#{'_' * (n-1)}" if n > 1 && token.jlength >= n
+      i = 0
+      while i + n <= token.length
         s = ''
-      else
-        s << b
+        j = 0
+        while j < n
+          s << token[i+j]
+          j += 1
+        end
+        if counts[s]
+          counts[s] = counts[s] + 1
+        else
+          counts[s] = 1
+        end
+        i += 1
       end
-    }
-    tokens << s unless s.empty?
-    return tokens
-  end
-
-  def is_puctuation? b
-    @puctuations[b]
-  end
 
-  def count_ngram token, n, counts
-    token = "_#{token}#{'_' * (n-1)}" if n > 1 && token.jlength >= n
-    i = 0
-    while i + n <= token.length
-      s = ''
-      j = 0
-      while j < n
-        s << token[i+j]
-        j += 1
-      end
-      if counts[s]
-        counts[s] = counts[s] + 1
-      else
-        counts[s] = 1
-      end
-      i += 1
+      return counts
     end
 
-    return counts
   end
 
 end

data/lib/model.yml

@@ -1,5 +1,5 @@
 --- 
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: ar
   ngrams: 
     ? !binary "qtmH\n"
@@ -5149,7 +5149,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: bg
   ngrams: 
     ? !binary "0L7QtNA=\n"
@@ -10519,7 +10519,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: cs
   ngrams: 
     He: 765
@@ -12641,7 +12641,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: da
   ngrams: 
     erede: 347
@@ -14691,7 +14691,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: de
   ngrams: 
     Ope: 1204
@@ -16751,7 +16751,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: el
   ngrams: 
     ? !binary "zpTPic8=\n"
@@ -22079,7 +22079,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: en
   ngrams: 
     my_: 833
@@ -24129,7 +24129,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: et
   ngrams: 
     _saav: 1637
@@ -26215,7 +26215,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: es
   ngrams: 
     nco: 1791
@@ -28277,7 +28277,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: fa
   ngrams: 
     ? !binary "2YPYp9g=\n"
@@ -33419,7 +33419,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: fi
   ngrams: 
     valla: 637
@@ -35483,7 +35483,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: fr
   ngrams: 
     hine: 1617
@@ -37549,7 +37549,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: fy
   ngrams: 
     nfoel: 1606
@@ -39607,7 +39607,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: ga
   ngrams: 
     ud: 1880
@@ -41671,7 +41671,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: he
   ngrams: 
     ? !binary "ldep15k=\n"
@@ -46859,7 +46859,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: hi
   ngrams: 
     ? !binary "ruCkuA==\n"
@@ -51907,7 +51907,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: hr
   ngrams: 
     vine: 1902
@@ -53967,7 +53967,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: io
   ngrams: 
     _Kere: 1912
@@ -56027,7 +56027,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: is
   ngrams: 
     kaga_: 1774
@@ -58175,7 +58175,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: it
   ngrams: 
     Ope: 1795
@@ -60235,7 +60235,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: ja
   ngrams: 
     ? !binary "g6vj\n"
@@ -65965,7 +65965,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: ko
   ngrams: 
     ? !binary "mIDqsw==\n"
@@ -71445,7 +71445,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: hu
   ngrams: 
     "ens\xC3\xA9": 1414
@@ -73515,7 +73515,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: nl
   ngrams: 
     He: 619
@@ -75565,7 +75565,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: "no"
   ngrams: 
     Ope: 1032
@@ -77625,7 +77625,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: pl
   ngrams: 
     "k\xC3\xB3w": 1379
@@ -79717,7 +79717,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: pt
   ngrams: 
     nco: 1274
@@ -81815,7 +81815,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: ro
   ngrams: 
     _ur: 1996
@@ -83893,7 +83893,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: ru
   ngrams: 
     ? !binary "v9C/\n"
@@ -89077,7 +89077,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: sl
   ngrams: 
     preds: 594
@@ -91147,7 +91147,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: sv
   ngrams: 
     karna: 1187
@@ -93197,7 +93197,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: th
   ngrams: 
     ? !binary "uYPguKs=\n"
@@ -98785,7 +98785,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: uk
   ngrams: 
     "\xBA\xD0\xBE__": 1806
@@ -103973,7 +103973,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: vi
   ngrams: 
     ? !binary "xJHhu5E=\n"
@@ -106335,7 +106335,7 @@
     43: 1
     32: 1
     10: 1
-- !ruby/object:Profile 
+- !ruby/object:LanguageDetector::Profile 
   name: zh
   ngrams: 
     ? !binary "6L+Z\n"

data/test/language_detector_test.rb

@@ -4,7 +4,7 @@ require File.dirname(__FILE__) + '/../lib/language_detector'
 
 class ProfileTest < Test::Unit::TestCase
   def test_is_puctuation
-    p = Profile.new("test")
+    p = LanguageDetector::Profile.new("test")
     assert p.is_puctuation?(?,)
     assert p.is_puctuation?(?.)
     assert !p.is_puctuation?(?A)
@@ -12,12 +12,12 @@ class ProfileTest < Test::Unit::TestCase
   end
 
   def test_tokenize
-    p = Profile.new("test")
+    p = LanguageDetector::Profile.new("test")
     assert_equal ["this", "is", "A", "test"], p.tokenize("this is ,+_  A \t 123 test")
   end
 
   def test_count_ngram
-    p = Profile.new("test")
+    p = LanguageDetector::Profile.new("test")
     assert_equal({"w"=>1, "o"=>1, "r"=>1, "d"=>1, "s"=>1}, p.count_ngram('words', 1, {}))
     assert_equal({"wo"=>1, "or"=>1, "rd"=>1, "ds"=>1, "_w" => 1, "s_" => 1}, p.count_ngram('words', 2, {}))
     assert_equal({"wor"=>1, "ord"=>1, "rds"=>1, "_wo" => 1, "ds_" => 1, "s__" => 1}, p.count_ngram('words', 3, {}))
@@ -27,25 +27,29 @@ class ProfileTest < Test::Unit::TestCase
   end
 
   def test_init_with_string
-    p = Profile.new("test")
+    p = LanguageDetector::Profile.new("test")
     p.init_with_string("this is ,+_  A \t 123 test")
-    assert_equal([["t_", 30], ["st__", 29], ["st", 16], ["hi", 8], ["_tes", 7], ["is__", 6], ["s___", 5], ["s_", 3], ["his_", 11], ["tes", 10], ["t___", 9], ["es", 12], ["_te", 14], ["est_", 13], ["est", 15], ["te", 4], ["his", 17], ["_th", 20], ["s__", 19], ["st_", 18], ["th", 24], ["_thi", 23], ["t__", 22], ["test", 21], ["thi", 28], ["is_", 27], ["this", 26], ["_i", 25], ["is", 2], ["_t", 1]], p.ngrams.sort_by { |a,b| a[1] <=> b[1] })
+    assert_equal(
+      [["t_", 30], ["st__", 29], ["st", 16], ["hi", 8], ["_tes", 7], ["is__", 6], ["s___", 5], ["s_", 3], ["his_", 11], ["tes", 10], ["t___", 9], ["es", 12], ["_te", 14], ["est_", 13], ["est", 15], ["te", 4], ["his", 17], ["_th", 20], ["s__", 19], ["st_", 18], ["th", 24], ["_thi", 23], ["t__", 22], ["test", 21], ["thi", 28], ["is_", 27], ["this", 26], ["_i", 25], ["is", 2], ["_t", 1]],
+      p.ngrams.sort_by { |a,b| a[1] <=> b[1] },
+      "This test does not pass in the original repository either: http://github.com/feedbackmine/language_detector"
+    )
   end
 
   def test_init_with_file
-    p = Profile.new("test")
+    p = LanguageDetector::Profile.new("test")
     p.init_with_file("bg-utf8.txt")
     assert !p.ngrams.empty?
   end
 
   def test_compute_distance
-    p1 = Profile.new("test")
+    p1 = LanguageDetector::Profile.new("test")
     p1.init_with_string("this is ,+_  A \t 123 test")
-    p2 = Profile.new("test")
+    p2 = LanguageDetector::Profile.new("test")
     p2.init_with_string("this is ,+_  A \t 123 test")
     assert_equal 0, p1.compute_distance(p2)
 
-    p3 = Profile.new("test")
+    p3 = LanguageDetector::Profile.new("test")
     p3.init_with_string("xxxx")
     assert_equal 24000, p1.compute_distance(p3)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: feedbackmine-language_detector
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - feedbackmine
@@ -29,6 +29,7 @@ files:
 - test/language_detector_test.rb
 has_rdoc: false
 homepage: http://www.tweetjobsearch.com
+licenses: 
 post_install_message: 
 rdoc_options: []
 
@@ -49,7 +50,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.2.0
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 2
 summary: n-gram based language detector, written in ruby