rlid 0.1.0 → 0.1.1

data/lib/rlid.rb

@@ -1,7 +1,7 @@
 require 'rlid/language_guesser/naive_bayes_guesser'
 
 module Rlid
-  @guesser = NaiveBayesProbabilityGuesser.new
+  @guesser = SmartBayesGuesser.new
   def self.guess_language(string)
     @guesser.guess_language(string)
   end

data/lib/rlid/common.rb

@@ -96,11 +96,20 @@ LANGUAGES = Language.all_codes3
 COMMON_LANGUAGES = [:dut, :eng, :ita, :por, :fre, :ger]
 
 
-# for ngrams
+def self.scrollbar(perc, size=80)
+  realsize = size-2
+  pos = (perc.to_f * realsize).round
+  bar = "=" * pos + " " * (realsize-pos)
+  bar[pos-1] = ">" if pos > 0 and pos < realsize
+  print "|", bar, "|", "\r"
+end
+
 
 end # module Rlid
 
 
+
+
 # add methods to String
 class String
   def each_ngram(n=3)
@@ -131,9 +140,9 @@ class String
 
     padding = "|" * (n-1)
 
-    if string.size == 1
-      string = "|" + string + " "
-    elsif string.size == 1
+    #if string.size == 1
+    #  string = "|" + string + " "
+    if string.size < n-1
       string = padding + string + " "
     else
       string = padding + string + padding

data/lib/rlid/language_guesser/naive_bayes_guesser.rb

@@ -27,7 +27,6 @@ class NaiveBayesGuesser < LanguageGuesser
 end
 
 class NaiveBayesProbabilityGuesser < NaiveBayesGuesser
-  MAX = 3
   def guess_language(string)
     results = {}
     tot = 0.0 # for normalization
@@ -50,6 +49,23 @@ class NaiveBayesProbabilityGuesser < NaiveBayesGuesser
   end
 end
 
+class SmartBayesGuesser < LanguageGuesser
+  def initialize(default=1)
+    #print "Smart Bayes: loading models.."
+    @models = SmartBayesModels.new
+    #puts " Done!"
+  end
+  def guess_language(string)
+    results = @models.probabilities(string)
+    results.delete :nnn
+    tot = results.values.inject(0.0){|s,x| s+x}
+    results.each_key do |lang|
+      results[lang] /= tot
+    end
+    LanguageProbabilities.new(results)
+  end
+end
+
 
 class NaiveBayesPriorGuesser < NaiveBayesProbabilityGuesser
   def initialize(prior=TestProbabilities.new(:eng))

data/lib/rlid/models/generate_naive_bayes_models.rb

@@ -2,8 +2,10 @@
 
 module Rlid
 
+$LOAD_PATH <<  File.expand_path("#{__FILE__}/../../../")
 require 'rlid/models/naive_bayes_models'
 
 NaiveBayesModels.generate_models
+NaiveBayesModels.generate_models("bigrams", 2)
 
 end # module Rlid

data/lib/rlid/models/naive_bayes_models.rb

@@ -6,35 +6,45 @@ require 'set'
 
 require 'rlid/common'
 
+
+# > prova di una stringa molto lunga lunghissima davvero lunga yyyy
+# default = 10
+# ita(99.97) : cat(0.026) : spa(0.0023)
+# default = 1
+# ita(99.995) : cat(0.0045) : por(0.00019)
+# default = 0.1
+# ita(99.9990) : cat(0.00086) : rum(3.7e-05)
+
 class NaiveBayesModels
-  attr_accessor :default_count
-  # ngram leght
-  N = 3
+  attr_accessor :default_count, :n
   # top ngrams kept for every language
   CUTOFF = 3000
   # special feature
   OTHER = nil
 
-  MAX_STRING_LENGTH = 75
+  MAX_STRING_LENGTH = 20
 
-  FILEPATH = "#{DATA_DIRECTORY}/naive_bayes_models"
+  FILENAME = "naive_bayes_models"
 
-  def initialize(default_count=1)
-    @default_count=default_count
+  def initialize(n=3)
+    @n=n
+    @default_count=1
   end
 
-  def self.generate_models
-    models = NaiveBayesModels.new(nil)
+  def self.generate_models(file=nil, n=3)
+    file ||= FILENAME
+    models = NaiveBayesModels.new(n)
     puts "Training started.."
     models.train
-    File.open(FILEPATH, "w") do |file|
-      file.write Marshal.dump(models)
-      puts "Models saved to #{FILEPATH}"
+    File.open( "#{DATA_DIRECTORY}/#{file}", "w") do |f|
+      f.write Marshal.dump(models)
+      puts "Models saved to #{DATA_DIRECTORY}/#{file}"
     end
   end
 
-  def self.load
-    Marshal.load(File.read(FILEPATH))
+  def self.load(file=nil)
+    file ||= FILENAME
+    Marshal.load(File.read("#{DATA_DIRECTORY}/#{file}"))
   end
 
   def probabilities(string)
@@ -43,13 +53,26 @@ class NaiveBayesModels
     end
     @ngram_frequency.keys.each do |lang|
       prob = 1
-      string[0..MAX_STRING_LENGTH].each_ngram do |ngram|
+      string[0..MAX_STRING_LENGTH].each_ngram(@n) do |ngram|
+        if lang == :eng
+          #print ngram, ",  "
+        end
         prob *= frequency_of(lang, ngram)
       end
       yield lang, prob
     end
   end
 
+  # returns a hash
+  def probabilities_h(string)
+    #puts "#{@n}: #{total_ngrams(:ita)}"
+    res = {}
+    probabilities(string) do |lang, prob|
+      res[lang] = prob
+    end
+    res
+  end
+
   def train
     ngram_counts = get_ngram_counts
     # ngrams for which we want to store information (all languages)
@@ -81,13 +104,14 @@ class NaiveBayesModels
       puts_info(lang)
     end
 
+    # add language :nnn
     n = @ngram_frequency.values.map{|x| x[OTHER]}.max * 3 / 2 # (* 1.5)
     @total_ngrams_found[:nnn] = n
     @ngram_frequency[Language::NO_LANGUAGE_CODE] = {OTHER => n}
     @total_ngrams_not_found[:nnn] = @stored_ngrams.size
     
     #puts "total frequencies saved: #{freqs}"
-    #puts "defauld values used: #{default_count} (#{100*default_count/freqs}%)"
+    #puts "default values used: #{default_count} (#{100*default_count/freqs}%)"
     #@ngram_frequency
   end
 
@@ -101,7 +125,6 @@ protected
       #warn "  :#{ngram}: is in OTHER!" if lang == :eng
       ngram = OTHER
     end
-    count = 0
     if @ngram_frequency[lang].include?(ngram)
       count = @ngram_frequency[lang][ngram]
     else
@@ -130,7 +153,7 @@ private
     Language.each_file("corpus") do |file, lang|
       puts "- I'm learning #{lang}"
       ngram_counts[lang] = Hash.new(0) # default is 1
-      file.read.each_ngram(N) do |ngram|
+      file.read.each_ngram(@n) do |ngram|
         ngram_counts[lang][ngram] += 1
       end
 
@@ -154,4 +177,28 @@ private
 end
 
 
+class SmartBayesModels
+  def initialize
+    @trigrams = NaiveBayesModels.load
+    @bigrams = NaiveBayesModels.load("bigrams")
+    @trigrams.default_count = 0.1
+    @bigrams.default_count = 0.1
+  end
+
+  def probabilities string
+    probtri = @trigrams.probabilities_h(string)
+    probbi  = @bigrams.probabilities_h(string)
+    res = {}
+    probtri.each_key do |lang|
+      if probbi[lang] != 0
+        res[lang] = probtri[lang]/probbi[lang]
+      else
+        res[lang] = probtri[lang]
+      end
+      #puts "#{lang} = #{probtri[lang]}/#{probbi[lang]} (#{res[lang]})"
+    end
+    res
+  end
+end
+
 end # module Rlid

data/lib/rlid/probabilities/language_probabilities.rb

@@ -11,7 +11,9 @@ class Percentage
 
   def to_s
     if @value <= 0.98
-      return "%.2g" % (@value * 100)
+      "%.2g" % (@value * 100)
+    elsif @value == 1.0
+      "100"
     else
       complement = 1.0 - @value
       # complement =
@@ -23,7 +25,7 @@ class Percentage
         digits += 1
         res = "%.#{digits}f" % (@value * 100)
       end
-      return res
+      res
     end
   end
 
@@ -68,6 +70,10 @@ class LanguageProbabilities
     end.join(" : ")
   end
 
+  def [](lang)
+    @percentage[lang]
+  end
+
   def first
     sorted.first[LANG]
   end
@@ -126,7 +132,11 @@ protected
   end
 
   def sorted
-    @percentage.to_a.sort!{|x,y| y[PERC] <=> x[PERC]}
+    begin
+      @percentage.to_a.sort!{|x,y| y[PERC] <=> x[PERC]}
+    rescue
+      p @percentage
+    end
   end
 
   attr_accessor :percentage

data/lib/rlid/tmp.rb

@@ -51,7 +51,7 @@ module Rlid
   end
 end
 
-puts require 'irb'
+puts RUBY_VERSION
 
 
 #irb self

metadata

@@ -1,71 +1,61 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: rlid
-version: !ruby/object:Gem::Version 
+version: !ruby/object:Gem::Version
+  version: 0.1.1
   prerelease: 
-  version: 0.1.0
 platform: ruby
-authors: 
+authors:
 - Fela Winkelmolen
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-02-01 00:00:00 +01:00
-default_executable: 
+date: 2013-07-04 00:00:00.000000000 Z
 dependencies: []
-
-description: Language identification library specialized in guessing the language of short strings.
+description: Language identification library specialized in guessing the language
+  of short strings.
 email: fela.kde@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- lib/interactive_guesser.rb
+files:
 - lib/rlid.rb
-- lib/rlid/common.rb
+- lib/interactive_guesser.rb
 - lib/rlid/web.rb
+- lib/rlid/common.rb
 - lib/rlid/tmp.rb
 - lib/rlid/language_guesser/model_distance_guesser.rb
-- lib/rlid/language_guesser/naive_bayes_guesser.rb
 - lib/rlid/language_guesser/language_guesser.rb
-- lib/rlid/models/generate_naive_bayes_models.rb
-- lib/rlid/models/generate_models.rb
-- lib/rlid/models/cosine_distance_model.rb
+- lib/rlid/language_guesser/naive_bayes_guesser.rb
 - lib/rlid/models/model.rb
-- lib/rlid/models/naive_bayes_models.rb
+- lib/rlid/models/cosine_distance_model.rb
 - lib/rlid/models/ordered_ngrams.rb
+- lib/rlid/models/naive_bayes_models.rb
+- lib/rlid/models/generate_models.rb
+- lib/rlid/models/generate_naive_bayes_models.rb
 - lib/rlid/probabilities/language_probabilities.rb
 - data/naive_bayes_models
-has_rdoc: true
 homepage: https://github.com/fela/rlid
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
       version: 1.9.1
-required_rubygems_version: !ruby/object:Gem::Requirement 
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.5.0
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Language identification library
 test_files: []
-