rlid 0.1.0

data/lib/interactive_guesser.rb

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+$: << File.dirname( __FILE__)
+
+require 'rlid'
+
+
+loop do
+  print "> "
+  puts Rlid.guess_language(gets).to_s
+end
+

data/lib/rlid.rb

@@ -0,0 +1,8 @@
+require 'rlid/language_guesser/naive_bayes_guesser'
+
+module Rlid
+  @guesser = NaiveBayesProbabilityGuesser.new
+  def self.guess_language(string)
+    @guesser.guess_language(string)
+  end
+end

data/lib/rlid/common.rb

@@ -0,0 +1,144 @@
+module Rlid
+
+DATA_DIRECTORY =  File.expand_path("#{__FILE__}/../../../data")
+
+class Language
+  CODES = [
+    [:en, :eng, 'English'],
+    [:es, :spa, 'Spanish'],
+    [:de, :ger, 'German'],
+    [:fr, :fre, 'French'],
+    [:it, :ita, 'Italian'],
+    [:pt, :por, 'Portoguese'],
+    [:nl, :dut, 'Dutch'],
+    [:pl, :pol, 'Polish'],
+    [:id, :ind, 'Malay/Indonesian'],
+    [:sv, :swe, 'Swedish'],
+    [:tr, :tur, 'Turkish'],
+    [:vi, :vie, 'Vietnamese'],
+    [:ro, :rum, 'Romanian'],
+    [:cs, :cze, 'Czech'],
+    [:da, :dan, 'Danish'],
+    [:fi, :fin, 'Finnish'],
+    [:hu, :hun, 'Hungarian'],
+    [:el, :ell, 'Greek'],
+    [:ca, :cat, 'Catalan'],
+    [:no, :nor, 'Norvegian'],
+    [:sk, :slo, 'Slovak'],
+    [:is, :ice, 'Icelandic'],
+    #[:nn, :nnn, 'No Language']
+    #[:ff, :fff], # fdakjlfdaj;
+  ]
+
+  # indexes
+  CODE2 = 0
+  CODE3 = 1
+  NAME = 2
+  NO_LANGUAGE_CODE = :nnn
+
+  def Language.all_codes2
+    CODES.map{|c| c[CODE2]}
+  end
+
+  def Language.all_codes3
+    CODES.map{|c| c[CODE3]}
+  end
+
+
+  # enters each directory and passes the directory name to the block
+#  def Language.each_dir
+#    all_codes2.each do |lang_code|
+#      Dir.chdir("#{DATA_DIRECTORY}/#{lang_code}") do |dir|
+#        yield dir
+#      end
+#    end
+#  end
+
+  def Language.code2to3 code2
+    begin
+      CODES.select{|x| x[CODE2].to_s == code2.to_s}[0][CODE3]
+    rescue
+      nil
+    end
+  end
+
+  def Language.name_of(code)
+    if code == :nn or code == :nnn
+      return "No Language"
+    end
+    index = all_codes3.index(code) or all_codes2.index(code)
+    CODES[index][NAME]
+  end
+
+  def Language.each_file(filename, mode="r")
+    all_codes3.each do |lang_code|
+      filepath = "#{DATA_DIRECTORY}/#{lang_code}/#{filename}"
+      File.open(filepath, mode) do |file|
+        yield file, lang_code
+      end
+    end
+  end
+
+  def Language.each_2files(filename1, filename2, mode1="r", mode2="r")
+    all_codes3.each do |lang_code|
+      filepath1 = "#{DATA_DIRECTORY}/#{lang_code}/#{filename1}"
+      filepath2 = "#{DATA_DIRECTORY}/#{lang_code}/#{filename2}"
+      File.open(filepath1, mode1) do |file1|
+      File.open(filepath2, mode2) do |file2|
+        yield file1, file2, lang_code
+      end
+      end
+    end
+  end
+end
+
+LANGUAGES = Language.all_codes3
+COMMON_LANGUAGES = [:dut, :eng, :ita, :por, :fre, :ger]
+
+
+# for ngrams
+
+end # module Rlid
+
+
+# add methods to String
+class String
+  def each_ngram(n=3)
+    string = preprocess(n)
+    string.chars.each_cons(n) do |chars|
+      yield chars.join
+    end
+  end
+
+#private
+  def preprocess(n)
+    string = self.dup
+
+    # remove spaces at the start and end
+    string.gsub!(/\A\s+/, '')
+    string.gsub!(/\s+\Z/, '')
+
+    # remove non alphabetic characters
+    string.gsub!(/[^[:alpha:]'\n]/, ' ')
+    # substitute newlines with ||
+    string.gsub!(/\s*\n\s*/, '|'*(n-1))
+    string.gsub!(/\s+/, ' ')
+    # remove spaces at the start and end
+    string.gsub!(/\A\s+/, '')
+    string.gsub!(/\s+\Z/, '')
+    
+    string.downcase!
+
+    padding = "|" * (n-1)
+
+    if string.size == 1
+      string = "|" + string + " "
+    elsif string.size == 1
+      string = padding + string + " "
+    else
+      string = padding + string + padding
+    end
+    string
+  end
+end
+

data/lib/rlid/language_guesser/language_guesser.rb

@@ -0,0 +1,15 @@
+module Rlid
+
+class LanguageGuesser
+  attr_accessor :name
+  def initialize
+    @name = "unknown"
+  end
+
+  def guess_language(string)
+    raise "#{self.class} should be subclassed"
+    string # never called, supresses unused variable warning
+  end
+end
+
+end # module Rlid

data/lib/rlid/language_guesser/model_distance_guesser.rb

@@ -0,0 +1,43 @@
+module Rlid
+
+require 'rlid/language_guesser/language_guesser'
+
+class ModelDistanceGuesser < LanguageGuesser
+  def initialize(model_class)
+    @model_class = model_class
+    print "Loading models.. "
+    @language_models = model_class.language_models
+    @name = "Model Distance"
+    puts "Done!"
+  end
+
+  def guess_language(string)
+    model = @model_class.new(string)
+    min_language = min_distance = nil
+    @language_models.each do |lang, lang_model|
+      dist = lang_model - model
+      if min_distance == nil or dist < min_distance
+        min_distance = dist
+        min_language = lang
+      end
+    end
+    min_language
+  end
+end
+
+
+#class OnlineGuesser < ModelDistanceGuesser
+#  def guess_language(string)
+#    min_language = min_distance = nil
+#    @language_models.each do |lang, lang_model|
+#      dist = lang_model.distance_from(string)
+#      if min_distance == nil or dist < min_distance
+#        min_distance = dist
+#        min_language = lang
+#      end
+#    end
+#    min_language
+#  end
+#end
+
+end # module Rlid

data/lib/rlid/language_guesser/naive_bayes_guesser.rb

@@ -0,0 +1,74 @@
+module Rlid
+
+require 'rlid/language_guesser/language_guesser'
+require 'rlid/models/naive_bayes_models'
+require 'rlid/probabilities/language_probabilities'
+
+class NaiveBayesGuesser < LanguageGuesser
+  def initialize(default=1)
+    print "Naive Bayes: loading models.."
+    @models = NaiveBayesModels.load
+    @models.default_count = default
+    @name = "Naive Bayes"
+    puts " Done!"
+  end
+  
+  def guess_language(string)
+    max_prob = 0.0
+    best_language = nil
+    @models.probabilities(string) do |lang, prob|
+      if prob > max_prob
+        max_prob = prob
+        best_language = lang
+      end
+    end
+    best_language
+  end
+end
+
+class NaiveBayesProbabilityGuesser < NaiveBayesGuesser
+  MAX = 3
+  def guess_language(string)
+    results = {}
+    tot = 0.0 # for normalization
+    @models.probabilities(string) do |lang, p|
+      size = string.preprocess(3).size
+      long = Math.log(1 + size)
+      # higher means lower
+      short = 1
+      exp = short/long
+      prob = p**exp
+      results[lang] = prob
+      tot += prob
+    end
+    # normalize
+    results.each_key do |k|
+      results[k] /= tot if tot != 0
+    end
+
+    LanguageProbabilities.new(results)
+  end
+end
+
+
+class NaiveBayesPriorGuesser < NaiveBayesProbabilityGuesser
+  def initialize(prior=TestProbabilities.new(:eng))
+    if not prior.is_a?(LanguageProbabilities)
+      raise InvalidArgument
+    end
+    @prior = prior
+    super()
+  end
+
+  def set_prior(prior)
+    @prior = prior
+  end
+
+  alias :super_guess_language :guess_language
+  def guess_language(string)
+    conditional = super_guess_language(string)
+    (conditional * @prior).first
+  end
+end
+
+end # module Rlid

data/lib/rlid/models/cosine_distance_model.rb

@@ -0,0 +1,69 @@
+module Rlid
+
+
+require 'rlid/models/model'
+require 'rlid/common'
+
+
+class FrequencyModel < NGramModel
+  N = 3 # trigrams
+  def initialize(string, cutoff=3000)
+    super(string, N, cutoff)
+  end
+
+  def save(file)
+    file.write Marshal.dump(@ngram_frequency)
+  end
+
+  def load(file)
+    @ngram_frequency = Marshal.load(file.read)
+  end
+
+  def generate_model(ngram_count)
+    # top ngrams (transformed into arrays)
+    arrays = ngram_count.to_a.sort{|x, y| y[1] <=> x[1]}
+    top = arrays[0...@cutoff] # will be kept
+
+    tot = 0.0 # total, for normalization
+    @ngram_frequency = Hash.new # key is ngram value is position
+    top.each_with_index do |ngram_and_count, i|
+      ngram, count = ngram_and_count
+      @ngram_frequency[ngram] = count
+      tot += count
+    end
+    
+    # normalization
+    @ngram_frequency.each do |ngram, count|
+      @ngram_frequency[ngram] /= tot
+    end
+  end
+
+  def self.filename
+    # FIXME should be frequency3000
+    return "cosine_distance3000"
+  end
+
+protected
+  attr_reader :ngram_frequency
+end
+
+class CosineDistanceModel < FrequencyModel
+  def -(other)
+    if not other.is_a?(CosineDistanceModel)
+      raise InvalidArgument
+    end
+    prod = 0
+    other.ngram_frequency.each do |ngram, freq_other|
+      freq_self = ngram_frequency[ngram]
+      if freq_self != nil
+        prod += (freq_self * freq_other)**0.2
+      end
+    end
+    1 - prod
+  end
+end
+
+
+
+
+end # module Rlid

data/lib/rlid/models/generate_models.rb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby1.9.1
+module Rlid
+
+require 'rlid/common'
+require 'rlid/models/ordered_ngrams'
+require 'rlid/models/cosine_distance_model'
+
+# models to train
+MODELS = [CosineDistanceModel]
+
+MODELS.each do |model|
+puts "training #{model}"
+Language.each_2files('corpus', model.filename, 'r', 'w') do |corpus, file, l|
+  puts "learning #{l}.."
+  model.new(corpus.read).save file
+end
+end
+
+puts ">>  Successfully trained!!  <<"
+
+
+end # module Rlid

data/lib/rlid/models/generate_naive_bayes_models.rb

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby1.9.1
+
+module Rlid
+
+require 'rlid/models/naive_bayes_models'
+
+NaiveBayesModels.generate_models
+
+end # module Rlid

data/lib/rlid/models/model.rb

@@ -0,0 +1,55 @@
+module Rlid
+
+
+# abstract class
+class Model
+  def initialize string
+    raise "#{self.class} should be subclassed"
+    string # never called, supresses unused variable warning
+  end
+end
+
+
+
+# in subclasses generate_model filename, load and save should be implemented
+class NGramModel < Model
+  def initialize(string=nil, n=3, cutoff=300)
+    @n = n
+    @cutoff = cutoff
+
+    if string == nil then return end
+
+    # ngrams and count of each
+    ngram_count = Hash.new(0)
+
+    string.each_ngram(@n) do |ngram|
+      ngram_count[ngram] += 1
+    end
+
+    generate_model(ngram_count)
+  end
+
+
+  def self.language_models
+    if not defined?(filename)
+      raise "#{self.class} should implement 'filename' accessor!"
+    end
+    res = Hash.new
+    Language.each_file(filename) do |file, lang|
+      model = self.new(nil)
+      model.load(file)
+      res[lang] = model
+    end
+    res
+  end
+
+protected
+  # should be implemented in the subclass
+  # ngram_count is a hash: ngram => count
+  def generate_model(ngram_count)
+    raise "#{self.class} should be subclassed"
+    ngram_count # never called, supresses unused variable warning
+  end
+end
+
+end # module Rlid