rlid 0.1.0

data/lib/rlid/models/naive_bayes_models.rb

@@ -0,0 +1,157 @@
+#!/usr/bin/env ruby1.9.1
+
+module Rlid
+
+require 'set'
+
+require 'rlid/common'
+
+class NaiveBayesModels
+  attr_accessor :default_count
+  # ngram leght
+  N = 3
+  # top ngrams kept for every language
+  CUTOFF = 3000
+  # special feature
+  OTHER = nil
+
+  MAX_STRING_LENGTH = 75
+
+  FILEPATH = "#{DATA_DIRECTORY}/naive_bayes_models"
+
+  def initialize(default_count=1)
+    @default_count=default_count
+  end
+
+  def self.generate_models
+    models = NaiveBayesModels.new(nil)
+    puts "Training started.."
+    models.train
+    File.open(FILEPATH, "w") do |file|
+      file.write Marshal.dump(models)
+      puts "Models saved to #{FILEPATH}"
+    end
+  end
+
+  def self.load
+    Marshal.load(File.read(FILEPATH))
+  end
+
+  def probabilities(string)
+    if not string.is_a? String
+      raise InvalidArgument
+    end
+    @ngram_frequency.keys.each do |lang|
+      prob = 1
+      string[0..MAX_STRING_LENGTH].each_ngram do |ngram|
+        prob *= frequency_of(lang, ngram)
+      end
+      yield lang, prob
+    end
+  end
+
+  def train
+    ngram_counts = get_ngram_counts
+    # ngrams for which we want to store information (all languages)
+    @stored_ngrams = top_ngrams(ngram_counts)
+
+    puts "- processing ngrams"
+    # content: ngram_frequency[lang][ngram] = freq
+    @ngram_frequency = Hash.new
+    # content: total_ngrams_found[lang] = total count of ngrams encountered
+    @total_ngrams_found= Hash.new
+    # content: total_ngrams_not_found[lang] = n of ngrams not found
+    @total_ngrams_not_found = Hash.new
+
+    ngram_counts.each do |lang, counts|
+      @ngram_frequency[lang] = Hash.new(0)
+      @total_ngrams_found[lang] = 0
+      counts.each do |ngram, count|
+        if @stored_ngrams.include?(ngram)
+          @ngram_frequency[lang][ngram] = count
+        else
+          @ngram_frequency[lang][OTHER] += count
+        end
+        @total_ngrams_found[lang] += count
+      end
+      
+      not_found = (@stored_ngrams - @ngram_frequency[lang].keys).size
+      @total_ngrams_not_found[lang] = not_found
+
+      puts_info(lang)
+    end
+
+    n = @ngram_frequency.values.map{|x| x[OTHER]}.max * 3 / 2 # (* 1.5)
+    @total_ngrams_found[:nnn] = n
+    @ngram_frequency[Language::NO_LANGUAGE_CODE] = {OTHER => n}
+    @total_ngrams_not_found[:nnn] = @stored_ngrams.size
+    
+    #puts "total frequencies saved: #{freqs}"
+    #puts "defauld values used: #{default_count} (#{100*default_count/freqs}%)"
+    #@ngram_frequency
+  end
+
+protected
+  def total_ngrams(lang)
+    @total_ngrams_found[lang] + @total_ngrams_not_found[lang] * @default_count
+  end
+
+  def frequency_of(lang, ngram)
+    if not @stored_ngrams.include?(ngram)
+      #warn "  :#{ngram}: is in OTHER!" if lang == :eng
+      ngram = OTHER
+    end
+    count = 0
+    if @ngram_frequency[lang].include?(ngram)
+      count = @ngram_frequency[lang][ngram]
+    else
+      count = @default_count
+    end
+    count.to_f / total_ngrams(lang)
+  end
+
+private
+
+  def puts_info(lang)
+    # default count of 1 is supposed
+    tot = @total_ngrams_found[lang] + @total_ngrams_not_found[lang]
+    d = (100.0 * @total_ngrams_not_found[lang] / tot).round(1)
+    o = (100.0 * @ngram_frequency[lang][OTHER] / tot).round(1)
+    puts "  #{lang} processed tot:#{tot}, default:#{d}%, other:#{o}%"
+  end
+
+  # auxiliary functions
+
+  # gets all ngram_counts and returns an hash having:
+  # ngram_counts[lang][ngram] = count
+  def get_ngram_counts
+    @stored_ngrams = Set.new
+    ngram_counts = Hash.new
+    Language.each_file("corpus") do |file, lang|
+      puts "- I'm learning #{lang}"
+      ngram_counts[lang] = Hash.new(0) # default is 1
+      file.read.each_ngram(N) do |ngram|
+        ngram_counts[lang][ngram] += 1
+      end
+
+      # top ngrams (transformed into arrays)
+      arrays = ngram_counts[lang].to_a.sort{|x, y| y[1] <=> x[1]}
+      @stored_ngrams += arrays[0...CUTOFF].map{|x| x[0]}
+    end
+    ngram_counts
+  end
+
+  # extract the top ngrams for every language
+  def top_ngrams(ngram_counts)
+    res = Set.new
+    ngram_counts.values.each do |hash|
+      # top ngrams (transformed into arrays)
+      arrays = hash.to_a.sort{|x, y| y[1] <=> x[1]}
+      res += arrays[0...CUTOFF].map{|x| x[0]}
+    end
+    res
+  end
+end
+
+
+end # module Rlid

data/lib/rlid/models/ordered_ngrams.rb

@@ -0,0 +1,100 @@
+module Rlid
+
+require 'rlid/models/model'
+require 'rlid/common'
+
+# a subclass should define the filename
+class OrderedNGrams < NGramModel
+  N = 3
+  def initialize(string, cutoff=300)
+    super(string, N, cutoff)
+  end
+
+  def save(file)
+    @ngram_pos.each do |ngram, pos|
+      file.write "#{ngram}    #{pos}\n"
+    end
+  end
+
+  def load(file)
+    @ngram_pos = Hash.new
+    pos = 0
+    file.each_line do |line|
+      # keep only the first @n characters of the line
+      ngram = line.gsub(/^(.{#{N}}).*\n?/, '\1')
+      @ngram_pos[ngram] = pos
+      pos += 1
+    end
+  end
+
+  def generate_model(ngram_count)
+    # top ngrams (transformed into arrays)
+    top = ngram_count.to_a.sort{|x, y| y[1] <=> x[1]}[0...@cutoff]
+    @ngram_pos = Hash.new # key is ngram value is position
+    i = 0
+    top.each {|n,| @ngram_pos[n] = i; i +=1}
+  end
+  
+  def -(other)
+    if not other.is_a?(OrderedNGrams)
+      raise InvalidArgument
+    end
+    dist = 0
+    other.ngram_pos.each do |ngram, pos_other|
+      pos_self = ngram_pos[ngram]
+      if pos_self != nil
+        dist += (pos_self - pos_other).abs
+      else
+        dist += @cutoff # max distance
+      end
+    end
+    dist
+  end
+
+protected
+  attr_reader :ngram_pos
+end
+
+
+class NGramsKDE < OrderedNGrams
+  def self.filename
+    return "3grams300kde"
+  end
+end
+
+class NGrams300 < OrderedNGrams
+  def self.filename
+    return "3grams300"
+  end
+end
+
+class NGrams800 < OrderedNGrams
+  def initialize(string)
+    super(string, 800)
+  end
+  def self.filename
+    return "3grams800"
+  end
+end
+
+class NGrams3000 < OrderedNGrams
+  def initialize(string)
+    super(string, 3000)
+  end
+  def self.filename
+    return "3grams3000"
+  end
+end
+
+
+class NGrams4000 < OrderedNGrams
+  def initialize(string)
+    super(string, 4000)
+  end
+  def self.filename
+    return "3grams4000"
+  end
+end
+
+
+end # module Rlid

data/lib/rlid/probabilities/language_probabilities.rb

@@ -0,0 +1,199 @@
+module Rlid
+
+require 'rlid/common'
+
+
+
+class Percentage
+  def initialize value
+    @value = value
+  end
+
+  def to_s
+    if @value <= 0.98
+      return "%.2g" % (@value * 100)
+    else
+      complement = 1.0 - @value
+      # complement =
+      log = -Math.log10(complement).ceil
+      digits = log - 1
+      res = "%.#{digits}f" % (@value * 100)
+      last = -1
+      while res[last] == ?9
+        digits += 1
+        res = "%.#{digits}f" % (@value * 100)
+      end
+      return res
+    end
+  end
+
+  def to_f
+    @value
+  end
+end
+
+
+
+
+
+
+
+class LanguageProbabilities
+  MAX_OUTPUT = 3
+
+  def initialize(args={})
+    @percentage = Hash.new(0)
+    args.each do |languages, percentage|
+      add(languages, percentage)
+    end
+  end
+
+  def random_language
+    r = rand
+    sum = 0
+    @percentage.each do |language, perc|
+      sum += perc
+      return language if sum > r
+      #puts "#{sum}(#{r})"
+    end
+    warn "rounding error!! (sum is not 1!!)"
+    @percentage.keys.first
+  end
+
+  def to_s
+    sorted[0...MAX_OUTPUT].map do |x|
+      # calculate the digits
+      formatted_perc = Percentage.new(x[PERC]).to_s
+      "#{x[LANG]}(#{formatted_perc})"
+    end.join(" : ")
+  end
+
+  def first
+    sorted.first[LANG]
+  end
+
+  def *(other)
+    if not other.is_a? LanguageProbabilities
+      p other.inspect
+      raise InvalidArgument.new(other)
+    end
+    res = LanguageProbabilities.new()
+    @percentage.each_key do |lang|
+      res.percentage[lang] = percentage[lang] * other.percentage[lang]
+    end
+    res.normalize
+    res
+  end
+
+private
+  def add(languages, perc)
+    if perc < 0 or perc > 1
+      perc = perc.round
+    end
+    languages = [languages] if not languages.is_a? Array
+    perc = perc.to_f / languages.size
+    languages.each {|l| @percentage[l] = perc}
+  end
+
+#  def add_remainder
+#    languages = LANGUAGES - @percentage.keys
+#    perc = 1.0 - sum
+#    add(languages, perc)
+#  end
+
+protected
+  # indexes
+  LANG = 0
+  PERC = 1
+
+  def random_language_and_delete
+    l = random_language
+    @percentage.delete(l)
+    normalize
+    l
+  end
+
+  def normalize
+    tot = sum
+    return if tot == 0
+    @percentage.each_key do |key|
+      @percentage[key] /= tot
+    end
+  end
+
+  def sum
+    @percentage.values.inject(0){|s, v| s + v}
+  end
+
+  def sorted
+    @percentage.to_a.sort!{|x,y| y[PERC] <=> x[PERC]}
+  end
+
+  attr_accessor :percentage
+end
+
+
+
+
+
+class TestProbabilities < LanguageProbabilities
+  def initialize(lang, perc_lang=0.8, perc_lang_and_common=0.98)
+    @lang = lang
+    @perc_lang = perc_lang
+    common = COMMON_LANGUAGES - [lang]
+    other = LANGUAGES - COMMON_LANGUAGES - [lang]
+    @common_size = common.size
+    @other_size = other.size
+    @perc_common = perc_lang_and_common - perc_lang
+    @perc_other = 1 - perc_lang_and_common
+    super(lang => @perc_lang, common => @perc_common, other => @perc_other)
+  end
+
+  def random_permutation
+    lang = random_language
+    return self if lang == @lang
+
+    probs = probabilities
+
+    top_lang = probs.first
+
+    probs.percentage[lang], probs.percentage[top_lang] =
+      probs.percentage[top_lang], probs.percentage[lang]
+
+    probs
+  end
+
+#    common = []
+#    @common_size.times do
+#      common << probs.random_language_and_delete
+#    end
+#    other = []
+#    @other_size.times do
+#      other << probs.random_language_and_delete
+#    end
+#
+#    LanguageProbabilities.new(
+#      lang => @perc_lang,
+#      common => @perc_common,
+#      other => @perc_other)
+
+  def probabilities
+    LanguageProbabilities.new(percentage)
+  end
+end
+
+#class Array
+#  def sum
+#    inject(0) {|s,v| s+v}
+#  end
+#end
+#x = TestProbabilities.new(:eng)
+#res = Hash.new(0)
+#100000.times{res[x.random_permutation.random_language] += 1}
+#res.each{|k,v| puts "#{k}: #{v/1000.0}"}
+#
+#res.values.select{|x| x > 1000 and x < 10000}.sum / 1000.0
+#res.values.select{|x| x > 10000}.sum / 1000.0
+#res.values.select{|x| x < 1000}.sum / 1000.0
+
+end # module Rlid