simplificator-babel 0.0.4 → 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,6 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg
+lib/data/**

data/README.markdown

@@ -1,18 +1,61 @@
 #babel
 
 Babel is a gem to identify in what language a text is written.
-It is based on the n-gram approach by Cacnar and Trenkle as described in http://www.sfs.uni-tuebingen.de/iscl/Theses/kranig.pdf
+It is based on the n-gram approach by Cavnar and Trenkle as described 
+in http://www.sfs.uni-tuebingen.de/iscl/Theses/kranig.pdf
 
 
 ##usage
     require 'rubygems'
-    require 'simplificator-babel'
-
-    # Train babel: feed it some texts
-    'An english text to train and learn'.language= 'en'
-    'Ein deutscher Text'.language= 'de'
+    require 'babel'
+    
+    def guess_language(s)
+      puts "'#{s}' is probably '#{s.language}'"
+    end
+    # load the default profiles
+    Babel.load_profiles
+    
+    # Let's see what Babel thinks about these texts
+    guess_language 'Montags ist es ruhig'
+    guess_language 'le coq est mort'
     
-    puts 
+    # Replace a profile with my own profile
+    Babel.load_profile('eng', '/path/to/my/english/profile.yml')
+    
+    # Merge profile data
+    Babel.load_profile('eng', '/path/to/my/other/english/profile.yml', :merge => true)
+    
+    # Show Top-3 Languages for a sentence
+    puts "What language could this be written in?".languages[0..2]
+
+##profiles
+Profiles are collections of n-grams and the number of occurence of each ngram.
+Babel uses n-grams with length 2-5 (bigram, trigram, tetragram, pentagram).
+You can create your own profile and decide what n-grams to use and whether 
+you want to limit or not if you want to.
+
+These profiles are shipped with the gem: 
+ * german (deu) (this profile is built from udhr_deu_1996.txt)
+ * english (eng)
+ * french (fra)
+ * spanish (spa)
+ * italian (ita)
+ 
+Want another profile built in? Send an email to info@simplificator.com and if there are enough 
+requests we add the profile.
+
+The profiles that are shipped with babel are based on the texts found at
+http://www.unicode.org/udhr/index_by_code.html
+
+##generating profiles
+Profiles can be generated with the data found in http://www.unicode.org/udhr/assemblies/udhr_txt.zip or with any other text.
+Once a profile is generated, Babel can store it in YAML format and load it again from YAML.
+
+there is a rake task which simplifies profile generation:
+   rake babel:build_profile lang=foo file=myfile.txt dir=destination-directory
+
+the file which is generated from this command can be loaded by 
+   Babel.load_profile 'foo', 'profile_foo.yml'
 
 ##Copyright
 

data/Rakefile

@@ -10,6 +10,9 @@ begin
     gem.homepage = "http://github.com/simplificator/babel"
     gem.authors = ["simplificator"]
     gem.add_dependency('ya2yaml', '>= 0.2.6')
+    gem.files.exclude 'lib/data'
+    #gem.files.exclude 'lib/data/*.xml'
+    gem.files.include 'lib/data/*.zip'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
 rescue LoadError
@@ -53,4 +56,47 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
+require 'rubygems'
+require 'zip/zip'
+require 'lib/babel'
+
+namespace :babel do
+  task :unpack_data do
+    dir = File.join(File.dirname(__FILE__), 'lib', 'data')
+    file = File.join(dir, 'udhr_txt.zip')
+    Zip::ZipFile.open(file) do |zip|
+      zip.each do |entry|
+        destination = File.join(dir, entry.name)
+        FileUtils.mkdir_p(File.dirname(destination))
+        FileUtils.rm(destination) if File.exists?(destination)
+        zip.extract(entry, destination)
+      end
+    end
+    FileUtils.cp(File.join(dir, 'udhr_deu_1996.txt'), File.join(dir, 'udhr_deu.txt'))
+  end
+  
+  task :build_profile do
+    if ENV['lang']
+      lang = ENV['lang']
+      file = ENV['file']
+      dir = ENV['dir'] || File.dirname(__FILE__)
+      skip = ENV['skip']
+      limit = ENV['limit']
+      unless file
+        skip ||= 5 # skip header in data files. english all the time
+        file = File.join(File.dirname(__FILE__), 'lib', 'data', "udhr_#{lang}.txt")
+      end
+      puts "Learning about #{lang} from #{file} and save it to #{dir}"
+      File.open(file, 'r') do |f|
+        f.each_with_index do |line, index|
+          if index > skip 
+           Babel.learn(lang, line)
+          end
+        end
+      end
+      Babel.save_profile(lang, :dir => dir, :limit => limit)
+    end
+  end
+  
+end
 

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 4
+:patch: 0
 :major: 0
-:minor: 0
+:minor: 1

data/babel.gemspec

@@ -0,0 +1,64 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{babel}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["simplificator"]
+  s.date = %q{2009-07-13}
+  s.email = %q{info@simplificator.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.markdown"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.markdown",
+     "Rakefile",
+     "VERSION.yml",
+     "babel.gemspec",
+     "lib/babel.rb",
+     "lib/babel/babel.rb",
+     "lib/babel/profile.rb",
+     "lib/babel/string_extensions.rb",
+     "lib/profiles/profile_deu.yml",
+     "lib/profiles/profile_eng.yml",
+     "lib/profiles/profile_fra.yml",
+     "lib/profiles/profile_ita.yml",
+     "lib/profiles/profile_spa.yml",
+     "lib/profiles/udhr_txt.zip",
+     "samples/guessing.rb",
+     "test/babel_test.rb",
+     "test/profile_test.rb",
+     "test/string_extensions_test.rb",
+     "test/test_helper.rb"
+  ]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/simplificator/babel}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.2}
+  s.summary = %q{Utility to guess the language of a text}
+  s.test_files = [
+    "test/babel_test.rb",
+     "test/profile_test.rb",
+     "test/string_extensions_test.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<ya2yaml>, [">= 0.2.6"])
+    else
+      s.add_dependency(%q<ya2yaml>, [">= 0.2.6"])
+    end
+  else
+    s.add_dependency(%q<ya2yaml>, [">= 0.2.6"])
+  end
+end

data/lib/babel/babel.rb

@@ -1,54 +1,115 @@
+#
+# 
+# Profile Generation:
+# Whenever it's about generating a Profile (Babel.learn, Babel.distances and Babel.guess)
+# you can pass 
+#  * :min_length (2)
+#  * :max_length (5)
+#  * :pad (true)
+# They are just forwared to String.n_grams (default values in braces)
+# It's highly recomended that you use the same settings for learning and guessing....
+
+
+
 module Babel
   @profiles = {}
   PROFILE_DIR = File.join(File.dirname(__FILE__), '..', 'profiles')
+  
+  # Learn that a text is in a given language.
+  # Calls Profile.learn for the profile with the given language.
   def self.learn(lang, text, options = {})
     lang = lang.to_s
-    profile = @profiles[lang] ||= Profile.new()
+    profile = @profiles[lang] ||= Profile.new(lang)
     profile.learn(text, options)
   end
 
-
+  # Clear all the profiles
   def self.clear_profiles
     @profiles = {}
   end
+  # find the profile for a language
+  def self.profile(lang)
+    @profiles[lang]
+  end
   
-  def self.guess(source, options = {})
-    found = nil
-    Babel.distances(source).each do |entry|
-      found = entry if found.nil? || entry.last < found.last
+  # register a profile
+  # pass :merge => true to merge into an existing profile
+  def self.register_profile(profile, options = {})
+    if options[:merge] && @profiles[profile.language]
+      @profiles[profile.language].merge(profile)
+    else
+      @profiles[profile.language] = profile
     end
-    found.first if found
   end
   
-  # An array of arrays of [language, distance] arrays
-  def self.distances(text)
-    source = Profile.new.learn(text)
-    @profiles.map { |lang, target| [lang, source.distance(target)] }
+  # Guess the language of a text.
+  # As soon as there is at least one profile, this method always
+  # returns a value (perhaps the wrong) one... 
+  # I.e. if only "eng" profile is registered, then this method will always retun "eng"
+  # not matter what text pass
+  #
+  def self.guess(source, options = {})
+    distances = Babel.distances(source, options)
+    distances.first.first if distances.first
   end
-
-  private
   
-  def self.file_name(dir, lang)
-    File.join(dir, "profile_#{lang}.yml")
+  # An array of arrays of [language, distance] arrays.
+  # The language with the shortest distance is the most probable solution.
+  # Sorted by distance, ascending (first item is most probable)
+  def self.distances(text, options = {})
+    source = Profile.new.learn(text, options)
+    @profiles.map { |lang, target| [lang, source.distance(target)] }.sort {|o1, o2| o1.last <=> o2.last}
   end
   
-  # Load a specific profile ()
+
+  # Load all the profiles from a given directory.
+  # Loads all .yml files so be careful what directory you specify.
+  # options are:
+  #  * :dir the directory, defaults to Babel::PROFILE_DIR
+  # See Babel.load_profile() for other options
   def self.load_profiles(options = {})
     dir = options[:directory] || PROFILE_DIR
     Dir[File.join(PROFILE_DIR, '*.yml')].each do |file|
-      file =~ /profile_(.+)\.yml/
-      @profiles[$1] = YAML.load_file(file)
+      Babel.load_profile(file, options)
     end
   end
   
+  # Load a single profile
+  # Options are:
+  #  * :merge see Babel.register_profile for details
+  def self.load_profile(file, options = {})
+    Babel.register_profile(YAML.load_file(file), options)
+  end
+  
+  # Save the profiles to a specifified directory.
+  # See Babel.save_profile() for options
   def self.save_profiles(options = {})
-    dir = options[:directory] || PROFILE_DIR
     @profiles.each do |lang, profile|
-      profile.limit(options[:limit]) if options.has_key?(:limit)
-      File.open(file_name(dir, lang), 'wb') do |file|
-        file.write(profile.ya2yaml)
-      end
+      Babel.save_profile(lang, options)
+    end
+  end
+  
+  # Save a specific profile
+  # Options are:
+  # * :dir -> the directory wo save the files to. Defaults to Babel::PROFILE_DIR
+  # * :limit -> Call limit() on the profile before save. This reduces the size of the profile 
+  #             for the cost of (possibly) less accurate language guessing
+  def self.save_profile(lang, options = {})
+    dir = options[:dir] || PROFILE_DIR
+    profile = Babel.profile(lang)
+    profile.limit(options[:limit]) if options[:limit]
+    File.open(file_name(dir, lang), 'wb') do |file|
+      file.write(profile.ya2yaml)
     end
+  end  
+
+  private
+  
+  # Build the file name for a profile file
+  # Naming scheme: profile_<LANG>.yml
+  def self.file_name(dir, lang)
+    File.join(dir, "profile_#{lang}.yml")
   end
+  
 end
 

data/lib/babel/profile.rb

@@ -1,43 +1,61 @@
 module Babel
   class Profile
-    def initialize()
-      @profile = {}
+    attr_reader :language
+    attr_reader :data
+    def initialize(language = nil)
+      @data = {}
       @total_occurences = 0
+      @language = language
     end
     
+    
+    # learn a text
+    # following options are used when generating the n-grams:
+    #  * min_length => 2
+    #  * max_length => 5
+    #  * pad => true
     def learn(text, options = {})
       options = {:min_length => 2, :max_length => 5, :pad => true}.merge(options)
       text = clean(text)
       text.split(' ').each do |word|
-        ngrams = word.ngrams(options)
-        ngrams.each do |ngram|
+        word.n_grams(options).each do |ngram|
           self.occured(ngram)
         end
       end
+      # after learning rank the new n-grams
       self.rank
       self # return self so we can chain learn commans. profile.learn('asasas').learn('asdsad')
     end
     
     
+    def merge(other)
+      if self.language != other.language
+        raise ArgumentError.new("self has a language of #{self.language} but profile to merge has #{other.language}")
+      end
+      other.data.each do |key, value|
+        self.occured(key, value.first)
+      end
+    end
+    
     # TODO: needed?
     def clean(text)
       return text
-      text = text.gsub('?', '')
-      text = text.gsub('.', '')
-      text = text.gsub(';', '')
+      text = text.gsub(/[0-9]/, '')
       text = text.gsub(':', '')
+      text = text.gsub('/', '')
+      text = text.gsub('_', '')
       text = text.gsub('(', '')
       text = text.gsub(')', '')
-      text = text.gsub('/', '')
-      text = text.gsub(/[0-9]*/, '')
-      text = text.gsub('+', '')
-      text
+      text = text.gsub(';', '')
+      text = text.gsub('?', '')
+      
+      return text
     end
+    
     # limit this profile to n items
     # profile needs to be ranked first
-    # do not use this if you plan to extend the profile later on
     def limit(boundary = 100)
-      @profile.reject! do |key, value|
+      @data.reject! do |key, value|
         raise 'Please call rank() first' if value.last == 0
         boundary < value.last
       end
@@ -46,37 +64,37 @@ module Babel
     # rank the current profile
     # ngrams are sorted by occurence and then ranked
     def rank
-      @profile.values.sort do |o1, o2|
-        o2.first <=> o1.first
-      end.each_with_index do |item, index|
-        item[1] = index + 1
-      end
+      #@data.values.sort do |o1, o2|
+      #  o2.first <=> o1.first
+      #end.each_with_index do |item, index|
+      #  item[1] = index + 1
+      #end
       
-      @profile.values.each do |value|
+      @data.values.each do |value|
         value[1] = value[0] / @total_occurences.to_f
       end
     end
     
-    # Called when a ngram is occured, optional you can pass an
+    # Called when a n-gram is occured, optional you can pass an
     # amount (how many times the ngram occured)
     def occured(ngram, amount = 1)
-      (@profile[ngram] ||= [0, 0])[0] += amount
+      (@data[ngram] ||= [0, 0])[0] += amount
       @total_occurences += amount
     end
     
     # find the occurence of a ngram. if it never occured, returns 0
     def occurence(ngram)
-      @profile[ngram] ? @profile[ngram].first : 0
+      @data[ngram] ? @data[ngram].first : 0
     end
     
     # find the ranking of a ngram. if it is not yet ranked, return 0
     def ranking(ngram)
-      @profile[ngram] ? @profile[ngram].last : 0
+      @data[ngram] ? @data[ngram].last : 0
     end  
     
     # Calculate the distance to another profile
     def distance(other)
-      @profile.inject(0) do |memo, item|
+      @data.inject(0) do |memo, item|
         other_ranking = other.ranking(item.first)
         if other_ranking == 0
           memo += 1
@@ -88,7 +106,7 @@ module Babel
     
       
     def to_s
-      @profile.inspect
+      @data.inspect
     end
   end
 end

data/lib/babel/string_extensions.rb

@@ -1,15 +1,19 @@
 class String 
-  
-  # TODO: recursive?
-  def ngrams(options = {})
+  # Generate n-grams for a string.
+  # options are:
+  #  :min_length : minimum length of the n-grams (defaults to 1)
+  #  :max_length : maximum length of the n-grams (defaults to self.length)
+  #  :pad : pad wiht '_' to generate all possible n-grams (defaults to false)
+  def n_grams(options = {})
+    # TODO: recursive?
+    # TODO: use min/max length for loop index instead of looping
+    # all and then use if test to decide if to add or not
     min_length = options[:min_length] || 1
     max_length = options[:max_length] || self.length
     pad = options[:pad] || false
     value = options[:preserve_case] ? self : self.downcase
     value = "_#{value}#{'_' * (value.length - 1)}" if pad
     res = []
-    # TODO: use min/max length for loop index instead of looping
-    # all and then use if test to decide if to add or not
     0.upto(value.length - 1) do |index|
       index.upto(value.length - 1) do |len|
         if value[index..len].length >= min_length && value[index..len].length <= max_length
@@ -20,22 +24,23 @@ class String
     res  
   end
   
-  # def byte_grams(options = {})
-  #   min_length = options[:min_length] || 1
-  #   max_length = options[:max_length] || self.length
-  #   value = options[:preserve_case] ? self : self.downcase
-  #   res = []
-  #   
-  # end
-  
-  
-  # Ask Babel about the language of this text
-  # Can return nil if no language found
+  # Ask Babel about the language of this text.
+  # Convenience method, just calls Babel.guess().
+  # See Babel.guess for description.
   def language(options = {})
     Babel.guess(self, options)
   end
+  # Ask Bable about the languages this text could be.
+  # It will return all the registered languages with the most probable
+  # Language first. You might want to restrict this before presenting to 
+  # the user.
+  def languages(options = {})
+    Babel.distances(self, options).map() {|item| item.first}
+  end
   
-  # Tell Babel that this text is in a given language
+  # Tell Babel that this text is in a given language.
+  # Convenience method, just calls Babel.learn().
+  # See Babel.learn for description
   def language=(lang, options = {})
     Babel.learn(lang, self, options)
   end

data/lib/profiles/{profile_deu_1996.yml → profile_deu.yml}

@@ -1,5 +1,5 @@
 --- !ruby/object:Babel::Profile 
-  profile: 
+  data: 
     (i: 
       - 1
       - 1.48557506610809e-05
@@ -25359,4 +25359,5 @@
     ützu: 
       - 1
       - 1.48557506610809e-05
+  language: deu
   total_occurences: 67314

data/lib/profiles/profile_eng.yml

@@ -1,5 +1,5 @@
 --- !ruby/object:Babel::Profile 
-  profile: 
+  data: 
     ? ",_"
     : 
       - 94
@@ -20791,4 +20791,5 @@
     ‐se: 
       - 1
       - 1.85742412422453e-05
+  language: eng
   total_occurences: 53838

data/lib/profiles/profile_fra.yml

@@ -1,5 +1,5 @@
 --- !ruby/object:Babel::Profile 
-  profile: 
+  data: 
     ? ",_"
     : 
       - 118
@@ -24961,4 +24961,5 @@
     ’é: 
       - 8
       - 0.000123525415354209
+  language: fra
   total_occurences: 64764