simplificator-babel 0.0.4

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Simplificator GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,19 @@
+#babel
+
+Babel is a gem to identify in what language a text is written.
+It is based on the n-gram approach by Cacnar and Trenkle as described in http://www.sfs.uni-tuebingen.de/iscl/Theses/kranig.pdf
+
+
+##usage
+    require 'rubygems'
+    require 'simplificator-babel'
+
+    # Train babel: feed it some texts
+    'An english text to train and learn'.language= 'en'
+    'Ein deutscher Text'.language= 'de'
+    
+    puts 
+
+##Copyright
+
+Copyright (c) 2009 Simplificator GmbH. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,56 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "babel"
+    gem.summary = %Q{Utility to guess the language of a text}
+    gem.email = "info@simplificator.com"
+    gem.homepage = "http://github.com/simplificator/babel"
+    gem.authors = ["simplificator"]
+    gem.add_dependency('ya2yaml', '>= 0.2.6')
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "babel #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 4
+:major: 0
+:minor: 0

data/lib/babel/babel.rb

@@ -0,0 +1,54 @@
+module Babel
+  @profiles = {}
+  PROFILE_DIR = File.join(File.dirname(__FILE__), '..', 'profiles')
+  def self.learn(lang, text, options = {})
+    lang = lang.to_s
+    profile = @profiles[lang] ||= Profile.new()
+    profile.learn(text, options)
+  end
+
+
+  def self.clear_profiles
+    @profiles = {}
+  end
+  
+  def self.guess(source, options = {})
+    found = nil
+    Babel.distances(source).each do |entry|
+      found = entry if found.nil? || entry.last < found.last
+    end
+    found.first if found
+  end
+  
+  # An array of arrays of [language, distance] arrays
+  def self.distances(text)
+    source = Profile.new.learn(text)
+    @profiles.map { |lang, target| [lang, source.distance(target)] }
+  end
+
+  private
+  
+  def self.file_name(dir, lang)
+    File.join(dir, "profile_#{lang}.yml")
+  end
+  
+  # Load a specific profile ()
+  def self.load_profiles(options = {})
+    dir = options[:directory] || PROFILE_DIR
+    Dir[File.join(PROFILE_DIR, '*.yml')].each do |file|
+      file =~ /profile_(.+)\.yml/
+      @profiles[$1] = YAML.load_file(file)
+    end
+  end
+  
+  def self.save_profiles(options = {})
+    dir = options[:directory] || PROFILE_DIR
+    @profiles.each do |lang, profile|
+      profile.limit(options[:limit]) if options.has_key?(:limit)
+      File.open(file_name(dir, lang), 'wb') do |file|
+        file.write(profile.ya2yaml)
+      end
+    end
+  end
+end
+

data/lib/babel/profile.rb

@@ -0,0 +1,94 @@
+module Babel
+  class Profile
+    def initialize()
+      @profile = {}
+      @total_occurences = 0
+    end
+    
+    def learn(text, options = {})
+      options = {:min_length => 2, :max_length => 5, :pad => true}.merge(options)
+      text = clean(text)
+      text.split(' ').each do |word|
+        ngrams = word.ngrams(options)
+        ngrams.each do |ngram|
+          self.occured(ngram)
+        end
+      end
+      self.rank
+      self # return self so we can chain learn commans. profile.learn('asasas').learn('asdsad')
+    end
+    
+    
+    # TODO: needed?
+    def clean(text)
+      return text
+      text = text.gsub('?', '')
+      text = text.gsub('.', '')
+      text = text.gsub(';', '')
+      text = text.gsub(':', '')
+      text = text.gsub('(', '')
+      text = text.gsub(')', '')
+      text = text.gsub('/', '')
+      text = text.gsub(/[0-9]*/, '')
+      text = text.gsub('+', '')
+      text
+    end
+    # limit this profile to n items
+    # profile needs to be ranked first
+    # do not use this if you plan to extend the profile later on
+    def limit(boundary = 100)
+      @profile.reject! do |key, value|
+        raise 'Please call rank() first' if value.last == 0
+        boundary < value.last
+      end
+    end
+    
+    # rank the current profile
+    # ngrams are sorted by occurence and then ranked
+    def rank
+      @profile.values.sort do |o1, o2|
+        o2.first <=> o1.first
+      end.each_with_index do |item, index|
+        item[1] = index + 1
+      end
+      
+      @profile.values.each do |value|
+        value[1] = value[0] / @total_occurences.to_f
+      end
+    end
+    
+    # Called when a ngram is occured, optional you can pass an
+    # amount (how many times the ngram occured)
+    def occured(ngram, amount = 1)
+      (@profile[ngram] ||= [0, 0])[0] += amount
+      @total_occurences += amount
+    end
+    
+    # find the occurence of a ngram. if it never occured, returns 0
+    def occurence(ngram)
+      @profile[ngram] ? @profile[ngram].first : 0
+    end
+    
+    # find the ranking of a ngram. if it is not yet ranked, return 0
+    def ranking(ngram)
+      @profile[ngram] ? @profile[ngram].last : 0
+    end  
+    
+    # Calculate the distance to another profile
+    def distance(other)
+      @profile.inject(0) do |memo, item|
+        other_ranking = other.ranking(item.first)
+        if other_ranking == 0
+          memo += 1
+        else
+          memo += (other_ranking - item.last.last).abs
+        end
+      end
+    end
+    
+      
+    def to_s
+      @profile.inspect
+    end
+  end
+end

data/lib/babel/string_extensions.rb

@@ -0,0 +1,42 @@
+class String 
+  
+  # TODO: recursive?
+  def ngrams(options = {})
+    min_length = options[:min_length] || 1
+    max_length = options[:max_length] || self.length
+    pad = options[:pad] || false
+    value = options[:preserve_case] ? self : self.downcase
+    value = "_#{value}#{'_' * (value.length - 1)}" if pad
+    res = []
+    # TODO: use min/max length for loop index instead of looping
+    # all and then use if test to decide if to add or not
+    0.upto(value.length - 1) do |index|
+      index.upto(value.length - 1) do |len|
+        if value[index..len].length >= min_length && value[index..len].length <= max_length
+            res << value[index..len] 
+        end
+      end
+    end
+    res  
+  end
+  
+  # def byte_grams(options = {})
+  #   min_length = options[:min_length] || 1
+  #   max_length = options[:max_length] || self.length
+  #   value = options[:preserve_case] ? self : self.downcase
+  #   res = []
+  #   
+  # end
+  
+  
+  # Ask Babel about the language of this text
+  # Can return nil if no language found
+  def language(options = {})
+    Babel.guess(self, options)
+  end
+  
+  # Tell Babel that this text is in a given language
+  def language=(lang, options = {})
+    Babel.learn(lang, self, options)
+  end
+end

data/lib/babel.rb

@@ -0,0 +1,10 @@
+if RUBY_VERSION < '1.9'
+  require 'jcode'
+  $KCODE = 'u'
+end
+
+require File.dirname(__FILE__) + '/babel/string_extensions'
+require File.dirname(__FILE__) + '/babel/babel'
+require File.dirname(__FILE__) + '/babel/profile'
+
+require 'ya2yaml'