langa 1.0.0

data/lib/langa/languageanalyzer.rb

@@ -0,0 +1,134 @@
+=begin
+
+  Copyright © 2007 John Vorhauer
+  Contact me at langa@vorhauer.de near 50°55'N+6°55'E.
+  
+  This file is part of Langa.
+
+  Langa is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Langa is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with Langa.  If not, see <http://www.gnu.org/licenses/>.
+
+  For a detailed functional description of Langa see README file
+  
+=end
+
+require 'rubygems'
+require 'languages'
+
+=begin rdoc
+
+  The class LanguageAnalyzer is the heart of Langa. It has two main use
+  cases:
+
+  = Recognize the language of a textfile
+  In this mode the LanguageAnalyzer identifies the language of a textfile by
+  comparing the fingerprint of the textfile against the ones documented in
+  the language configuration file 'language.dna'. Call
+  
+    la = LanguageAnalyzer.new
+    la.analyze_file('german-file') -> 'deu'
+  
+  If you wish additional codepage conversion for the input file call
+
+    la.analyze_file('german-file-iso-8859-1', '8859-1') -> 'deu'
+
+  = Create a new language fingerprint
+  If you have a big textfile of a previously unknown language, you can 
+  calculate the fingerprint of this language and add it to the language
+  configuration file 'language.dna'. Call
+
+    la = LanguageAnalyzer.new
+    la.scan_language_dna('landir/*')
+    
+  to scan all files from the landir directory. To automatically identify
+  the iso 639 language codes and the codepage that should be used for reading,
+  name the input files in a form '<iso-code>.<Language>.<codepage>.txt', i.e.
+  'landir/deu.German.utf-8.txt'.
+     
+=end
+
+class LanguageAnalyzer
+
+  # Create a new instance of the LanguageAnalyzer
+  #   la = LanguageAnalyzer.new
+  def initialize(language_file='language.dna')
+    @languages = Languages.new(language_file)
+  end
+
+  # Get the keys of all known languages.
+  #   la.keys -> ['deu', 'eng', ...]
+  def keys
+    @languages.keys.sort
+  end
+
+  # Get the keys of all known languages.
+  #   la.config('deu') -> {'name'=>'German', 'iso1'=>'de', ...}
+  def config(key)
+    @languages.config(key)
+  end
+
+  # Get the source files of all known languages.
+  #   la.sources -> ["corpora/ger.german.utf-8.txt", ...]
+  def sources
+    @languages.values_for('source').keys
+  end
+
+  # Analyze the language of a file. With the +full_detail+ toggle you
+  # can get a complete protokoll of teh analysis.
+  #   la.analyze_file('german-file-utf8') -> 'deu'
+  #   la.analyze('german-file-iso-8859-1', '8859-1') -> 'deu'
+  def analyze( filename, codepage='utf-8', full_detail=false )
+    dna = DNA.new
+    dna.feed(filename, codepage)
+    fp = dna.fingerprint
+    
+    lang_score = Array.new
+    @languages.keys.each do |key|
+      lang = @languages.config(key)
+      lang_score << [dna.distance(lang['dna']), key, lang['name']]
+    end
+    full_detail ? lang_score.sort {|a,b| a[1]<=>b[1]}  : lang_score.sort[0][1]
+  end
+
+  # Create a new dna fingerprint for a big language file. The file should have
+  # at least 100.000 letters. The more, the better for the quality of the
+  # fingerprint and therefor for the quality of language recognition.
+  # To scan all files from a directory, use a wildcard. To automatically
+  # identify the iso 639 language codes and the codepage that should be used
+  # for reading, name the input files in a form
+  # '<iso-code>.<Language>.<codepage>.txt', i.e. 'landir/deu.German.utf-8.txt'.
+  #   la.scan_language_dna('landir/*')
+  # Copy the output to to the language configuration file 'language.dna'.
+  def scan_language_dna( pattern = '*', codepage = 'utf-8' )
+    lang, language, cp = nil, nil, codepage
+    Dir[ pattern ].each do |filename|
+#      filename =~ %r|/([^\.]+)\.([^\.]+)\.([^\.]+)|
+#      lang, language, cp = $1, $2, $3
+
+      dna = DNA.new
+      dna.feed(filename, cp)
+
+      puts Languages.to_paste('<iso 639-3 code>', {
+        'name' => '<full language name>',
+        'iso1' => '<iso 639-1 code (optional)>',
+        'source' => filename,
+        'size' => dna.size,
+        'utf8' => dna.to_utf8,
+        'fingerprint' => dna.to_s })
+    end
+  end
+
+end
+
+#la = LanguageAnalyzer.new
+#p la.sources

data/lib/langa/languages.rb

@@ -0,0 +1,147 @@
+=begin
+
+  Copyright © 2007 John Vorhauer
+  Contact me at langa@vorhauer.de near 50°55'N+6°55'E.
+  
+  This file is part of Langa.
+
+  Langa is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Langa is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with Langa.  If not, see <http://www.gnu.org/licenses/>.
+
+  For a detailed functional description of Langa see README file
+  
+=end
+
+require 'rubygems'
+require 'yaml'
+require 'dna'
+
+=begin rdoc
+The class Languages handles attributes of different languages, particularly
+the dna fingerprint used for language recognition.
+
+The attributes for each language are stored in a yaml files of the form:
+
+  <three letter iso 639-3 language code>:
+      name:   <name of the language>
+      iso1:   <two letter iso 639-1 language code (optional)>
+      bibl:   <three letter iso 639-2 bibliographic code (optional)>
+      source: <source file used for fingerprint creation>
+      size:   <number of relevant characters for fingerprint creation>
+      utf8:   <utf-8 representation of fingerprint>
+      fingerprint: <dna fingerprint of language>
+
+i.e. this is shown for the german language
+
+  deu:
+    name:   German
+    iso1:   de
+    bibl:   ger
+    source: corpora/ger.german.utf-8.txt
+    size:   92273185
+    utf8:   enirtsadhlugcmobfkwzpvüäjöyxq
+    fingerprint: 101-16251+110-9918+105-7865+114-7637+116-6348...
+    
+For ISO 639-x codes see http://www.sil.org/ISO639-3/codes.asp 
+
+=end
+
+class Languages
+
+  @languages = nil
+
+  # Create the YAML representation of a language configuration for
+  # manually pasting this to the language configuration file.
+  # Languages.to_paste('deu', {...}) -> "deu:\n    name:   German\n ..."
+  def Languages.to_paste(key, config, indent=4)
+    ind = ' ' * indent
+    cnf = config.dup
+    str = "#{key}:\n"
+    ['name','iso1','bibl','source','size','utf8','fingerprint'].each do |key|
+      if cnf.has_key?(key)
+        str << "%s%s:%s%s\n" % [ind, key, 
+          ind[(key.size+1).modulo(indent)..indent-1], cnf[key]]
+        cnf.delete(key)
+      end
+    end
+    cnf.each do |key, value|
+      str << "%s%s:%s%s\n" % [ind, key,
+        ind[(key.size+1).modulo(indent)..indent-1], value]
+    end
+    str
+  end
+
+  # Create a Language object to query language attributes
+  #   la = Languages.new('language.dna') 
+  def initialize(config_file)
+    @languages = load_language_configuration(config_file)
+    self
+  end
+
+  # Get the keys of all known languages. The keys are named according to
+  # ISO 639-3
+  #   la.keys -> ['deu', 'eng', ...]
+  def keys
+    @languages.keys.sort
+  end
+
+  # Get the values of all languages for given attribute +name+. With +lcase+
+  # you can force the results to be lowercased.
+  #   la.values_for('name') -> {'German'=>'deu', 'English'=>'eng', ...}
+  #   la.values_for('name', true) -> {'german'=>'deu', 'english'=>'eng', ...}
+  def values_for(name, lcase=false)
+    return nil if name.nil?
+    
+    result = Hash.new
+    @languages.each do |key, val|
+      unless val[name].nil?
+        result[lcase ? val[name].downcase : val[name]] = key
+      end
+    end
+    result
+  end
+  
+  # Get the complete configuration for a specific language. You can use any
+  # ISO 639 shortcut as a key (i.e. for german you can use 'deu', 'de' and
+  # 'ger')
+  #   la.config('deu') -> {"name"=>"German", "iso1"=>"de", "bibl"=>"ger", ...}
+  def config(key)
+    # => tranlate key, unless present
+    unless @languages.has_key?(key)
+      map = values_for('iso1')
+      unless map.has_key?(key)
+        map = values_for('bibl')
+        unless map.has_key?(key)
+          map = values_for('name', true)
+          key = nil unless map.has_key?(key)
+        end
+      end
+      key = map[key] unless key.nil?
+    end
+
+    key.nil? ? nil : @languages[key]
+  end
+
+  
+private
+
+  def load_language_configuration(filename)
+    YAML.load(File.open(filename)).each_value do |lang|    
+      lang['dna'] = DNA.new(lang['fingerprint'])
+    end    
+  end  
+
+end
+
+#la = Languages.new
+#p la.values_for('name', true)

data/lib/langa/randomtestfiles.rb

@@ -0,0 +1,140 @@
+=begin
+
+  Copyright © 2007 John Vorhauer
+  Contact me at langa@vorhauer.de near 50°55'N+6°55'E.
+  
+  This file is part of Langa.
+
+  Langa is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Langa is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with Langa.  If not, see <http://www.gnu.org/licenses/>.
+
+  For a detailed functional description of Langa see README file
+  
+=end
+
+require 'rubygems'
+
+require 'fileutils'
+require 'languages'
+
+=begin
+
+  The class RandomTestFiles creates various files in different languages for
+  testing purposes. Therefor it scans the language configuration for source
+  files and, if existing, extracts a portion from random position of it to an
+  extra file in a testing directory. The files are removed, when object life
+  cycle ends.
+
+=end   
+
+class RandomTestFiles
+  
+  RTF_DIR = 'rltf.temp'.freeze
+
+  def initialize(file_count, chunk_size, &block)
+    # Store test parameter
+    @count = file_count
+    @size = chunk_size
+
+    # remember root path
+    @root = File.join(File.dirname(__FILE__), '..', '..')
+    @rtf_dir = File.join(@root, RTF_DIR)
+
+    # check source files for existance
+    @sources = []
+    source_file = dest_files = nil
+    Languages.new.values_for('source').each do |source, lang|
+      source_file = File.join(@root, source)
+
+      if File.exist?(source_file)
+        dest_files = []
+
+        # => create number of desired output files
+        (1..file_count).each do |i|
+          dest_files << 
+            File.join(@rtf_dir, source + "-#{@count}-#{@size}")
+        end
+
+        # => store filename
+        @sources << [lang, source_file, dest_files]
+      end
+    end
+
+    # => initialize random seed for different results
+    srand (Time.new.to_f * 10000).to_i
+    
+    self.each &block if block_given?
+    
+    self
+  end
+
+  alias :create :initialize
+
+  def each
+    # => create temp directory
+    FileUtils.mkdir_p(RTF_DIR)
+
+    @sources.each do |source| la, lang, filename = source
+      yield la, copy_random_pieces(filename, @file_count, @chunk_size)
+    end
+
+    self.clear
+  end
+  
+  def clear
+#    FileUtils.rm(RLTF_DIRECTORY + '/*.txt')
+    FileUtils.rm_r(RLTF_DIRECTORY)
+  end
+  
+  def copy_random_pieces(filename, file_count, chunk_size)
+    files = []
+    
+    path = File.join(File.dirname(__FILE__), '..', '..')
+    source_file = File.join(path, filename)
+    
+    # => remember size of source, so that we don't grab past EOF
+    file_size = File.exist?(source_file) ? File.size(source_file) : 0
+    return nil if chunk_size > file_size
+  
+    # => open source file
+    File.open(source_file) do |file|
+    
+      # => create number of desired output files
+      (1..file_count).each do |i|
+      
+        # => grab a randomly piece of fixed size from the source
+        file.seek(rand(file_size-chunk_size), IO::SEEK_SET)
+        piece = file.read(chunk_size)
+
+        # => fix potentially broken utf-8 sequences at start of piece
+        piece = piece[1..-1] while (0x80..0xbf).include?(piece[0])
+        # => fix potentially broken utf-8 sequences at end of piece
+        piece = piece[0..-2] while (0x80..0xbf).include?(piece[-1])
+        piece = piece[0..-2] if (0xc0..0xff).include?(piece[-1])
+      
+        # => write piece to test files
+        source_file =~ /([^\/]+).txt$/i
+        files << File.join(path, "#{RLTF_DIRECTORY}/#{$1}.#{chunk_size}.#{i}.txt")
+        File.open(files[-1], 'w') do |fout|
+          fout.write piece
+        end
+      end
+    end
+    files
+  end
+
+end
+
+#rlf = RandomTestFiles.new(5,100).each do |a,b|
+#  p [a, b]
+#end

data/lib/langa/utilities.rb

@@ -0,0 +1,53 @@
+=begin
+
+  Copyright © 2007 John Vorhauer
+  Contact me at langa@vorhauer.de near 50°55'N+6°55'E.
+  
+  This file is part of Langa.
+
+  Langa is free software: you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation, either version 3 of the License, or
+  (at your option) any later version.
+
+  Langa is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  along with Langa.  If not, see <http://www.gnu.org/licenses/>.
+
+  For a detailed functional description of Langa see README file
+  
+=end
+
+=begin rdoc
+
+  Extend class Array and class String to uniformly handle code conversion
+  to and from unicode
+
+=end
+
+class String
+  
+  # Converts a string to an array of unicode values.
+  #   'äöü'.to_unicode => [228, 246, 252]
+  #   'äöü' as UTF-8 is equivalent to [195, 164, 195, 182, 195, 188]
+  def to_unicode
+    self.unpack('U*')
+  end
+
+end
+
+class Array
+  
+  # Converts an array of unicodes values to an utf-8 coded string
+  #   [228, 246, 252].to_uft8 -> 'äöü'
+  
+  def to_utf8
+		self.pack('U*')
+  end
+
+end
+