lingo 1.8.4.2 → 1.8.5

data/lib/lingo/database/source/key_value.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -37,16 +37,17 @@ class Lingo
 
       class KeyValue < self
 
+        DEFAULT_SEPARATOR = '*'
+
         def initialize(id, lingo)
-          super
-          @pat = /^(#{@wrd})#{Regexp.escape(@sep ||= '*')}(#{@wrd})$/
+          super(id, lingo, Language::LA_UNKNOWN)
+          @pat = /^(#{@wrd})#{Regexp.escape(@sep ||= DEFAULT_SEPARATOR)}(#{@wrd})$/
         end
 
         private
 
         def convert_line(line, key, val)
-          key, val = key.strip, val.strip
-          [key, %W[#{val unless key == val}##{@def}]]
+          [key.strip, %W[#{val.strip}##{@def}]]
         end
 
       end

data/lib/lingo/database/source/multi_key.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -38,12 +38,15 @@ class Lingo
 
       class MultiKey < self
 
+        DEFAULT_SEPARATOR = ';'
+
         def initialize(id, lingo)
           super
-          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep ||= ';')}#{@wrd})*$/
+          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep ||= DEFAULT_SEPARATOR)}#{@wrd})*$/
         end
 
         def set(db, key, val)
+          key += "##{@def}" if @def
           val.each { |v| db[v] = [key] }
         end
 

data/lib/lingo/database/source/multi_value.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -36,14 +36,16 @@ class Lingo
 
       class MultiValue < self
 
+        DEFAULT_SEPARATOR = ';'
+
         def initialize(id, lingo)
           super
-          @pat, @idx = /^#{@wrd}(?:#{Regexp.escape(@sep ||= ';')}#{@wrd})*$/, -1
+          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep ||= DEFAULT_SEPARATOR)}#{@wrd})*$/
         end
 
         def set(db, key, val)
-          db[key = "#{IDX_REF}#{@idx += 1}"] = val
-          val.each { |v| db[v] = [key] }
+          values = val.map { |v| @def ? "#{v}##{@def}" : v }
+          val.each { |v| db[v] = values }
         end
 
         private

data/lib/lingo/database/source/single_word.rb

@@ -37,16 +37,15 @@ class Lingo
       class SingleWord < self
 
         def initialize(id, lingo)
-          super
+          super(id, lingo, Language::LA_NOUN)
           @pat = /^(#{@wrd})$/
-          @def = @config.fetch('def-wc',     's').downcase
           @mul = @config.fetch('def-mul-wc', @def).downcase
         end
 
         private
 
         def convert_line(line, key, val)
-          [key = key.strip, %W[##{key =~ /\s/ ? @mul : @def}]]
+          [k = key.strip, %W[#{k}##{k.include?(' ') ? @mul : @def}]]
         end
 
       end

data/lib/lingo/database/source/word_class.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -36,17 +36,36 @@ class Lingo
 
       class WordClass < self
 
+        DEFAULT_SEPARATOR = ','
+
+        GENDER_SEPARATOR  = '.'
+
         def initialize(id, lingo)
           super
-          @pat = /^(#{@wrd})#{Regexp.escape(@sep ||= ',')}((?:#{@wrd}#\w)+)$/
+
+          gen = Regexp.escape(GENDER_SEPARATOR)
+          sep = Regexp.escape(@sep ||= DEFAULT_SEPARATOR)
+
+          w, a = '\w%1$s(?:\|\w%1$s)*', '[+]?'
+          wc   = "##{w % a}(?:#{gen}#{w % ''})?"
+
+          @pat = /^(#{@wrd})#{sep}((?:#{@wrd}#{wc})+)$/
         end
 
         private
 
         def convert_line(line, key, val)
-          [key = key.strip, val.strip.scan(/(\S.+?)\s*#(\w)/).map! { |v, c|
-            "#{v unless key == v}##{c}"
-          }]
+          values = []
+
+          val.strip.scan(/(\S.*?)\s*#(\S+)/) { |k, v|
+            v, f = v.split('.')
+
+            v.split('|').product(f ? f.split('|') : [nil]) { |w, g|
+              values << "#{k}##{w}##{g}"
+            }
+          }
+
+          [key.strip, values]
         end
 
       end

data/lib/lingo/database/source.rb

@@ -53,7 +53,7 @@ class Lingo
 
       attr_reader :pos
 
-      def initialize(id, lingo)
+      def initialize(id, lingo, def_wc_default = nil)
         @config = lingo.database_config(id)
 
         source_file = Lingo.find(:dict, name = @config['name'], relax: true)
@@ -68,7 +68,8 @@ class Lingo
 
         raise SourceFileNotFoundError.new(name, id) unless @src.exist?
 
-        @def = @config.fetch('def-wc', Language::LA_UNKNOWN).downcase
+        @def = @config.fetch('def-wc', def_wc_default)
+        @def = @def.downcase if @def
         @sep = @config['separator']
 
         @wrd = "(?:#{Language::Char::ANY})+"
@@ -87,7 +88,8 @@ class Lingo
         @src.each_line($/, encoding: ENC) { |line|
           @pos += length = line.bytesize
 
-          next if line =~ /\A\s*#/ || line.strip.empty?
+          line.strip!
+          next if line.empty? || line.start_with?('#')
 
           line.chomp!
           line.replace(Unicode.downcase(line))

data/lib/lingo/database.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -24,7 +24,7 @@
 ###############################################################################
 #++
 
-require_relative 'database/show_progress'
+require_relative 'database/progress'
 require_relative 'database/crypter'
 require_relative 'database/source'
 
@@ -40,15 +40,9 @@ class Lingo
   class Database
 
     FLD_SEP = '|'
-    IDX_REF = '^'
     KEY_REF = '*'
     SYS_KEY = '~'
 
-    IDX_REF_ESC = Regexp.escape(IDX_REF)
-    KEY_REF_ESC = Regexp.escape(KEY_REF)
-
-    INDEX_PATTERN = %r{\A#{IDX_REF_ESC}\d+\z}
-
     BACKENDS       = []
     BACKEND_BY_EXT = {}
 
@@ -68,13 +62,11 @@ class Lingo
 
     end
 
-    attr_reader :backend
-
     def initialize(id, lingo)
       @id, @lingo, @config, @db = id, lingo, lingo.database_config(id), nil
 
-      @srcfile = Lingo.find(:dict, @config['name'], relax: true)
-      @crypter = @config.has_key?('crypt') && Crypter.new
+      @srcfile = Lingo.find(:dict, config['name'], relax: true)
+      @crypter = config.key?('crypt') && Crypter.new
 
       @val = Hash.new { |h, k| h[k] = [] }
 
@@ -93,6 +85,8 @@ class Lingo
       convert unless uptodate?
     end
 
+    attr_reader :lingo, :config, :backend
+
     def closed?
       !@db || _closed?
     end
@@ -125,18 +119,13 @@ class Lingo
 
     def [](key)
       val = _val(key) unless closed?
-      return unless val
-
-      # Äquvalenzklassen behandeln
-      val.split(FLD_SEP).map { |v|
-        v =~ INDEX_PATTERN ? _val(v) : v
-      }.compact.join(FLD_SEP).split(FLD_SEP)
+      val.split(FLD_SEP) if val
     end
 
     def []=(key, val)
       return if closed?
 
-      val = @val[key].concat(val).sort!
+      val = @val[key].concat(val)
       val.uniq!
 
       val = val.join(FLD_SEP)
@@ -144,7 +133,7 @@ class Lingo
     end
 
     def warn(*msg)
-      @lingo.warn(*msg)
+      lingo.warn(*msg)
     end
 
     private
@@ -171,9 +160,24 @@ class Lingo
       get_backend(mod) or raise BackendNotAvailableError.new(mod, file)
     end
 
+    def config_hash
+      hashes = [config]
+
+      if use_lex = config['use-lex']
+        hashes.concat(lingo.
+          dictionary_config['databases'].
+          values_at(*use_lex.split(SEP_RE)))
+      end
+
+      Crypter.digest(hashes.inspect)
+    end
+
     def uptodate?(file = @stofile)
       src = Pathname.new(@srcfile)
-      @source_key = lambda { [src.size, src.mtime].join(FLD_SEP) }
+
+      @source_key = lambda {
+        [src.size, src.mtime, VERSION, config_hash].join(FLD_SEP)
+      }
 
       sys_key = open { @db[SYS_KEY] } if File.exist?(file)
       sys_key && (!src.exist? || sys_key == @source_key.call)
@@ -217,7 +221,7 @@ class Lingo
     end
 
     def _val(key)
-      if val = _get(@crypter ? @crypter.digest(key) : key)
+      if val = _get(@crypter ? Crypter.digest(key) : key)
         _encode!(val)
         @crypter ? @crypter.decode(key, val) : val
       end
@@ -227,33 +231,25 @@ class Lingo
       str.force_encoding(ENC)
     end
 
-    def convert(verbose = @lingo.config.stderr.tty?)
-      src = Source.get(@config.fetch('txt-format', 'key_value'), @id, @lingo)
-
-      if lex = @config['use-lex']
-        a = [{ 'source' => lex.split(SEP_RE), 'mode' => @config['lex-mode'] }, @lingo]
-        d, g = Language::Dictionary.new(*a), Language::Grammar.new(*a); a = nil
+    def convert(verbose = lingo.config.stderr.tty?)
+      src = Source.get(config.fetch('txt-format', 'key_value'), @id, lingo)
 
-        sep, block = ' ', lambda { |f|
-          (r = d.find_word(f)).unknown? &&
-            (c = (r = g.find_compound(f)).compo_form) ? c.form : r.norm
-        }
-      end
+      sep, key_map, val_map = prepare_lex
 
-      ShowProgress.new(self, src, verbose) { |progress| create {
+      Progress.new(self, src, verbose) { |progress| create {
         src.each { |key, val|
-          progress[src.pos]
+          progress << src.pos
 
           if key
             key.chomp!('.')
 
-            if lex && key.include?(sep)
-              k = key.split(sep).map!(&block).join(sep)
+            if sep && key.include?(sep)
+              key = key.split(sep).map!(&key_map).join(sep)
+              val = val.map { |v| val_map[v.split(sep)].join(sep) } if val_map
 
-              c = k.count(sep) + 1
-              self[k.split(sep)[0, 3].join(sep)] = ["#{KEY_REF}#{c}"] if c > 3
-
-              key, val = k, val.map { |v| v.start_with?('#') ? key + v : v }
+              if (cnt = key.count(sep)) > 2
+                self[key.split(sep)[0, 3].join(sep)] = ["#{KEY_REF}#{cnt + 1}"]
+              end
             end
           end
 
@@ -264,6 +260,71 @@ class Lingo
       } }
     end
 
+    def prepare_lex
+      use_lex = config['use-lex'] or return
+
+      args = [{
+        'source' => use_lex.split(SEP_RE),
+        'mode'   => config['lex-mode']
+      }, lingo]
+
+      dic = Language::Dictionary.new(*args)
+      gra = Language::Grammar.new(*args)
+
+      args = nil
+
+      if inflect = config['inflect']
+        inflect, wc = inflect == true ? %w[s e] : inflect.split(SEP_RE), 'a'
+
+        if cfg = lingo.dictionary_config['inflect'] and suffixes = cfg[wc]
+          wc, re = /#{wc}/, /\A[^#]+/
+        else
+          warn "#{self.class}: No suffixes to inflect ##{wc}: #{@id}"
+          inflect = false
+        end
+      end
+
+      [' ', lambda { |form|
+        word = dic.find_word(form)
+
+        if word.unknown?
+          compo = gra.find_compound(form)
+
+          if compo_form = compo.compo_form
+            compo_form.form
+          else
+            compo.norm
+          end
+        else
+          word.norm
+        end
+      }, inflect && lambda { |forms|
+        inflectables = []
+
+        forms.each { |form|
+          word = dic.find_word(word_form = form[re])
+
+          if word.identified? and lexical = word.get_class(wc).first
+            inflectables << form if form == lexical.form
+          else
+            unless inflectables.empty?
+              comp = gra.find_compound(word_form) if word.unknown?
+              word = comp.head || comp if comp && !comp.unknown?
+
+              if word.attr?(*inflect)
+                suffix = suffixes[word.genders.compact.first]
+                inflectables.each { |lex_form| lex_form << suffix } if suffix
+              end
+            end
+
+            break
+          end
+        }
+
+        forms
+      }]
+    end
+
   end
 
 end

data/lib/lingo/error.rb

@@ -89,7 +89,7 @@ class Lingo
     end
 
     def to_s
-      error("An error occured when trying to #{action} `#{file}'")
+      error("An error occured while trying to #{action} `#{file}'")
     end
 
   end
@@ -194,7 +194,29 @@ class Lingo
     end
 
     def to_s
-      error("#{class_name}: An error occured while trying to load '#{lib}'")
+      error("#{class_name}: An error occured while trying to load `#{lib}'")
+    end
+
+  end
+
+  class TokenizeError < LingoError
+
+    attr_reader :line, :file, :num, :err
+
+    def initialize(line, file, num, err)
+      @line, @file, @num, @err = line, file, num, err
+    end
+
+    def to_s
+      line, file = self.line, self.file
+
+      if line.is_a?(String) && line.length > 48
+        line = line[0, 45] + '...'
+      end
+
+      file &&= "#{file}:#{num}: "
+
+      error("An error occured while trying to tokenize #{file}#{line.inspect}")
     end
 
   end

data/lib/lingo/language/dictionary.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -30,8 +30,6 @@ class Lingo
 
     class Dictionary
 
-      KEY_REF_RE = %r{\A#{Database::KEY_REF_ESC}\d+}
-
       def self.open(*args)
         yield dictionary = new(*args)
       ensure
@@ -39,7 +37,7 @@ class Lingo
       end
 
       def initialize(config, lingo)
-        unless config.has_key?('source')
+        unless config.key?('source')
           raise ArgumentError, "Required parameter `source' missing."
         end
 
@@ -70,22 +68,18 @@ class Lingo
       #
       # Erstellt aus dem String ein Wort und sucht nach diesem im Wörterbuch.
       def find_word(str)
-        (@_word ||= {})[str] ||= Word.new(str, WA_UNKNOWN).tap { |w|
-          unless (lexicals = select_with_suffix(str)).empty?
-            w.lexicals = lexicals
-            w.attr = WA_IDENTIFIED
-          end
-        }
+        (@_word ||= {})[str] ||=
+          Word.new(str, WA_UNKNOWN).identify(select_with_suffix(str))
       end
 
-      def find_synonyms(obj, syn = [])
+      def find_synonyms(obj, syn = [], com = true)
         lex = obj.lexicals
         lex = [obj] if lex.empty? && obj.unknown?
 
-        com, ref = obj.attr == WA_COMPOUND, KEY_REF_RE
+        com &&= obj.attr == WA_COMPOUND
 
         lex.each { |l|
-          select(l.form, syn) { |i| i =~ ref } unless com &&
+          select(l.form, syn) unless com &&
             l.attr != LA_COMPOUND || l.attr == LA_SYNONYM
         }
 
@@ -97,14 +91,11 @@ class Lingo
       # Sucht alle Wörterbücher durch und gibt den ersten Treffer zurück (+mode = first+), oder alle Treffer (+mode = all+)
       def select(str, lex = [])
         @src.each { |src|
-          l = src[str] or next
-          lex.concat(block_given? ? l.delete_if { |i| yield i } : l)
+          lex.concat(src[str] || next)
           break unless @all
         }
 
-        lex.sort!
-        lex.uniq!
-
+        lex.empty? && block_given? ? yield(lex) : lex.uniq!
         lex
       end
 
@@ -113,7 +104,17 @@ class Lingo
       # Sucht alle Wörterbücher durch und gibt den ersten Treffer zurück (+mode = first+), oder alle Treffer (+mode = all+).
       # Sucht dabei auch Wörter, die um wortklassenspezifische Suffixe bereinigt wurden.
       def select_with_suffix(str)
-        select_with_affix(:suffix, str)
+        select(str) { |lex|
+          each_affix(str) { |form, attr|
+            unless (selected = select(form)).empty?
+              if selected.first.attr == LA_COMPOUND
+                lex.concat(selected) if selected.last.attr?(attr)
+              else
+                selected.each { |l| lex << l if l.attr?(attr) }
+              end
+            end
+          }
+        }
       end
 
       # _dic_.select_with_infix( _aString_ ) -> _ArrayOfLexicals_
@@ -121,44 +122,15 @@ class Lingo
       # Sucht alle Wörterbücher durch und gibt den ersten Treffer zurück (+mode = first+), oder alle Treffer (+mode = all+).
       # Sucht dabei auch Wörter, die eine Fugung am Ende haben.
       def select_with_infix(str)
-        select_with_affix(:infix, str)
-      end
-
-      # _dic_.suffix_lexicals( _aString_ ) -> _ArrayOfLexicals_
-      #
-      # Gibt alle möglichen Lexicals zurück, die von der Endung her auf den String anwendbar sind:
-      #
-      # dic.suffix_lexicals("Hasens") -> [(hasen/s), (hasen/e), (has/e)]
-      def suffix_lexicals(str)
-        affix_lexicals(:suffix, str)
-      end
-
-      # _dic_.gap_lexicals( _aString_ ) -> _ArrayOfLexicals_
-      #
-      # Gibt alle möglichen Lexicals zurück, die von der Endung her auf den String anwendbar sind:
-      def infix_lexicals(str)
-        affix_lexicals(:infix, str)
-      end
-
-      private
-
-      def select_with_affix(affix, str)
-        lex = select(str)
-
-        affix_lexicals(affix, str).each { |a| select(a.form, lex) { |b|
-          affix == :suffix && a.attr != b.attr
-        } } if lex.empty?
-
-        lex
+        select(str) { |lex|
+          each_affix(str, :infix) { |form, _| select(form, lex) }
+        }
       end
 
-      def affix_lexicals(affix, str)
-        lex = instance_variable_get("@#{affix}es").map { |r, e, t|
-          Lexical.new("#{$`}#{e == '*' ? '' : e}#{$'}", t) if str =~ r
+      def each_affix(str, affix = :suffix)
+        instance_variable_get("@#{affix}es").each { |r, e, t|
+          yield "#{$`}#{e == '*' ? '' : e}#{$'}", t if str =~ r
         }
-
-        lex.compact!
-        lex
       end
 
     end