lingo 1.8.1 → 1.8.2

data/lib/lingo/language/grammar.rb

@@ -29,7 +29,7 @@ class Lingo
   module Language
 
     # Die Klasse Grammar beinhaltet grammatikalische Spezialitäten einer Sprache. Derzeit findet die
-    # Kompositumerkennung hier ihren Platz, die mit der Methode find_compositum aufgerufen werden kann.
+    # Kompositumerkennung hier ihren Platz, die mit der Methode find_compound aufgerufen werden kann.
     # Die Klasse Grammar wird genau wie ein Dictionary initialisiert. Das bei der Initialisierung angegebene Wörterbuch ist Grundlage
     # für die Erkennung der Kompositumteile.
 
@@ -40,31 +40,26 @@ class Lingo
 
       HYPHEN_RE = %r{\A(.+)-([^-]+)\z}
 
-      # initialize(config, dictionary_config) -> _Grammar_
-      # config = Attendee-spezifische Parameter
-      # dictionary_config = Datenbankkonfiguration aus de.lang
+      def self.open(*args)
+        yield grammar = new(*args)
+      ensure
+        grammar.close if grammar
+      end
+
       def initialize(config, lingo)
         init_cachable
         init_reportable
 
         @dic, @suggestions = Dictionary.new(config, lingo), []
 
-        cfg = lingo.dictionary_config['compositum']
-
-        # Ein Wort muss mindestens 8 Zeichen lang sein, damit
-        # überhaupt eine Prüfung stattfindet.
-        @min_word_size = (cfg['min-word-size'] || 8).to_i
-
-        # Die durchschnittliche Länge der Kompositum-Wortteile
-        # muss mindestens 4 Zeichen lang sein, sonst ist es kein
-        # gültiges Kompositum.
-        @min_avg_part_size = (cfg['min-avg-part-size'] || 4).to_i
-
-        # Der kürzeste Kompositum-Wortteil muss mindestens 1 Zeichen lang sein
-        @min_part_size = (cfg['min-part-size'] || 1).to_i
+        cfg = lingo.dictionary_config['compound'] ||
+              lingo.dictionary_config['compositum']  # DEPRECATE compositum
 
-        # Ein Kompositum darf aus höchstens 4 Wortteilen bestehen
-        @max_parts = (cfg['max-parts'] || 4).to_i
+        {
+          min_word_size: 8, min_avg_part_size: 4, min_part_size: 1, max_parts: 4
+        }.each { |k, v|
+          instance_variable_set("@#{k}", cfg.fetch(k.to_s.tr('_', '-'), v).to_i)
+        }
 
         # Die Wortklasse eines Kompositum-Wortteils kann separat gekennzeichnet
         # werden, um sie von Wortklassen normaler Wörter unterscheiden zu
@@ -75,7 +70,7 @@ class Lingo
         # Bestimmte Sequenzen können als ungültige Komposita erkannt werden,
         # z.B. ist ein Kompositum aus zwei Adjetiven kein Kompositum, also
         # skip-sequence = 'aa'
-        @sequences = cfg.fetch('skip-sequences', []).map(&:downcase)
+        @sequences = cfg.fetch('skip-sequences', []).map!(&:downcase)
       end
 
       def close
@@ -86,12 +81,12 @@ class Lingo
         super.update(@dic.report)
       end
 
-      # find_compositum(str) -> word wenn level=1
-      # find_compositum(str) -> [lex, sta] wenn level!=1
+      # find_compound(str) -> word wenn level=1
+      # find_compound(str) -> [lex, sta] wenn level!=1
       #
-      # find_compositum arbeitet in verschiedenen Leveln, da die Methode auch rekursiv aufgerufen wird. Ein Level größer 1
+      # find_compound arbeitet in verschiedenen Leveln, da die Methode auch rekursiv aufgerufen wird. Ein Level größer 1
       # entspricht daher einem rekursiven Aufruf
-      def find_compositum(str, level = 1, tail = false)
+      def find_compound(str, level = 1, tail = false)
         key, top, empty = str.downcase, level == 1, [[], [], '']
 
         if top && hit?(key)
@@ -108,16 +103,21 @@ class Lingo
 
         inc('Komposita geprüft')
 
-        res = permute_compositum(key, level, tail)
-        val = !(lex = res.first).empty? && valid?(str, *res[1..-1])
+        lex, sta, seq = res = permute_compound(key, level, tail)
+
+        val = !lex.empty? &&
+          sta.size              <= @max_parts         &&
+          sta.min               >= @min_part_size     &&
+          str.length / sta.size >= @min_avg_part_size &&
+          (@sequences.empty? || !@sequences.include?(seq))
 
         if top
           if val
             inc('Komposita erkannt')
 
-            com.attr = WA_KOMPOSITUM
+            com.attr = WA_COMPOUND
             com.lexicals = lex.map { |l|
-              l.attr == LA_KOMPOSITUM ? l :
+              l.attr == LA_COMPOUND ? l :
                 Lexical.new(l.form, l.attr + @append_wc)
             }
           end
@@ -128,14 +128,14 @@ class Lingo
         end
       end
 
-      # permute_compositum( _aString_ ) ->  [lex, sta, seq]
-      def permute_compositum(str, level, tail)
-        return test_compositum($1, '-', $2, level, tail) if str =~ HYPHEN_RE
+      # permute_compound( _aString_ ) ->  [lex, sta, seq]
+      def permute_compound(str, level = 1, tail = false)
+        return test_compound($1, '-', $2, level, tail) if str =~ HYPHEN_RE
 
         sug, len = @suggestions[level] ||= [], str.length
 
         1.upto(len - 1) { |i|
-          res = test_compositum(str[0, i], '', str[i, len], level, tail)
+          res = test_compound(str[0, i], '', str[i, len], level, tail)
 
           unless (lex = res.first).empty?
             return res unless lex.last.attr == LA_TAKEITASIS
@@ -146,10 +146,10 @@ class Lingo
         sug.empty? ? [[], [], ''] : sug.first.tap { sug.clear }
       end
 
-      # test_compositum() ->  [lex, sta, seq]
+      # test_compound() ->  [lex, sta, seq]
       #
       # Testet einen definiert zerlegten String auf Kompositum
-      def test_compositum(fstr, infix, bstr, level, tail)
+      def test_compound(fstr, infix, bstr, level = 1, tail = false)
         sta, seq, empty = [fstr.length, bstr.length], %w[? ?], [[], [], '']
 
         if !(blex = @dic.select_with_suffix(bstr)).sort!.empty?
@@ -159,10 +159,10 @@ class Lingo
           # 2. Word w/ infix, unless tail part
           bform, seq[1] = bstr, blex.first.attr
         elsif infix == '-'
-          blex, bsta, bseq = find_compositum(bstr, level + 1, tail)
+          blex, bsta, bseq = find_compound(bstr, level + 1, tail)
 
           if !blex.sort!.empty?
-            # 3. Compositum
+            # 3. Compound
             bform, seq[1], sta[1..-1] = blex.first.form, bseq, bsta
           else
             # 4. Take it as is
@@ -176,10 +176,10 @@ class Lingo
           # 1. Word w/ infix
           fform, seq[0] = fstr, flex.first.attr
         else
-          flex, fsta, fseq = find_compositum(fstr, level + 1, true)
+          flex, fsta, fseq = find_compound(fstr, level + 1, true)
 
           if !flex.sort!.empty?
-            # 2. Compositum
+            # 2. Compound
             fform, seq[0], sta[0..0] = flex.first.form, fseq, fsta
           elsif infix == '-'
             # 3. Take it as is
@@ -189,21 +189,12 @@ class Lingo
           end
         end
 
-        flex.concat(blex).delete_if { |l| l.attr == LA_KOMPOSITUM }.
-          push(Lexical.new(fform + infix + bform, LA_KOMPOSITUM)).sort!
+        flex.concat(blex).delete_if { |l| l.attr == LA_COMPOUND }.
+          push(Lexical.new(fform + infix + bform, LA_COMPOUND)).sort!
 
         [flex, sta, seq.join]
       end
 
-      private
-
-      def valid?(str, sta, seq)
-        sta.size               <= @max_parts         &&
-        sta.sort.first         >= @min_part_size     &&
-        str.length / sta.size  >= @min_avg_part_size &&
-        (@sequences.empty? || !@sequences.include?(seq))
-      end
-
     end
 
   end

data/lib/lingo/language/lexical.rb

@@ -39,14 +39,14 @@ class Lingo
       def <=>(other)
         return 1 unless other.is_a?(self.class)
 
-        if attr == other.attr
+        a1, a2 = attr, other.attr
+
+        if a1 == a2
           form <=> other.form
         else
-          attr.empty? ? 1 : other.attr.empty? ? -1 : begin
-            a = LA_SORTORDER.index(attr)
-            b = LA_SORTORDER.index(other.attr)
-
-            a ? b ? b <=> a : -1 : b ? 1 : attr <=> other.attr
+          a1.empty? ? 1 : a2.empty? ? -1 : begin
+            i1, i2 = [a1, a2].map(&LA_SORTORDER.method(:index))
+            i1 ? i2 ? i2 <=> i1 : -1 : i2 ? 1 : a1 <=> a2
           end
         end
       end

data/lib/lingo/language/lexical_hash.rb

@@ -37,6 +37,12 @@ class Lingo
       include Cachable
       include Reportable
 
+      def self.open(*args)
+        yield lexical_hash = new(*args)
+      ensure
+        lexical_hash.close if lexical_hash
+      end
+
       def initialize(id, lingo)
         init_cachable
         init_reportable(id)

data/lib/lingo/language/word.rb

@@ -33,8 +33,16 @@ class Lingo
 
     class Word < WordForm
 
-      def self.new_lexical(form, attr, lex_attr)
-        new(form, attr) << Lexical.new(form, lex_attr)
+      class << self
+
+        def new_lexicals(form, attr, lex)
+          new(form, attr) << lex
+        end
+
+        def new_lexical(form, attr, lex_attr)
+          new_lexicals(form, attr, Lexical.new(form, lex_attr))
+        end
+
       end
 
       # Exakte Representation der originären Zeichenkette, so wie sie im Satz
@@ -56,23 +64,32 @@ class Lingo
       end
 
       def lexicals(compound_parts = true)
-        if !compound_parts && attr == WA_KOMPOSITUM
-          @lexicals.select { |lex| lex.attr == LA_KOMPOSITUM }
+        if !compound_parts && attr == WA_COMPOUND
+          @lexicals.select { |lex| lex.attr == LA_COMPOUND }
         else
           @lexicals
         end
       end
 
-      def lexicals=(lexis)
-        if lexis.is_a?(Array)
-          @lexicals = lexis.sort.uniq
+      def lexicals=(lex)
+        if lex.is_a?(Array)
+          @lexicals = lex.sort.uniq
         else
-          raise TypeError, "wrong argument type #{lexis.class} (expected Array)"
+          raise TypeError, "wrong argument type #{lex.class} (expected Array)"
         end
       end
 
+      def add_lexicals(lex)
+        @lexicals.concat(lex)
+
+        @lexicals.sort!
+        @lexicals.uniq!
+
+        self
+      end
+
       def attrs(compound_parts = true)
-        lexicals(compound_parts).map { |lex| lex.attr }
+        lexicals(compound_parts).map(&:attr)
       end
 
       def parts
@@ -100,15 +117,15 @@ class Lingo
       end
 
       def compo_form
-        if attr == WA_KOMPOSITUM
-          get_class(LA_KOMPOSITUM).first
-        else
-          nil
-        end
+        get_class(LA_COMPOUND).first if attr == WA_COMPOUND
+      end
+
+      def full_compound?
+        attr == WA_COMPOUND && get_class('x+').empty?
       end
 
       def <<(*other)
-        lexicals.concat(other.flatten)
+        lexicals.concat(other.tap(&:flatten!))
         self
       end
 

data/lib/lingo/language/word_form.rb

@@ -71,7 +71,7 @@ class Lingo
       end
 
       def eql?(other)
-        self.class.equal?(other.class) && to_s == other.to_s
+        self.class.equal?(other.class) && (self <=> other) == 0
       end
 
       alias_method :==, :eql?

data/lib/lingo/language.rb

@@ -50,7 +50,7 @@ class Lingo
     # Status, wenn das Word nicht gefunden werden konnte
     WA_UNKNOWN    = '?'
     # Wort ist als Kompositum erkannt worden
-    WA_KOMPOSITUM = 'KOM'
+    WA_COMPOUND   = 'KOM'
     # Wort ist eine Mehrwortgruppe
     WA_MULTIWORD  = 'MUL'
     # Wort ist eine Mehrwortgruppe
@@ -58,31 +58,20 @@ class Lingo
     # Word ist unbekannt, jedoch Teil einer Mehrwortgruppe
     WA_UNKMULPART = 'MU?'
 
-    LA_SUBSTANTIV = 's'
-    LA_ADJEKTIV   = 'a'
-    LA_VERB       = 'v'
-    LA_EIGENNAME  = 'e'
-    LA_KOMPOSITUM = 'k'
-    LA_MULTIWORD  = 'm'
-    LA_SEQUENCE   = 'q'
-    LA_WORTFORM   = 'w'
-    LA_SYNONYM    = 'y'
-    LA_STOPWORD   = 't'
-    LA_TAKEITASIS = 'x'
-    LA_UNKNOWN    = '?'
-
     LA_SORTORDER = [
-      LA_MULTIWORD,
-      LA_KOMPOSITUM,
-      LA_SUBSTANTIV,
-      LA_VERB,
-      LA_ADJEKTIV,
-      LA_EIGENNAME,
-      LA_WORTFORM,
-      LA_STOPWORD,
-      LA_TAKEITASIS,
-      LA_SYNONYM,
-      LA_UNKNOWN
+      LA_SEQUENCE   = 'q',
+      LA_MULTIWORD  = 'm',
+      LA_COMPOUND   = 'k',
+      LA_NOUN       = 's',
+      LA_VERB       = 'v',
+      LA_ADJECTIVE  = 'a',
+      LA_NAME       = 'e',
+      LA_WORDFORM   = 'w',
+      LA_STOPWORD   = 't',
+      LA_TAKEITASIS = 'x',
+      LA_SYNONYM    = 'y',
+      LA_STEM       = 'z',
+      LA_UNKNOWN    = '?'
     ].reverse.join
 
   end

data/lib/lingo/reportable.rb

@@ -31,27 +31,29 @@ class Lingo
   module Reportable
 
     def init_reportable(prefix = nil)
-      @counters, @prefix = Hash.new(0), prefix ? "#{prefix}: " : ''
+      @reportable_hash   = Hash.new(0)
+      @reportable_prefix = prefix ? "#{prefix}: " : ''
     end
 
-    def inc(counter)
-      @counters[counter] += 1
+    def inc(key)
+      @reportable_hash[key] += 1
     end
 
-    def add(counter, value)
-      @counters[counter] += value
+    def add(key, val)
+      @reportable_hash[key] += val
     end
 
-    def set(counter, value)
-      @counters[counter] = value
+    def set(key, val)
+      @reportable_hash[key] = val
     end
 
-    def get(counter)
-      @counters[counter]
+    def get(key)
+      @reportable_hash[key]
     end
 
     def report
-      @counters.each_with_object({}) { |(k, v), r| r["#{@prefix}#{k}"] = v }
+      q = @reportable_prefix
+      @reportable_hash.each_with_object({}) { |(k, v), r| r["#{q}#{k}"] = v }
     end
 
   end

data/lib/lingo/show_progress.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  class ShowProgress
+
+    def initialize(obj, max, name = nil, doit = true, text = 'progress')
+      return yield self unless max && doit
+
+      @out = obj.instance_variable_get(:@lingo).config.stderr
+
+      # To get the length of the formatted string we have
+      # to actually substitute the placeholder.
+      fmt = ' [%3d%%]'
+      len = (fmt % 0).length
+
+      # Now we know how far to "go back" to
+      # overwrite the formatted string...
+      back = "\b" * len
+
+      @fmt = fmt       + back
+      @clr = ' ' * len + back
+
+      print name, ': ' if name
+
+      @rat, @cnt, @next = max / 100.0, 0, 0
+      print text
+      step
+
+      yield self
+
+      print "#{@clr} done.\n"
+    end
+
+    def [](value)
+      if defined?(@cnt)
+        @cnt = value
+        step if @cnt >= @next
+      end
+    end
+
+    private
+
+    def step
+      percent = @cnt / @rat
+      @next = (percent + 1) * @rat
+
+      print @fmt % percent if percent.finite?
+    end
+
+    def print(*args)
+      @out.print(*args)
+    end
+
+  end
+
+end

data/lib/lingo/version.rb

@@ -4,7 +4,7 @@ class Lingo
 
     MAJOR = 1
     MINOR = 8
-    TINY  = 1
+    TINY  = 2
 
     class << self
 

data/lib/lingo.rb

@@ -25,6 +25,8 @@
 #++
 
 require 'stringio'
+require 'pathname'
+require 'fileutils'
 require 'benchmark'
 require 'nuggets/file/ext'
 require 'nuggets/env/user_home'
@@ -43,7 +45,8 @@ class Lingo
   CURR = ENV['LINGO_CURR'] || '.'
 
   # The search path for Lingo dictionary and configuration files.
-  PATH = ENV['LINGO_PATH'] || [CURR, HOME, BASE].join(File::PATH_SEPARATOR)
+  PATH = ENV['LINGO_PATH'].nil? ? [CURR, HOME, BASE] :
+         ENV['LINGO_PATH'].split(File::PATH_SEPARATOR)
 
   ENV['LINGO_PLUGIN_PATH'] ||= File.join(HOME, 'plugins')
 
@@ -59,7 +62,7 @@ class Lingo
   # Default encoding
   ENC = 'UTF-8'.freeze
 
-  STRING_SEPARATOR_RE = %r{[; ,|]}
+  SEP_RE = %r{[; ,|]}
 
   class << self
 
@@ -79,7 +82,7 @@ class Lingo
       glob = File.join('??', glob) if type == :dict
 
       [].tap { |list| walk(path, options) { |dir|
-        Dir[File.join(dir, glob)].sort.each { |file|
+        Dir[File.join(dir, glob)].sort!.each { |file|
           pn = Pathname.new(file)
           list << realpath_for(pn, path) if pn.file?
         }
@@ -110,29 +113,69 @@ class Lingo
       File.join(options_for(type)[:dir], basename(type, file))
     end
 
+    def append_path(*path)
+      include_path(path)
+    end
+
+    def prepend_path(*path)
+      include_path(path, true)
+    end
+
+    def get_const(name, klass = self)
+      klass.const_get(name.camelcase)
+    rescue NameError
+      raise NameNotFoundError.new(klass, name)
+    end
+
     private
 
+    def include_path(path, pre = false)
+      PATH.insert(pre ? 0 : -1, *path.map!(&:to_s))
+    end
+
     def find_file(file, path, options)
-      pn = Pathname.new(file_with_ext(file, options)).cleanpath
+      if glob = options[:glob]
+        file = File.chomp_ext(file)
+        options[:ext] ||= '*'
+      end
+
+      file = file_with_ext(file, options)
+      pn   = Pathname.new(file).cleanpath
 
       if pn.relative?
         walk(path, options) { |dir|
           pn2 = pn.expand_path(dir)
-          pn = pn2 and break if pn2.exist?
+          ex  = pn2.exist?
+
+          pn2 = Pathname.glob(pn2).first if glob && !ex
+          pn  = pn2 and break if glob ? pn2 : ex
         }
       end
 
       realpath_for(pn, path)
+    rescue Errno::ENOENT
+      raise unless relax = options[:relax]
+      relax.respond_to?(:[]) ? relax[file] : file
     end
 
     def find_store(file, path, options)
-      base = basename(:dict, find(:dict, file, path))
+      base = basename(:dict, find(:dict, file, path) {
+        raise SourceFileNotFoundError.new(nil, find_file(file, path,
+          options.merge(glob: true, relax: lambda { |_file|
+            raise SourceFileNotFoundError.new(file, _file)
+          })
+        ))
+      })
 
       walk(path.reverse, options, false) { |dir|
         Pathname.new(dir).ascend { |i|
-          break  true                                 if i.file?
-          return File.chomp_ext(File.join(dir, base)) if i.writable?
-          break  true                                 if i.exist?
+          begin
+            stat = i.stat
+
+            break true if stat.file? || !stat.writable?
+            return File.chomp_ext(File.join(dir, base))
+          rescue Errno::ENOENT
+          end
         }
       }
 
@@ -148,7 +191,7 @@ class Lingo
     end
 
     def path_for(options)
-      options[:path] || PATH.split(File::PATH_SEPARATOR)
+      options[:path] || PATH
     end
 
     def file_with_ext(file, options)
@@ -223,30 +266,25 @@ class Lingo
 
     list.each { |hash|
       # {'attendee' => {'name'=>'Attendee', 'in'=>'nase', 'out'=>'ohr', 'param'=>'hase'}}
-      cfg = hash.values.first.merge('name' => hash.keys.first.camelcase)
+      cfg = hash.values.first.merge('name' => name = hash.keys.first.camelcase)
 
       %w[in out].each { |key| (cfg[key] ||= '').downcase! }
 
-      cfg['in']  = last_link                         if cfg['in'].empty?
-      cfg['out'] = "auto_link_out_#{auto_link += 1}" if cfg['out'].empty?
+      cfg['in']  = last_link                     if cfg['in'].empty?
+      cfg['out'] = "auto_link-#{auto_link += 1}" if cfg['out'].empty?
       last_link  = cfg['out']
 
-      data = config["language/attendees/#{cfg['name'].downcase}"]
-      cfg.update(data) if data
+      cfg.update(config["language/attendees/#{name.downcase}"] || {})
 
-      attendee = Attendee.const_get(cfg['name']).new(cfg, self)
-      @attendees << attendee
+      @attendees << attendee = Attendee.const_get(name).new(cfg, self)
 
-      cfg['in'].split(STRING_SEPARATOR_RE).each { |interest|
-        subscriber[interest] << attendee
-      }
-      cfg['out'].split(STRING_SEPARATOR_RE).each { |theme|
-        supplier[theme] << attendee
+      { 'in' => subscriber, 'out' => supplier }.each { |key, target|
+        cfg[key].split(SEP_RE).each { |ch| target[ch] << attendee }
       }
     }
 
-    supplier.each { |channel, attendees| attendees.each { |attendee|
-      attendee.add_subscriber(subscriber[channel])
+    supplier.each { |ch, attendees| attendees.each { |att|
+      att.add_subscriber(subscriber[ch])
     } }
   end
 
@@ -283,6 +321,7 @@ require_relative 'lingo/core_ext'
 require_relative 'lingo/cachable'
 require_relative 'lingo/reportable'
 require_relative 'lingo/agenda_item'
+require_relative 'lingo/show_progress'
 require_relative 'lingo/database'
 require_relative 'lingo/language'
 require_relative 'lingo/attendee'