lingo 1.9.0.pre1 → 1.9.0

data/lang/en.lang

@@ -68,9 +68,9 @@ language:
       # Suffixliste, Stand: *****ENGLISCH****
       # Suffixklasse: s = Substantiv, a = Adjektiv, v = Verb, e = Eigenwort, f = Fugung
       # Suffixe je Klasse: "<suffix>['/'<ersetzung>][ <suffix>['/'<ersetzung>]]"
-      - [s, 'es s ves/f ves/fe ies/y']
+      - [s, 'es es/is s ves/f ves/fe ies/y']
       - [a, 'er est r st ier/y iest/y ly al ally']
-      - [v, 'd ed en es ing s ing/e']
+      - [v, 'd ed en es ies/y ing s ing/e']
       - [e, 's']
       - [f, '']
 

data/lib/lingo.rb

@@ -32,6 +32,7 @@ require 'nuggets/file/ext'
 require 'nuggets/hash/nest'
 require 'nuggets/hash/seen'
 require 'nuggets/env/user_home'
+require 'nuggets/object/silence'
 require 'nuggets/string/camelscore'
 
 class Lingo
@@ -61,8 +62,12 @@ class Lingo
   }
 
   # Default encoding
-  Encoding.default_external = ENC = 'UTF-8'.freeze
-  Encoding.default_internal = ENC unless RUBY_ENGINE == 'jruby'
+  ENCODING = 'UTF-8'.freeze
+
+  silence {
+    Encoding.default_external = ENCODING
+    Encoding.default_internal = ENCODING unless RUBY_ENGINE == 'jruby'
+  }
 
   SEP_RE = %r{[; ,|]}
 
@@ -298,6 +303,11 @@ class Lingo
     } }
   end
 
+  def attendees(arg = Object)
+    @attendees.grep(arg.is_a?(Class) ? arg :
+      Attendee.const_get(arg.to_s.camelcase))
+  end
+
   def start
     @attendees.first.control(:TALK)
   end
@@ -319,9 +329,11 @@ require_relative 'lingo/error'
 require_relative 'lingo/debug'
 require_relative 'lingo/config'
 require_relative 'lingo/filter'
+require_relative 'lingo/array_utils'
+require_relative 'lingo/text_utils'
+require_relative 'lingo/language'
 require_relative 'lingo/progress'
 require_relative 'lingo/database'
-require_relative 'lingo/language'
 require_relative 'lingo/attendee'
 require_relative 'lingo/version'
 

data/lib/lingo/array_utils.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  module ArrayUtils
+
+    private
+
+    def combinations(first, *rest, &block)
+      first.product(*rest, &block)
+    end
+
+  end
+
+end

data/lib/lingo/attendee.rb

@@ -146,7 +146,7 @@ class Lingo
       end
     end
 
-    def get_enc(key = 'encoding', default = ENC)
+    def get_enc(key = 'encoding', default = ENCODING)
       Encoding.find(get_key(key, default))
     rescue ArgumentError => err
       raise ConfigLoadError.new(err)
@@ -182,8 +182,6 @@ class Lingo
 
 end
 
-require_relative 'text_utils'
-
 require_relative 'buffered_attendee'
 require_relative 'deferred_attendee'
 

data/lib/lingo/attendee/multi_worder.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -79,6 +79,8 @@ class Lingo
 
     class MultiWorder < BufferedAttendee
 
+      include ArrayUtils
+
       def init
         # combine lexical variants?
         #
@@ -220,7 +222,7 @@ class Lingo
         }
 
         if @combine
-          seq.shift.product(*seq) { |key|
+          combinations(*seq) { |key|
             @mul_dic.select(key.join(sep), mul)
             break unless @all || mul.empty?
           } && mul.uniq!

data/lib/lingo/attendee/sequencer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -97,23 +97,53 @@ class Lingo
 
     class Sequencer < BufferedAttendee
 
+      include ArrayUtils
+
       UNK = %w[#]
       NUM = %w[0]
 
-      CLS = /[[:alpha:]#{NUM.join}]/o
+      CLASS_RE = %r{[a-z#{NUM.join}]}o
+
+      REGEX_RE = %r{
+        ( #{CLASS_RE}+ )
+        |
+        \[
+          ( #{CLASS_RE}+ )
+        \]
+        |
+        \(
+          (?: \?: )?
+          ( #{CLASS_RE}+ (?: \| #{CLASS_RE}+ )* )
+        \)
+      }xo
+
+      FULL_CLASS_RE = %r{\A(?:#{CLASS_RE})+\z}o
+      FULL_REGEX_RE = %r{\A(?:#{REGEX_RE})+\z}o
 
       def init
         @stopper = get_ary('stopper', DEFAULT_SKIP)
                      .push(WA_UNKNOWN, WA_UNKMULPART)
 
         @mwc = get_key('multiword', LA_MULTIWORD)
-        @cls = []
 
-        @seq = get_key('sequences').map { |str, fmt|
-          @cls.concat(cls = (str = str.downcase).scan(CLS))
+        @cls, @seq = [], []
+
+        get_key('sequences').each { |str, fmt|
+          seq, fmt = lambda { |*a| @seq << (a << fmt) },
+            fmt == true ? '|' : fmt ? fmt.gsub(/\d+/, '%\&$s') : nil
+
+          @cls.concat(cls = (str = str.downcase).scan(CLASS_RE))
 
-          (str =~ /\W/ ? [Regexp.new(str), nil] : [str, cls]).push(
-            fmt == true ? '|' : fmt ? fmt.gsub(/\d+/, '%\&$s') : nil)
+          case str
+            when FULL_CLASS_RE then seq[str, cls]
+            when FULL_REGEX_RE then m = []
+              str.scan(REGEX_RE) { |m1, m2, m3|
+                m1 ? m1.each_char { |c| m << [c] } : m << (
+                m2 ? m2.chars : m3.split('|').map(&:chars)) }
+
+              combinations(*m) { |q| seq[q.join, q.flatten] }
+            else seq[Regexp.new(str), nil]
+          end
         }
 
         @cls.uniq!
@@ -130,90 +160,109 @@ class Lingo
       end
 
       def process_buffer
-        flush(@buffer.size < 2 ? @buffer : begin
-          arg = [[], buf = [], map = [], @seq]
-
-          iter, skip, rewind = @buffer.each_with_index, 0, lambda {
-            iter.rewind; skip.times { iter.next }; skip = 0
-          }
-
-          loop {
-            obj, idx = begin
-              iter.next
-            rescue StopIteration
-              raise unless skip > 0
-
-              buf.slice!(0, skip)
-              map.slice!(0, skip)
-
-              rewind.call
-            end
-
-            att = (tok = obj.is_a?(Token)) ? obj.number? ? NUM : UNK :
-              obj.is_a?(Word) && !obj.unknown? ? obj.compound_attrs : UNK
-
-            if (att &= @cls).empty?
-              find_seq(*arg)
-              rewind.call if skip > 0
-            else
-              obj.each_lex(@mwc) { |lex|
-                lex.form.count(' ').succ.times { iter.next }
-                break skip = idx + 1
-              } unless tok
-
-              buf << obj
-              map << att
-            end
-          }
-
-          @buffer.concat(find_seq(*arg))
-        end)
+        process_seq if @buffer.size > 1
+        flush(@buffer)
       end
 
       private
 
-      def find_seq(mat, buf, map, seq)
-        return mat if buf.empty?
+      def process_seq
+        buf, map = [], []
 
-        forms, args = [], []
+        iter, skip, rewind = @buffer.each_with_index, 0, lambda {
+          iter.rewind; skip.times { iter.next }; skip = 0
+        }
 
-        map.replace(map.shift.product(*map)).map! { |i| i.join }.uniq!
+        loop {
+          obj, idx = begin
+            iter.next
+          rescue StopIteration
+            raise unless skip > 0
+
+            buf.slice!(0, skip)
+            map.slice!(0, skip)
+
+            rewind.call
+          end
+
+          att = (tok = obj.is_a?(Token)) ? obj.number? ? NUM : UNK :
+            obj.is_a?(Word) && !obj.unknown? ? obj.compound_attrs : UNK
+
+          if (att &= @cls).empty?
+            find_seq(buf, map)
+            rewind.call if skip > 0
+          else
+            obj.each_lex(@mwc) { |lex|
+              lex.form.count(' ').succ.times { iter.next }
+              break skip = idx + 1
+            } unless tok
+
+            buf << obj
+            map << att
+          end
+        }
 
-        map.each { |q|
-          seq.each { |str, cls, fmt|
-            _str, _cls = [str, cls]
+        find_seq(buf, map)
+      end
 
-            while pos = q.index(str, pos || 0)
-              _str, _cls = [$&, $&.chars] unless cls
+      def find_seq(buf, map)
+        return if buf.empty?
 
-              _tok = nil; args.clear
+        objs, args = [], []
 
-              _cls.each_with_index { |wc, i|
-                obj = buf[pos + i];_tok ||= obj.token
+        @seq.each { |str, cls, fmt|
+          if cls
+            len = cls.size
 
-                args[i] = obj.is_a?(Word) ? obj.lexicals.find { |lex|
-                  break lex.form if lex.attr == wc } : obj.form or break
-              } or next
+            buf.each_cons(len).zip(map.each_cons(len)) { |_buf, _map|
+              obj = _buf.each; objs.clear; args.clear
 
-              forms << [_str, _tok,
-                fmt =~ /\d/ ? fmt.gsub('%0$s', _str) % args :
-                fmt ? "#{_str}:#{args.join(fmt)}" : args.join(' ')]
+              next if _map.zip(cls) { |_wc, wc|
+                break true unless _wc.include?(wc) &&
+                  find_form(obj.next, wc, objs, args)
+              }
 
-              pos += 1
-            end
-          }
-        }.clear
+              forward_seq(fmt, str, objs, args)
+            }
+          else
+            combinations(*map) { |q|
+              q, pos = q.join, -1
 
-        forms.uniq!
+              while pos = q.index(str, pos += 1)
+                objs.clear; args.clear
 
-        forms.each { |s, t, f|
-          wrd = Word.new_lexical(f, WA_SEQUENCE, LA_SEQUENCE)
-          wrd.pattern, wrd.token = s, t
-          mat << wrd
+                next unless $&.each_char.with_index { |wc, i|
+                  find_form(buf[pos + i], wc, objs, args) or break
+                }
+
+                forward_seq(fmt, $&, objs, args)
+              end
+            }
+          end
         }
 
         buf.clear
-        mat
+        map.clear
+      end
+
+      def find_form(obj, wc, objs, args)
+        form = obj.is_a?(Word) ? obj.lexicals.find { |lex|
+          break lex.form if lex.attr == wc } : obj.form or return
+
+        objs << obj
+        args << form
+      end
+
+      def forward_seq(fmt, str, objs, args)
+        wrd_form, form = objs.map(&:form).join(' '),
+          fmt =~ /\d/ ? fmt.gsub('%0$s', str) % args :
+          fmt ? "#{str}:#{args.join(fmt)}" : args.join(' ')
+
+        wrd = Word.new(wrd_form, WA_SEQUENCE)
+        wrd << Lexical.new(form, LA_SEQUENCE)
+        wrd.pattern, wrd.token = str, objs.first.token
+
+        @buffer << wrd
       end
 
     end

data/lib/lingo/attendee/text_writer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -99,10 +99,8 @@ class Lingo
           when :LIR
             @lir = true unless @lir.nil?
           when :FILE
-            @no_sep = true
-
-            @io = stdout?(@ext) ? (@path = @ext; open_stdout) :
-              open_path(@path = set_ext(param, @ext), 'w')
+            @no_sep, @io = true, (@stdout = stdout?(@ext)) ?
+              open_stdout : open_path(get_path(param, @ext), 'w')
 
             @lir_rec_no, @lir_rec_buf = '', []
           when :RECORD
@@ -117,7 +115,7 @@ class Lingo
             @io.puts unless @lir || @no_puts
           when :EOF
             flush_lir_buffer if @lir
-            @io.close unless stdout?(@path)
+            @io.close unless @stdout
         end
       end
 

data/lib/lingo/attendee/vector_filter.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -24,8 +24,6 @@
 ###############################################################################
 #++
 
-require 'csv'
-
 class Lingo
 
   class Attendee
@@ -83,6 +81,8 @@ class Lingo
 
     class VectorFilter < self
 
+      include TextUtils
+
       DEFAULT_SRC_SEPARATOR = '|'
       DEFAULT_POS_SEPARATOR = '@'
 
@@ -186,7 +186,7 @@ class Lingo
         vec = vec.form if vec.is_a?(WordForm)
 
         vec = Unicode.downcase(vec)
-        vec << @src << src if @src && src
+        vec << @src << src.form if @src && src
 
         @sort_fmt ? vectors[vec] << pos : forward(vec_pos(vec, [pos]))
       end
@@ -198,7 +198,7 @@ class Lingo
           @vectors.each_value { |w| w.each_key { |v| df[v] += 1 } }
 
           if @tfidf.is_a?(String)
-            CSV.open(@tfidf, 'wb') { |c| df.sort.each { |v| c << v } }
+            open_csv(@tfidf, 'wb') { |c| df.sort.each { |v| c << v } }
           end
 
           yield lambda { |docnum|