lingo 1.8.6 → 1.8.7

data/dict/de/test_muh.txt

@@ -0,0 +1,6 @@
+albert einstein
+john f kennedy
+a priori
+ableitung nicht ganzzahliger ordnung
+academic learning time in physical education
+juristische person

data/dict/en/lingo-dic.txt

@@ -19913,7 +19913,7 @@ fodder=fodder #s|v
 foe=foe #s
 foederatus=foederatus #s
 foetal=foetal #a
-fœtid=fœtid #a
+foetid=foetid #a
 foetidness=foetidness #s
 foetus=foetus #s
 fog=fog #s|v
@@ -53175,7 +53175,6 @@ vedette=vedette #s
 veejay=veejay #s
 veel=veel #v
 veer=veer #s|v
-veg*n=veg*n #s|a
 vega=vega #s
 vegan=vegan #s|a
 veganism=veganism #s
@@ -55392,7 +55391,7 @@ zony=zony #s
 zoo=zoo #s
 zooarchaeology=zooarchaeology #s
 zoobie=zoobie #s
-zoœcium=zoœcium #s
+zooecium=zooecium #s
 zoogeography=zoogeography #s
 zoolater=zoolater #s
 zoological=zoological #a

data/lang/de.lang

@@ -56,12 +56,13 @@ language:
       usr-dic: { name: de/user-dic.txt,  txt-format: WordClass, separator: '=' }
 
       # Test dictionaries
-      tst-dic: { name: de/test_dic.txt, txt-format: WordClass }              # TEST: Lesen von zwei Quellen
-      tst-syn: { name: de/test_syn.txt, txt-format: MultiValue, def-wc: y }  # TEST: Mehrere Datenquellen
-      tst-mul: { name: de/test_mul.txt, use-lex: sys-dic, def-wc: m }        # TEST: Mehrere Multiwörterbücher
-      tst-mu2: { name: de/test_mu2.txt, use-lex: sys-dic, def-wc: m }        # TEST: Mehrere Multiwörterbücher
-      tst-sgw: { name: de/test_sgw.txt, txt-format: SingleWord }             # TEST: SingleWord-Format
-      tst-gen: { name: de/test_gen.txt, txt-format: WordClass }              # TEST: Genus
+      tst-dic: { name: de/test_dic.txt, txt-format: WordClass }
+      tst-syn: { name: de/test_syn.txt, txt-format: MultiValue, def-wc: y }
+      tst-mul: { name: de/test_mul.txt, use-lex: sys-dic, def-wc: m }
+      tst-mu2: { name: de/test_mu2.txt, use-lex: sys-dic, def-wc: m }
+      tst-muh: { name: de/test_muh.txt, txt-format: SingleWord, use-lex: sys-dic, def-wc: m, hyphenate: true }
+      tst-sgw: { name: de/test_sgw.txt, txt-format: SingleWord }
+      tst-gen: { name: de/test_gen.txt, txt-format: WordClass }
 
     compound:
       min-word-size:     '7'
@@ -118,8 +119,8 @@ language:
     #   SPAC = \s+
     #   NUMS = [+-]?(?:\d{4,}|\d{1,3}(?:\.\d{3,3})*)(?:\.|(?:,\d+)?%?)
     #   URLS = (?:www\.|mailto:|(?:news|https?|ftps?)://|\S+?[._]\S+?@\S+?\.)\S+
-    #   ABRV = (?:(?:(?:#{CHAR})+\.)+)(?:#{CHAR})+
-    #   WORD = (?:#{CHAR}|#{DIGIT}|-)+
+    #   ABRV = (?:(?:(?:CHAR)+\.)+)(?:CHAR)+
+    #   WORD = ALNUM(?:-*ALNUM)*
     #   PUNC = [!,.:;?¡¿]
-    #   OTHR = ["$#%&'()*+\-/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷]
+    #   OTHR = [-"$#%&'()*+\-/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷]
     #   HELP = \S*

data/lang/en.lang

@@ -69,7 +69,7 @@ language:
       # Suffixklasse: s = Substantiv, a = Adjektiv, v = Verb, e = Eigenwort, f = Fugung
       # Suffixe je Klasse: "<suffix>['/'<ersetzung>][ <suffix>['/'<ersetzung>]]"
       - [s, 'es s ves/f ves/fe ies/y']
-      - [a, 'er est r st ier/y iest/y']
+      - [a, 'er est r st ier/y iest/y ly al ally']
       - [v, 'd ed en es ing s ing/e']
       - [e, 's']
       - [f, '']

data/lib/lingo.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -61,7 +61,8 @@ class Lingo
   }
 
   # Default encoding
-  Encoding.default_external = Encoding.default_internal = ENC = 'UTF-8'.freeze
+  Encoding.default_external = ENC = 'UTF-8'.freeze
+  Encoding.default_internal = ENC unless RUBY_ENGINE == 'jruby'
 
   SEP_RE = %r{[; ,|]}
 
@@ -265,8 +266,7 @@ class Lingo
   end
 
   def invite(list = config['meeting/attendees'])
-    supplier   = Hash.nest { [] }
-    subscriber = Hash.nest { [] }
+    supplier, subscriber = Hash.array, Hash.array
 
     last_link, auto_link = '', 0
 

data/lib/lingo/attendee.rb

@@ -70,6 +70,8 @@ class Lingo
 
     include Language
 
+    TERMINALS = [:FILE, :RECORD, :EOF]
+
     DEFAULT_SKIP = [TA_PUNCTUATION, TA_OTHER].join(',')
 
     def initialize(config, lingo)
@@ -124,7 +126,15 @@ class Lingo
       @config.fetch(key, default)
     end
 
-    def get_array(key, default = nil, method = nil)
+    def get_int(*args)
+      Integer(get_key(*args))
+    end
+
+    def get_flo(*args)
+      ((val = get_key(*args)) && val.respond_to?(:to_f)) ? val.to_f : val
+    end
+
+    def get_ary(key, default = nil, method = nil)
       ary = get_key(key, default).split(SEP_RE)
       ary.map!(&method) if method
       ary
@@ -136,6 +146,12 @@ class Lingo
       end
     end
 
+    def get_enc(key = 'encoding', default = ENC)
+      Encoding.find(get_key(key, default))
+    rescue ArgumentError => err
+      raise ConfigLoadError.new(err)
+    end
+
     def dictionary(src, mod)
       Language::Dictionary.new({ 'source' => src, 'mode' => mod }, lingo)
     end
@@ -145,11 +161,11 @@ class Lingo
     end
 
     def set_dic
-      @dic = dictionary(get_array('source'), get_key('mode', 'all'))
+      @dic = dictionary(get_ary('source'), get_key('mode', 'all'))
     end
 
     def set_gra
-      @gra = grammar(get_array('source'), get_key('mode', 'all'))
+      @gra = grammar(get_ary('source'), get_key('mode', 'all'))
     end
 
     def warn(*msg)
@@ -166,23 +182,27 @@ class Lingo
 
 end
 
+require_relative 'text_utils'
+
 require_relative 'buffered_attendee'
 require_relative 'deferred_attendee'
 
 require_relative 'attendee/abbreviator'
+require_relative 'attendee/analysis_filter'
 require_relative 'attendee/debugger'
+require_relative 'attendee/debug_filter'  # < Debugger
 require_relative 'attendee/decomposer'
-require_relative 'attendee/dehyphenizer'
+require_relative 'attendee/hal_filter'
+require_relative 'attendee/lsi_filter'
 require_relative 'attendee/multi_worder'
-require_relative 'attendee/noneword_filter'
 require_relative 'attendee/object_filter'
-require_relative 'attendee/variator'
 require_relative 'attendee/sequencer'
 require_relative 'attendee/stemmer'
 require_relative 'attendee/synonymer'
 require_relative 'attendee/text_reader'
 require_relative 'attendee/text_writer'
-require_relative 'attendee/formatter'
+require_relative 'attendee/formatter'  # < TextWriter
 require_relative 'attendee/tokenizer'
+require_relative 'attendee/variator'
 require_relative 'attendee/vector_filter'
 require_relative 'attendee/word_searcher'

data/lib/lingo/attendee/analysis_filter.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'csv'
+
+class Lingo
+
+  class Attendee
+
+    class AnalysisFilter < self
+
+      FIELDS = {
+        string:   :form,
+        token:    :attr,
+        position: :position,
+        offset:   :offset,
+        word:     :attr,
+        pattern:  :pattern
+      }
+
+      def init
+        @csv, @header = CSV.new('', row_sep: ''), FIELDS.keys
+      end
+
+      def control(cmd, *)
+        :skip_command if cmd == :EOL
+      end
+
+      def process(obj, *)
+        forward_row(@header.tap { @header = nil }) if @header
+
+        obj.is_a?(Token) ?
+          forward_obj(obj, obj, obj, obj) : begin
+        tok = obj.token
+          forward_obj(obj, nil, tok, tok, obj, obj)
+        obj.lexicals.each { |lex|
+          forward_obj(lex, nil, tok, tok, lex, obj) }
+        end
+      end
+
+      private
+
+      def forward_obj(*args)
+        forward_row(FIELDS.map.with_index { |(_, method), index|
+          arg = args[index] and arg.send(method) })
+      end
+
+      def forward_row(row)
+        forward(@csv.add_row(row).string.dup)
+        @csv.string.clear
+        @csv.rewind
+      end
+
+    end
+
+  end
+
+end

data/lib/lingo/attendee/debug_filter.rb

@@ -0,0 +1,42 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  class Attendee
+
+    class DebugFilter < Debugger
+
+      def init
+        @filter = true
+        super('')
+      end
+
+    end
+
+  end
+
+end

data/lib/lingo/attendee/debugger.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -112,7 +112,7 @@ class Lingo
         end
       end
 
-      def process(obj)
+      def process(obj, *)
         debug(eval(@obj_eval)) { obj.inspect }
         forward(obj) unless @filter
       end
@@ -130,15 +130,6 @@ class Lingo
 
     end
 
-    class DebugFilter < Debugger
-
-      def init
-        @filter = true
-        super('')
-      end
-
-    end
-
   end
 
 end

data/lib/lingo/attendee/decomposer.rb

@@ -40,7 +40,8 @@ class Lingo
     #
     # === Mögliche Verlinkung
     # Erwartet:: Daten vom Typ *Word* (andere werden einfach durchgereicht) z.B. von Wordsearcher
-    # Erzeugt:: Daten vom Typ *Word* (erkannte Komposita werden entsprechend erweitert) z.B. für Synonymer, Ocr_variator, Multiworder, Sequencer, Noneword_filter, Vector_filter
+    # Erzeugt:: Daten vom Typ *Word* (erkannte Komposita werden entsprechend erweitert) z.B. für
+    # Synonymer, Ocr_variator, Multiworder, Sequencer, Vector_filter
     #
     # === Parameter
     # Kursiv dargestellte Parameter sind optional (ggf. mit Angabe der Voreinstellung).
@@ -65,9 +66,9 @@ class Lingo
     #   out> <Lingo|?>
     #   out> :,/PUNC:
     #   out> <ein = [(ein/w)]>
-    #   out> <Indexierungssystem|KOM = [(indexierungssystem/k), (indexierung/s), (system/s)]>
+    #   out> <Indexierungssystem|COM = [(indexierungssystem/k), (indexierung/s), (system/s)]>
     #   out> <mit = [(mit/w)]>
-    #   out> <Kompositumerkennung|KOM = [(kompositumerkennung/k), (erkennung/s), (kompositum/s)]>
+    #   out> <Kompositumerkennung|COM = [(kompositumerkennung/k), (erkennung/s), (kompositum/s)]>
     #   out> :./PUNC:
     #   out> *EOL('test.txt')
     #   out> *EOF('test.txt')
@@ -85,6 +86,8 @@ class Lingo
       def process(obj)
         if obj.is_a?(Word) && obj.unknown?
           com = @gra.find_compound(obj.form)
+          com.token = obj.token
+
           obj = com unless com.unknown?
         end
 

data/lib/lingo/attendee/formatter.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -35,7 +35,7 @@ class Lingo
 
         @ext    = get_key('ext', '-')
         @format = get_key('format', '%s')
-        @map    = get_key('map', Hash.nest { |k| k })
+        @map    = get_key('map', Hash.identity)
 
         @no_puts = true
       end
@@ -44,10 +44,10 @@ class Lingo
         if obj.is_a?(WordForm)
           str = obj.form
 
-          if obj.respond_to?(:lexicals)
-            lex = obj.lexicals.first  # TODO
-            att = @map[lex.attr] if lex
-            str = @format % [str, lex.form, att] if att
+          if obj.is_a?(Word)
+            # TODO: which lexical to select? (currently: first)
+            obj.each_lex { |lex|
+              att = @map[lex.attr] and str = @format % [str, lex.form, att] }
           end
         else
           str = obj.to_s

data/lib/lingo/attendee/hal_filter.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  class Attendee
+
+    class HalFilter < self
+
+      def init
+        require_lib('hal4r')
+
+        @lex  = get_re('lexicals', '[sy]')
+        @skip = get_ary('skip', DEFAULT_SKIP, :upcase)
+
+        @norm = get_key('norm', true)
+        @sep  = get_key('sep', '^')
+        @min  = get_flo('min', false)
+        @dim  = get_int('dim', 2)
+
+        @sort = get_key('sort', false)
+        @sort.downcase! if @sort.respond_to?(:downcase!)
+
+        @hal = Hal4R.new([], get_int('window-size', Hal4R::DEFAULT_WINDOW_SIZE))
+      end
+
+      def control(cmd, *)
+        case cmd
+          when :EOL       then :skip_command
+          when *TERMINALS then send_vectors unless @hal.empty?
+        end
+      end
+
+      def process(obj)
+        obj.is_a?(Word) && !@skip.include?(obj.attr) &&
+          # TODO: which lexical to select? (currently: first)
+          obj.lex_form(@lex) { |form| @hal << Unicode.downcase(form) }
+      end
+
+      private
+
+      def send_vectors
+        vec = []
+
+        fmt = @sort ? @sort == 'sto' ?
+          '%s {%.5f}' : '%2$.5f %1$s' : '%s %.5f' unless @sort == 'normal'
+
+        unless @sort
+          each_vector { |v| forward(fmt % v) }
+        else
+          each_vector { |v| vec << v }
+
+          !fmt ? vec.sort!.each { |v, _| forward(v) } :
+            vec.sort_by { |v, w| [-w, v] }.each { |v| forward(fmt % v) }
+
+          vec.clear
+        end
+
+        @hal.reset
+      end
+
+      def each_vector
+        @hal.each_distance(@norm, @dim) { |*t, v| v = 1 / v
+          yield [t.join(@sep), v] unless v.nan? || (@min && v < @min) }
+      end
+
+    end
+
+  end
+
+end