lingo 1.8.5 → 1.8.6

data/lib/lingo/attendee/variator.rb

@@ -28,6 +28,7 @@ class Lingo
 
   class Attendee
 
+    #--
     # Der Variator ermöglicht bei nicht erkannten Wörtern den listenbasierten
     # Austausch einzelner Wortteile einchließlich erneuter Wörterbuchsuche zur
     # Verbesserung der Worterkennungsquote.
@@ -69,11 +70,10 @@ class Lingo
     #   out> :./PUNC:
     #   out> *EOL('test.txt')
     #   out> *EOF('test.txt')
+    #++
 
     class Variator < self
 
-      protected
-
       def init
         @marker = get_key('marker', '*')
         @max    = get_key('max-var', max = 10000).to_i
@@ -89,8 +89,7 @@ class Lingo
         set_gra
       end
 
-      def control(cmd, param)
-        # can control
+      def control(*)
       end
 
       def process(obj)
@@ -118,9 +117,6 @@ class Lingo
 
       private
 
-      # Variiere die Bestandteile eines Arrays gemäß den Austauschvorgaben.
-      #
-      # variate( 'Tiieh', 'ieh', 'sch' ) => ['Tiieh', 'Tisch']
       def variate(variations, from, to)
         add, change, re = [], [from, to], Regexp.new(from)
 

data/lib/lingo/attendee/vector_filter.rb

@@ -24,10 +24,13 @@
 ###############################################################################
 #++
 
+require 'csv'
+
 class Lingo
 
   class Attendee
 
+    #--
     # Die Hauptaufgabe des VectorFilter ist die Erstellung eines Dokumenten-Index-Vektor.
     # Dabei werden die durch die anderen Attendees ermittelten Grundformen eines Wortes
     # gespeichert und bei einem Datei- oder Record-Wechsel weitergeleitet. Der VectorFilter
@@ -76,101 +79,165 @@ class Lingo
     #   out> *FILE('test.txt')
     #   out> "0.28571 indexierung"
     #   out> *EOF('test.txt')
+    #++
 
     class VectorFilter < self
 
-      DEFAULT_SRC_SEP = '|'
+      DEFAULT_SRC_SEPARATOR = '|'
+      DEFAULT_POS_SEPARATOR = '@'
+
+      DEFAULT_DICT_SEPARATOR = Database::Source::WordClass::DEFAULT_SEPARATOR
+
+      DEFAULT_GENDER_SEPARATOR = Database::Source::WordClass::GENDER_SEPARATOR
 
-      protected
+      TERMINALS = [:FILE, :RECORD, :EOF]
 
       def init
-        if @debug = get_key('debug', false)
-          @prompt = get_key('prompt', 'lex:) ')
-          @preamble = get_key('preamble', true)
-        else
-          @lex  = get_re('lexicals', '[sy]')
-          @skip = get_array('skip', DEFAULT_SKIP, :upcase)
+        @lex  = get_re('lexicals', '[sy]')
+        @skip = get_array('skip', DEFAULT_SKIP, :upcase)
+
+        @src = @pos = @sort_fmt = @sort_rel = @docnum = nil
 
-          @dict = get_key('dict', false)
-          @norm = get_key('norm', false) if @dict
-          @dict = Database::Source::WordClass::DEFAULT_SEPARATOR if @dict == true
+        @tokens, @vectors, @word_count = [], Hash.nest(1) { [] }, Hash.new(0)
 
+        if @dict = get_key('dict', false)
+          @norm = get_key('norm', false)
+          @dict = DEFAULT_DICT_SEPARATOR if @dict == true
+        else
           @src = get_key('src', false)
-          @src = DEFAULT_SRC_SEP if @src == true
+          @src = DEFAULT_SRC_SEPARATOR if @src == true
 
-          if sort = get_key('sort', ENV['LINGO_NO_SORT'] ? false : 'normal')
-            @sort_format, @sort_method = sort.downcase.split('_', 2)
-          end
+          @pos = get_key('pos', false)
+          @pos = DEFAULT_POS_SEPARATOR if @pos == true
+
+          @tokens = get_array('tokens', '', :upcase)
+          @tokens.concat(Tokenizer.rules) if @tokens.delete('ALL')
         end
 
-        @vectors, @word_count = [], 0
+        if sort = get_key('sort', ENV['LINGO_NO_SORT'] ? false : 'normal')
+          @sort_fmt, sort_method = sort.downcase.split('_', 2)
+
+          @sort_rel = rel = sort_method == 'rel'
+
+          unless @sort_fmt == 'normal'
+            if @tfidf = get_key('tfidf', false)
+              DeferredAttendee.enhance(self)
+              @docnum, rel = 0, true
+            end
+
+            _sort_fmt = @sort_fmt == 'sto' ? '%2$s {%1$X}' : '%X %s'
+            @sort_fmt = _sort_fmt.sub('X', rel ? '.5f' : 'd')
+          end
+        end
       end
 
-      def control(cmd, param)
+      def control(cmd, *)
         case cmd
-          when STR_CMD_EOL
-            skip_command
-          when STR_CMD_FILE, STR_CMD_RECORD, STR_CMD_EOF
-            send_vectors unless @vectors.empty?
+          when :EOL       then :skip_command
+          when *TERMINALS then send_vectors unless @docnum
         end
       end
 
+      def control_deferred(cmd, *)
+        @docnum += 1 if TERMINALS.include?(cmd)
+      end
+
       def process(obj)
-        if @debug
-          forward((@preamble = nil; @lingo.config.to_h.to_yaml)) if @preamble
-          forward("#{@prompt} #{obj.inspect}") if eval(@debug)
-        elsif obj.is_a?(Word) && !@skip.include?(obj.attr)
-          @word_count += 1
-
-          if @dict
-            vec, sep = [], Database::Source::WordClass::GENDER_SEPARATOR
-
-            obj.get_class(@lex).each { |lex|
-              str = "#{lex.form} ##{lex.attr}"
-              str << sep << lex.gender if lex.gender
-              vec << str
-            }
-
-            unless vec.empty?
-              wrd = @norm ? obj.lexicals.first.form : obj.form
-              vec = Unicode.downcase("#{wrd}#{@dict}#{vec.join(' ')}")
-              @sort_format ? @vectors << vec : forward(vec)
-            end
-          else
-            obj.get_class(@lex).each { |lex|
-              vec = Unicode.downcase(lex.form)
-              vec << @src << lex.src if @src && lex.src
-              @sort_format ? @vectors << vec : forward(vec)
-            }
-          end
+        if obj.is_a?(Token)
+          return unless @tokens.include?(obj.attr)
+        elsif obj.is_a?(Word)
+          return if @skip.include?(obj.attr)
+        else
+          return
+        end
+
+        @word_count[@docnum] += 1
+
+        @dict ? forward_dict(obj) : begin
+          pos = obj.position_and_offset if @pos
+
+          obj.is_a?(Token) ? forward_vector(obj, pos) :
+            obj.get_class(@lex).each { |lex| forward_vector(lex, pos, lex.src) }
         end
       end
 
       private
 
-      def send_vectors
-        if @sort_format == 'normal'
-          @vectors.uniq!
-          flush(@vectors.sort!)
-        else
-          cnt, fmt = Hash.new(0), '%d'
+      def vectors(docnum = nil)
+        @vectors[docnum || @docnum]
+      end
+
+      def word_count(docnum = nil)
+        @word_count[docnum || @docnum]
+      end
 
-          @vectors.each { |v| cnt[v] += 1 }.clear
-          vec = cnt.sort_by { |v, c| [-c, v] }
+      def forward_dict(obj, sep = DEFAULT_GENDER_SEPARATOR)
+        vectors = obj.get_class(@lex).map { |lex|
+          "#{lex.form} ##{lex.attr}".tap { |str|
+            str << sep << lex.gender if lex.gender
+          }
+        }
 
-          if @sort_method == 'rel'
-            fmt, wc = '%6.5f', @word_count.to_f
-            vec.each { |v| v[1] /= wc }
-          end
+        unless vectors.empty?
+          vec = @norm ? obj.lexicals.first.form : obj.form
+          forward_vector("#{vec}#{@dict}#{vectors.join(' ')}")
+        end
+      end
+
+      def forward_vector(vec, pos = nil, src = nil)
+        vec = vec.form if vec.is_a?(WordForm)
+
+        vec = Unicode.downcase(vec)
+        vec << @src << src if @src && src
+
+        @sort_fmt ? vectors[vec] << pos : forward(vec_pos(vec, [pos]))
+      end
+
+      def send_vectors
+        if @docnum
+          df, abs = Hash.new(0), @sort_rel ? nil : 1
+
+          @vectors.each_value { |w| w.each_key { |v| df[v] += 1 } }
 
-          if @sort_format == 'sto'
-            fmt, @word_count = "%s {#{fmt}}", 0
-          else
-            fmt.insert(1, '2$') << ' %1$s'
+          if @tfidf.is_a?(String)
+            CSV.open(@tfidf, 'wb') { |c| df.sort.each { |v| c << v } }
           end
 
-          vec.each { |v| forward(fmt % v) }
+          yield lambda { |docnum|
+            wc = abs || word_count(docnum)
+            flush_vectors(wc, docnum) { |c, v, vp| [c / df[v], vp] }
+          }
+        elsif @sort_fmt == 'normal'
+          flush(map_vectors { |_, _, vp| vp }.sort!)
+        else
+          flush_vectors(@sort_rel ? word_count : 1) { |c, _, vp| [c, vp] }
         end
+
+        @word_count.clear
+        @vectors.clear
+      end
+
+      alias_method :flush_deferred, :send_vectors
+
+      def map_vectors(wc = 1, docnum = nil)
+        v = vectors(docnum)
+        v.map { |vec, pos| yield pos.size / wc.to_f, vec, vec_pos(vec, pos) }
+      ensure
+        v.clear if v
+      end
+
+      def flush_vectors(*args, &block)
+        map_vectors(*args, &block)
+          .sort_by { |w, v| [-w, v] }
+          .each { |vec| forward(@sort_fmt % vec) }
+      end
+
+      def vec_pos(vec, pos)
+        pos.clear unless @pos
+
+        pos.compact!
+        pos.uniq!
+        pos.empty? ? vec : "#{vec}#{@pos}#{pos.join(',')}"
       end
 
     end

data/lib/lingo/attendee/word_searcher.rb

@@ -28,6 +28,7 @@ class Lingo
 
   class Attendee
 
+    #--
     # Der WordSearcher ist das Herzstück von Lingo. Er macht die Hauptarbeit und versucht
     # alle Token die nach einem sinnvollen Wort aussehen, in den ihm angegebenen
     # Wörterbüchern zu finden und aufzulösen. Dabei werden die im Wörterbuch gefundenen
@@ -64,6 +65,7 @@ class Lingo
     #   out> :./PUNC:
     #   out> *EOL('test.txt')
     #   out> *EOF('test.txt')
+    #++
 
     class WordSearcher < self
 
@@ -71,12 +73,12 @@ class Lingo
         set_dic
       end
 
-      def control(cmd, param)
-        # can control
+      def control(*)
       end
 
       def process(obj)
-        forward(obj.is_a?(Token) && obj.word? ? @dic.find_word(obj.form) : obj)
+        forward(obj.is_a?(Token) && obj.word? ?
+          @dic.find_word(obj.form, obj) : obj)
       end
 
     end

data/lib/lingo/buffered_attendee.rb

@@ -33,8 +33,6 @@ class Lingo
       super
     end
 
-    protected
-
     def process(obj)
       @buffer << obj
       process_buffer if process_buffer?
@@ -70,7 +68,7 @@ class Lingo
     end
 
     def control_multi(cmd)
-      if [STR_CMD_RECORD, STR_CMD_EOF].include?(cmd)
+      if [:RECORD, :EOF].include?(cmd)
         @eof_handling = true
 
         while valid_tokens_in_buffer > 1

data/lib/lingo/call.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -48,8 +48,9 @@ class Lingo
       end
     end
 
-    def talk(str, raw = false)
-      config.stdin.reopen(str)
+    def talk(input, raw = false)
+      config.stdin.reopen(
+        input.respond_to?(:read) ? input.read : input)
 
       start
 

data/lib/lingo/cli.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -50,6 +50,10 @@ class Lingo
 
     private
 
+    def config_present?(config)
+      Lingo.find(:config, config) {}
+    end
+
     def load_config(*)
       @config = {}
     end

data/lib/lingo/config.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -32,7 +32,7 @@ class Lingo
   class Config
 
     def initialize(*args)
-      @deprecated = Hash.new { |h, k| h[k] = true; false }
+      @deprecated = Hash.seen
 
       @cli, @opts = CLI.new, {}
 
@@ -42,7 +42,9 @@ class Lingo
       load_config('language', :lang)
       load_config('config')
 
-      deprecate(:textreader, :text_reader) if Array(self['meeting/attendees']).map(&:keys).flatten.include?('textreader')
+      if Array(self['meeting/attendees']).flat_map(&:keys).include?('textreader')
+        deprecate(:textreader, :text_reader)
+      end
 
       if r = get('meeting/attendees', 'text_reader') ||
              get('meeting/attendees', 'textreader')  # DEPRECATE textreader
@@ -56,6 +58,8 @@ class Lingo
       end
     end
 
+    attr_reader :language_file, :config_file
+
     def to_h
       @opts
     end
@@ -106,10 +110,10 @@ class Lingo
       @cli.send(:quit, *args)
     end
 
-    def deprecate(old, new, obj = self)
+    def deprecate(old, new, obj = self, what = :option)
       unless @deprecated[[source = obj.class.name.sub(/\ALingo::/, ''), old]]
         warn(
-          "DEPRECATION WARNING: #{source} option `#{old}' is deprecated " <<
+          "DEPRECATION WARNING: #{source} #{what} `#{old}' is deprecated " <<
           "and will be removed in Lingo 1.9. Please use `#{new}' instead."
         )
       end
@@ -127,6 +131,8 @@ class Lingo
 
     def load_config(key, type = key.to_sym)
       file = Lingo.find(type, @opts[key]) { quit }
+      instance_variable_set("@#{type}_file", file)
+
       File.open(file, encoding: ENC) { |f| @opts.update(SafeYAML.load(f)) }
     rescue Psych::SyntaxError => err
       err.message << " (in #{file})"

data/lib/lingo/ctl.rb

@@ -38,8 +38,8 @@ class Lingo
     PROG, VERSION, OPTWIDTH = $0, '0.0.2', 18
     PROGNAME, OPTIONS = File.basename(PROG), {}
 
-    COMMANDS, ALIASES = {}, Hash.new { |h, k|
-      h[k] = COMMANDS.key?(k) ? k : 'usage'
+    COMMANDS, ALIASES = {}, Hash.nest { |k|
+      COMMANDS.key?(k) ? k : 'usage'
     }
 
     USAGE = <<-EOT
@@ -187,7 +187,7 @@ Usage: #{PROG} <command> [arguments] [options]
 
       msg = opts ? [opts, 'Commands:'] : []
 
-      aliases = Hash.new { |h, k| h[k] = [] }
+      aliases = Hash.nest { [] }
       ALIASES.each { |k, v| aliases[v] << k }
 
       COMMANDS.each { |c, (d, *e)|