lingo 1.8.1 → 1.8.2

data/lib/lingo/attendee/vector_filter.rb

@@ -82,74 +82,73 @@ class Lingo
       protected
 
       def init
-        @lexis = Regexp.new(get_key('lexicals', '[sy]').downcase)
-        @sort = get_key('sort', 'normal').downcase
-        @skip = get_array('skip', TA_PUNCTUATION+','+TA_OTHER).collect {|s| s.upcase }
-        @vectors = Array.new
-        @word_count = 0
-
         if @debug = get_key('debug', false)
           @prompt = get_key('prompt', 'lex:) ')
+        else
+          @lex  = Regexp.new(get_key('lexicals', '[sy]').downcase)
+          @skip = get_array('skip', DEFAULT_SKIP, :upcase)
+
+          if sort = get_key('sort', 'normal')
+            @sort_format, @sort_method = sort.downcase.split('_', 2)
+          end
         end
+
+        @vectors, @word_count = [], 0.0
       end
 
-      def control(cmd, par)
+      def control(cmd, param)
         case cmd
           when STR_CMD_EOL
             skip_command
           when STR_CMD_FILE, STR_CMD_RECORD, STR_CMD_EOF
-            @debug ? @vectors.each(&method(:forward)) : sendVector
-            @vectors.clear
+            send_vectors unless @vectors.empty?
         end
       end
 
       def process(obj)
         if @debug
-          @vectors << "#{@prompt} #{obj.inspect}" if eval(@debug)
-        elsif obj.is_a?(Word)
-          @word_count += 1 if @skip.index(obj.attr).nil?
-          unless obj.lexicals.nil?
-            lexis = obj.get_class(@lexis) #lexicals.collect { |lex| (lex.attr =~ @lexis) ? lex : nil }.compact # get_class(@lexis)
-            lexis.each { |lex| @vectors << lex.form.downcase }
-            add('Anzahl von Vektor-Wörtern', lexis.size)
-          end
+          forward("#{@prompt} #{obj.inspect}") if eval(@debug)
+        elsif obj.is_a?(Word) && !@skip.include?(obj.attr)
+          @word_count += 1
+
+          cnt = obj.get_class(@lex).each { |lex|
+            vec = lex.form.downcase
+            @sort_format ? @vectors << vec : forward(vec)
+          }.size
+
+          add('Anzahl von Vektor-Wörtern', cnt)
         end
       end
 
       private
 
-      def sendVector
-        return if @vectors.size==0
-
+      def send_vectors
         add('Objekte gefiltert', @vectors.size)
 
-        # Array der Vector-Wörter zählen und nach Häufigkeit sortieren
-        if @sort=='normal'
-          @vectors = @vectors.compact.sort.uniq
+        if @sort_format == 'normal'
+          @vectors.sort!
+          @vectors.uniq!
+
+          @vectors.each(&method(:forward)).clear
         else
-          cnt = Hash.new(0)
-          @vectors.compact.each { |e| cnt[e]+=1 }
-          @vectors = cnt.to_a.sort { |x,y|
-            if (y[1]<=>x[1])==0
-              x[0]<=>y[0]
-            else
-              y[1]<=>x[1]
-            end
-          }
-        end
+          cnt, fmt = Hash.new(0), '%d'
+
+          @vectors.each { |v| cnt[v] += 1 }.clear
+          vec = cnt.sort_by { |v, c| [-c, v] }
 
-        # Vectoren je nach Parameter formatiert weiterleiten
-        @vectors.collect { |vec|
-          case @sort
-          when 'term_abs' then sprintf "%d %s", vec[1], vec[0]
-          when 'term_rel' then sprintf "%6.5f %s", vec[1].to_f/@word_count, vec[0]
-          when 'sto_abs'  then sprintf "%s {%d}", vec[0], vec[1]
-          when 'sto_rel'  then sprintf "%s {%6.5f}", vec[0], vec[1].to_f/@word_count
-          else sprintf "%s", vec
+          if @sort_method == 'rel'
+            vec.each { |v| v[1] /= @word_count }
+            fmt = '%6.5f'
           end
-        }.each(&method(:forward))
 
-        @word_count = 0 if @sort == 'sto_rel'
+          if @sort_format == 'sto'
+            fmt, @word_count = "%s {#{fmt}}", 0.0
+          else
+            fmt.insert(1, '2$') << ' %1$s'
+          end
+
+          vec.each { |v| forward(fmt % v) }
+        end
       end
 
     end

data/lib/lingo/attendee/word_searcher.rb

@@ -71,19 +71,18 @@ class Lingo
         set_dic
       end
 
-      def control(cmd, par)
-        @dic.report.each_pair { |key, value|
-          set(key, value)
-        } if cmd == STR_CMD_STATUS
+      def control(cmd, param)
+        report_on(cmd, @dic)
       end
 
       def process(obj)
         if obj.is_a?(Token) && obj.attr == TA_WORD
           inc('Anzahl gesuchter Wörter')
-          word = @dic.find_word(obj.form)
-          inc('Anzahl gefundener Wörter') unless word.unknown?
-          obj = word
+
+          obj = @dic.find_word(obj.form)
+          inc('Anzahl gefundener Wörter') unless obj.unknown?
         end
+
         forward(obj)
       end
 

data/lib/lingo/attendee.rb

@@ -24,6 +24,8 @@
 ###############################################################################
 #++
 
+require 'nuggets/string/evaluate'
+
 class Lingo
 
   # Lingo ist als universelles Indexierungssystem entworfen worden. Seine Stärke liegt in der einfachen Konfigurierbarkeit für
@@ -80,6 +82,8 @@ class Lingo
     STA_TIM_COMMANDS = 'Time to control  '
     STA_TIM_OBJECTS  = 'Time to process  '
 
+    DEFAULT_SKIP = [TA_PUNCTUATION, TA_OTHER].join(',')
+
     def initialize(config, lingo)
       @lingo = lingo
 
@@ -129,6 +133,15 @@ class Lingo
 
     private
 
+    def find_word(f, d = @dic, g = @gra)
+      w = d.find_word(f)
+      g && (block_given? ? !yield(w) : w.unknown?) ? g.find_compound(f) : w
+    end
+
+    def report_on(cmd, *rep)
+      rep.each { |r| r.report.each { |q| set(*q) } } if cmd == STR_CMD_STATUS
+    end
+
     def sta_for(key)
       %w[NUM TIM].map { |i| self.class.const_get("STA_#{i}_#{key.upcase}") }
     end
@@ -139,9 +152,9 @@ class Lingo
 
       return yield unless @lingo.report_time
 
-      @timer = Time.new
+      @timer = Time.now.to_i
       res = yield
-      add(t, Time.new - @timer)
+      add(t, Time.now.to_i - @timer)
       res
     end
 
@@ -184,7 +197,7 @@ class Lingo
         })
       }
 
-      @lingo.warn msg % arg
+      warn msg % arg
     end
 
     def report_status
@@ -192,8 +205,8 @@ class Lingo
 
       msg = "Attendee <%s> was connected from '%s' to '%s' reporting..."
 
-      @lingo.warn msg % @config.values_at(*%w[name in out]),
-        nil, report.sort.map { |k, v| " #{k} = #{v}" }, nil
+      warn msg % @config.values_at(*%w[name in out]), nil,
+        report.sort.map! { |k, v| " #{k} = #{v}" }, nil
     end
 
     def skip_command
@@ -217,8 +230,8 @@ class Lingo
       @config.fetch(key, default)
     end
 
-    def get_array(key, default = nil)
-      get_key(key, default).split(STRING_SEPARATOR_RE)
+    def get_array(key, default = nil, m = nil)
+      get_key(key, default).split(SEP_RE).tap { |ary| ary.map!(&m) if m }
     end
 
     def dictionary(src, mod)
@@ -237,6 +250,10 @@ class Lingo
       @gra = grammar(get_array('source'), get_key('mode', 'all'))
     end
 
+    def warn(*msg)
+      @lingo.warn(*msg)
+    end
+
   end
 
 end
@@ -252,6 +269,7 @@ require_relative 'attendee/noneword_filter'
 require_relative 'attendee/object_filter'
 require_relative 'attendee/variator'
 require_relative 'attendee/sequencer'
+require_relative 'attendee/stemmer'
 require_relative 'attendee/synonymer'
 require_relative 'attendee/text_reader'
 require_relative 'attendee/text_writer'

data/lib/lingo/buffered_attendee.rb

@@ -28,8 +28,6 @@ class Lingo
 
   class BufferedAttendee < Attendee
 
-    BufferInsert = Struct.new(:position, :object)
-
     def initialize(config, lingo)
       @buffer, @inserts = [], []
       super
@@ -38,30 +36,58 @@ class Lingo
     protected
 
     def process(obj)
-      @buffer.push(obj)
+      @buffer << obj
       process_buffer if process_buffer?
     end
 
     private
 
-    def forward_buffer
-      @inserts.sort_by!(&:position).each { |i|
-        @buffer.insert(i.position, i.object)
-      }.clear
+    def form_at(index, klass = WordForm)
+      obj = @buffer[index]
+      obj.form if obj.is_a?(klass)
+    end
 
+    def forward_buffer
+      @inserts.sort_by!(&:first).each { |i| @buffer.insert(*i) }.clear
       @buffer.each(&method(:forward)).clear
     end
 
+    def forward_number_of_token(len = default = @buffer.size, punct = !default)
+      begin
+        unless @buffer.empty?
+          forward(item = @buffer.delete_at(0))
+          len -= 1 unless punct && item.form == CHAR_PUNCT
+        end
+      end while len > 0
+    end
+
+    def valid_tokens_in_buffer
+      @buffer.count { |item| item.form != CHAR_PUNCT }
+    end
+
     def process_buffer?
-      true
+      !instance_variable_defined?(:@expected_tokens_in_buffer) ||
+      valid_tokens_in_buffer >= @expected_tokens_in_buffer
     end
 
     def process_buffer
       raise NotImplementedError
     end
 
-    def deferred_insert(pos, obj)
-      @inserts << BufferInsert.new(pos, obj)
+    def control_multi(cmd, dic = @dic)
+      report_on(cmd, dic)
+
+      if [STR_CMD_RECORD, STR_CMD_EOF].include?(cmd)
+        @eof_handling = true
+
+        while valid_tokens_in_buffer > 1
+          process_buffer
+        end
+
+        forward_number_of_token
+
+        @eof_handling = false
+      end
     end
 
   end

data/lib/lingo/cachable.rb

@@ -31,26 +31,26 @@ class Lingo
   module Cachable
 
     def init_cachable
-      @cache = Hash.new(false)
+      @cachable_hash = Hash.new(false)
     end
 
     def hit?(key)
-      @cache.has_key?(key)
+      @cachable_hash.has_key?(key)
     end
 
-    def store(key, value)
-      @cache[key] = cache_value(value)
-      value
+    def store(key, val)
+      @cachable_hash[key] = cache_value(val)
+      val
     end
 
     def retrieve(key)
-      cache_value(@cache[key])
+      cache_value(@cachable_hash[key])
     end
 
     private
 
-    def cache_value(value)
-      value.nil? ? nil : value.dup
+    def cache_value(val)
+      val.dup unless val.nil?
     end
 
   end

data/lib/lingo/config.rb

@@ -41,13 +41,12 @@ class Lingo
       load_config('config')
 
       Array(self['meeting/attendees']).each { |a|
-        r = a['text_reader'] || a['textreader'] or next
+        r = a['text_reader'] || a['textreader'] or next  # DEPRECATE textreader
 
         f = @cli.files
 
         if i = r['files']
-          r['files'] = i.strip == '$(files)' ?
-            f : i.split(STRING_SEPARATOR_RE)
+          r['files'] = i.strip == '$(files)' ? f : i.split(SEP_RE)
         elsif !f.empty?
           r['files'] = f
         end
@@ -57,12 +56,12 @@ class Lingo
     end
 
     def [](key)
-      key_to_nodes(key).inject(@opts) { |value, node| value[node] }
+      key_to_nodes(key).inject(@opts) { |hash, node| hash[node] }
     end
 
-    def []=(key, value)
+    def []=(key, val)
       nodes = key_to_nodes(key); node = nodes.pop
-      (self[nodes_to_key(nodes)] ||= {})[node] = value
+      (self[nodes_to_key(nodes)] ||= {})[node] = val
     end
 
     def stdin

data/lib/lingo/ctl.rb

@@ -25,7 +25,6 @@
 #++
 
 require 'optparse'
-require 'fileutils'
 
 class Lingo
 
@@ -88,7 +87,7 @@ Usage: #{PROG} <command> [arguments] [options]
        #{PROG} [-h|--help] [--version]
 EOT
 
-    def do
+    def ctl
       parse_options
       send("do_#{ALIASES[ARGV.shift]}")
     end
@@ -230,7 +229,7 @@ EOT
   end
 
   def self.ctl
-    Ctl.do
+    Ctl.ctl
   rescue => err
     raise if $VERBOSE
     abort "#{err.backtrace.first}: #{err} (#{err.class})"

data/lib/lingo/database/crypter.rb

@@ -24,6 +24,8 @@
 ###############################################################################
 #++
 
+require 'digest/sha1'
+
 class Lingo
 
   class Database
@@ -39,35 +41,16 @@ class Lingo
       end
 
       def encode(key, val)
-        hex = ''
-
-        crypt(key, val).each_byte { |byte|
-          # To get a hex representation for a char we just utilize
-          # the quotient and the remainder of division by base 16.
-          q, r = byte.divmod(16)
-          hex << HEX_CHARS[q] << HEX_CHARS[r]
-        }
-
-        [digest(key), hex]
+        [digest(key), crypt(key, val).each_byte.with_object('') { |b, s|
+          b.divmod(16).each { |i| s << HEX_CHARS[i] }
+        }]
       end
 
       def decode(key, val)
-        str, q, first = '', 0, false
-
-        val.each_byte { |byte|
-          byte = byte.chr(ENC)
-
-          # Our hex chars are 2 bytes wide, so we have to keep track
-          # of whether it's the first or the second of the two.
-          if first = !first
-            q = HEX_CHARS.index(byte)
-          else
-            # Now we got both parts, so let's revert the divmod(16)
-            str << q * 16 + HEX_CHARS.index(byte)
-          end
-        }
-
-        crypt(key, str)
+        crypt(key, val.each_byte.each_slice(2).with_object('') { |b, s|
+          q, r = b.map { |i| HEX_CHARS.index(i.chr(ENC)) }
+          s << q * 16 + r
+        })
       end
 
       private

data/lib/lingo/database/gdbm_store.rb

@@ -32,14 +32,12 @@ class Lingo
 
     module GDBMStore
 
-      private
+      Database.register(self, 'db')
 
-      def store_ext
-        '.db'
-      end
+      private
 
       def _open
-        GDBM.open(@dbm_name)
+        GDBM.open(@stofile)
       end
 
     end

data/lib/lingo/database/libcdb_store.rb

@@ -32,14 +32,12 @@ class Lingo
 
     module LibCDBStore
 
-      private
+      Database.register(self, 'cdb')
 
-      def store_ext
-        '.cdb'
-      end
+      private
 
       def create
-        LibCDB::CDB.open(@dbm_name, 'w') { |db|
+        LibCDB::CDB.open(@stofile, 'w') { |db|
           @db = db
           yield
         }
@@ -48,7 +46,7 @@ class Lingo
       end
 
       def _open
-        LibCDB::CDB.open(@dbm_name)
+        LibCDB::CDB.open(@stofile)
       end
 
     end

data/lib/lingo/database/sdbm_store.rb

@@ -32,26 +32,31 @@ class Lingo
 
     module SDBMStore
 
+      Database.register(self, %w[dir pag], -1, false)
+
       private
 
       def uptodate?
-        super(@dbm_name + '.pag')
+        super(@stofile + EXT.last)
       end
 
       def _clear
-        File.delete(*Dir["#{@dbm_name}.{pag,dir}"])
+        File.delete(*Dir["#{@stofile}{#{EXT.join(',')}}"])
       end
 
       def _open
-        SDBM.open(@dbm_name)
+        SDBM.open(@stofile)
+      end
+
+      def _get(key)
+        val = super
+        val && val.encode(ENC)
       end
 
       def _set(key, val)
         if val.length > 950
+          warn "Warning: Entry `#{key}' (#{@srcfile}) too long for SDBM. Truncating..."
           val = val[0, 950]
-
-          @lingo.warn "Warning: Entry `#{key}' (#{@src_file})" <<
-                      'too long for SDBM. Truncating...'
         end
 
         super

data/lib/lingo/database/show_progress.rb

@@ -28,50 +28,10 @@ class Lingo
 
   class Database
 
-    class ShowProgress
+    class ShowProgress < ShowProgress
 
-      def initialize(src, max, act = true)
-        @out, @act = src.instance_variable_get(:@lingo).config.stderr, act
-
-        # To get the length of the formatted string we have
-        # to actually substitute the placeholder.
-        fmt = ' [%3d%%]'
-        len = (fmt % 0).length
-
-        # Now we know how far to "go back" to
-        # overwrite the formatted string...
-        back = "\b" * len
-
-        @fmt = fmt       + back
-        @clr = ' ' * len + back
-
-        print src.instance_variable_get(:@config)['name'], ': '
-
-        @rat, @cnt, @next = max / 100.0, 0, 0
-        print 'convert '
-        step
-
-        yield self
-
-        print "#{@clr}ok\n"
-      end
-
-      def [](value)
-        @cnt = value
-        step if @cnt >= @next
-      end
-
-      private
-
-      def step
-        percent = @cnt / @rat
-        @next = (percent + 1) * @rat
-
-        print @fmt % percent
-      end
-
-      def print(*args)
-        @out.print(*args) if @act
+      def initialize(obj, max, act = true)
+        super(obj, max, obj.instance_variable_get(:@config)['name'], act, 'convert')
       end
 
     end

data/lib/lingo/database/source/key_value.rb

@@ -39,18 +39,14 @@ class Lingo
 
         def initialize(id, lingo)
           super
-
-          @separator = @config.fetch('separator', '*')
-          @line_pattern = Regexp.new('^(' + @legal_word + ')' + Regexp.escape(@separator) + '(' + @legal_word + ')$')
+          @pat = /^(#{@wrd})#{Regexp.escape(@sep ||= '*')}(#{@wrd})$/
         end
 
         private
 
         def convert_line(line, key, val)
           key, val = key.strip, val.strip
-          val = '' if key == val
-          val = [val + '#' + @wordclass]
-          [key, val]
+          [key, %W[#{val unless key == val}##{@def}]]
         end
 
       end

data/lib/lingo/database/source/multi_key.rb

@@ -40,9 +40,7 @@ class Lingo
 
         def initialize(id, lingo)
           super
-
-          @separator = @config.fetch('separator', ';')
-          @line_pattern = Regexp.new('^' + @legal_word + '(?:' + Regexp.escape(@separator) + @legal_word + ')*$')
+          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep ||= ';')}#{@wrd})*$/
         end
 
         def set(db, key, val)
@@ -52,8 +50,8 @@ class Lingo
         private
 
         def convert_line(line, key, val)
-          values = line.split(@separator).map { |value| value.strip }
-          [values[0], values[1..-1]]
+          values = line.split(@sep).each(&:strip!)
+          [values.shift, values]
         end
 
       end

data/lib/lingo/database/source/multi_value.rb

@@ -38,11 +38,7 @@ class Lingo
 
         def initialize(id, lingo)
           super
-
-          @separator = @config.fetch('separator', ';')
-          @line_pattern = Regexp.new('^' + @legal_word + '(?:' + Regexp.escape(@separator) + @legal_word + ')*$')
-
-          @idx = -1
+          @pat, @idx = /^#{@wrd}(?:#{Regexp.escape(@sep ||= ';')}#{@wrd})*$/, -1
         end
 
         def set(db, key, val)
@@ -53,7 +49,7 @@ class Lingo
         private
 
         def convert_line(line, key, val)
-          [nil, line.split(@separator).map { |value| value.strip }]
+          [nil, line.split(@sep).each(&:strip!)]
         end
 
       end