lingo 1.8.6 → 1.8.7

data/lib/lingo/attendee/lsi_filter.rb

@@ -0,0 +1,99 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  class Attendee
+
+    class LsiFilter < DeferredAttendee
+
+      def init
+        require_lib('lsi4r')
+
+        @lex  = get_re('lexicals', '[sy]')
+        @skip = get_ary('skip', DEFAULT_SKIP, :upcase)
+
+        @transform = get_key('transform', Lsi4R::DEFAULT_TRANSFORM)
+        @cutoff    = get_flo('cut',       Lsi4R::DEFAULT_CUTOFF)
+
+        @min = get_flo('min', false)
+        @abs = get_flo('abs', false)
+        @nul = get_flo('nul', false)
+        @new = get_key('new', true)
+
+        @sort = get_key('sort', false)
+        @sort.downcase! if @sort.respond_to?(:downcase!)
+
+        @docnum, @vectors = 0, Hash.new { |h, k| h[k] = [] }
+      end
+
+      def control(cmd, *)
+        :skip_command if cmd == :EOL
+      end
+
+      def control_deferred(cmd, *)
+        @docnum += 1 if TERMINALS.include?(cmd)
+      end
+
+      def process(obj)
+        if obj.is_a?(Word) && !@skip.include?(obj.attr)
+          vec = []
+          obj.each_lex(@lex) { |lex| vec << Unicode.downcase(lex.form) }
+          @vectors[@docnum].concat(vec) unless vec.empty?
+        end
+      end
+
+      private
+
+      def send_lsi
+        lsi = Lsi4R.new(@vectors); @vectors.clear
+
+        if lsi.build(transform: @transform, cutoff: @cutoff)
+          options, vec = { min: @min, abs: @abs, nul: @nul, new: @new }, []
+
+          fmt = @sort ? @sort == 'sto' ?
+            '%s {%.5f}' : '%2$.5f %1$s' : '%s %.5f' unless @sort == 'normal'
+
+          yield !@sort ? lambda { |docnum|
+            lsi.each_norm(docnum, options) { |_, *v| forward(fmt % v) }
+          } : lambda { |docnum|
+            lsi.each_norm(docnum, options) { |_, *v| vec << v }
+
+            !fmt ? vec.sort!.each { |v, _| forward(v) } :
+              vec.sort_by { |v, w| [-w, v] }.each { |v| forward(fmt % v) }
+
+            vec.clear
+          }
+        end
+      end
+
+      alias_method :flush_deferred, :send_lsi
+
+    end
+
+  end
+
+end

data/lib/lingo/attendee/multi_worder.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -40,7 +40,9 @@ class Lingo
     #
     # === Mögliche Verlinkung
     # Erwartet:: Daten vom Typ *Word* z.B. von Wordsearcher, Decomposer, Ocr_variator, MultiWorder
-    # Erzeugt:: Daten vom Typ *Word* (mit Attribut WA_MULTIWORD). Je erkannter Mehrwortgruppe wird ein zusätzliches Word-Objekt in den Datenstrom eingefügt. Z.B. für Ocr_variator, Sequencer, Noneword_filter, Vector_filter
+    # Erzeugt:: Daten vom Typ *Word* (mit Attribut WA_MULTIWORD). Je erkannter Mehrwortgruppe wird
+    # ein zusätzliches Word-Objekt in den Datenstrom eingefügt. Z.B. für Ocr_variator, Sequencer,
+    # Vector_filter
     #
     # === Parameter
     # Kursiv dargestellte Parameter sind optional (ggf. mit Angabe der Voreinstellung).
@@ -88,7 +90,7 @@ class Lingo
 
         lex_src, lex_mod, d = nil, nil, lingo.dictionary_config['databases']
 
-        (mul_src = get_array('source')).each { |src|
+        (mul_src = get_ary('source')).each { |src|
           s, m = d[src].values_at('use-lex', 'lex-mode')
 
           if lex_src.nil? || lex_src == s
@@ -106,76 +108,99 @@ class Lingo
         @lex_gra = grammar(lex_src, lex_mod)
 
         @syn_dic = if @combine && has_key?('use-syn')
-          dictionary(get_array('use-syn'), get_key('syn-mode', 'all'))
+          dictionary(get_ary('use-syn'), get_key('syn-mode', 'all'))
         end
 
         @expected_tokens_in_buffer, @eof_handling = 3, false
       end
 
       def control(cmd, *)
-        control_multi(cmd)
+        if [:RECORD, :EOF].include?(cmd)
+          @eof_handling = true
+
+          while process_buffer?(2)
+            process_buffer
+          end
+
+          forward_number_of_token
+
+          @eof_handling = false
+        end
       end
 
-      def process_buffer
-        unless form_at(0) == CHAR_PUNCT
-          unless (res = check_multiword_key(3)).empty?
-            len = res.map { |r| r.is_a?(Lexical) ? r.form.count(' ') + 1 : r }
-            len.sort!.reverse!
+      private
+
+      def form_at(index)
+        obj = @buffer[index]
+        obj.form if obj.is_a?(WordForm) && obj.form != CHAR_PUNCT
+      end
+
+      def forward_number_of_token(len = default = @buffer.size, punct = !default)
+        begin
+          unless @buffer.empty?
+            forward(item = @buffer.delete_at(0))
+            len -= 1 unless punct && item.form == CHAR_PUNCT
+          end
+        end while len > 0
+      end
+
+      def process_buffer?(num = @expected_tokens_in_buffer)
+        @buffer.count { |item| item.form != CHAR_PUNCT } >= num
+      end
 
-            unless (max = len.first) > 3
+      def process_buffer
+        if form_at(0)
+          if res = check_multiword(3, len = [])
+            if (max = len.first) <= 3
               create_and_forward_multiword(3, res)
               forward_number_of_token(3)
+            elsif !@eof_handling && @buffer.size < max
+              @expected_tokens_in_buffer = max
             else
-              unless @eof_handling || @buffer.size >= max
-                @expected_tokens_in_buffer = max
-              else
-                forward_number_of_token(len.find { |l|
-                  r = check_multiword_key(l)
-                  create_and_forward_multiword(l, r) unless r.empty?
-                } || 1)
-
-                @expected_tokens_in_buffer = 3
-                process_buffer if process_buffer?
-              end
+              forward_number_of_token(len.find { |l|
+                create_and_forward_multiword(l) } || 1)
+
+              @expected_tokens_in_buffer = 3
+              process_buffer if process_buffer?
             end
 
             return
           end
 
-          unless (res = check_multiword_key(2)).empty?
-            create_and_forward_multiword(2, res)
-            forward_number_of_token(1)
-          end
+          create_and_forward_multiword(2) && forward_number_of_token(1)
         end
 
         forward_number_of_token(1, false)
         @expected_tokens_in_buffer = 3
       end
 
-      private
+      def create_and_forward_multiword(len, lex = check_multiword(len))
+        return unless lex
 
-      def create_and_forward_multiword(len, lex)
         pos, parts = 0, []
 
         begin
-          if (form = form_at(pos)) == CHAR_PUNCT
-            @buffer.delete_at(pos)
-            parts[-1] += CHAR_PUNCT
-          else
+          if form = form_at(pos)
             @buffer[pos].attr = WA_UNKMULPART if @buffer[pos].unknown?
             parts << form
             pos += 1
+          else
+            @buffer.delete_at(pos)
+            parts[-1] += CHAR_PUNCT
           end
         end while pos < len
 
-        forward(Word.new_lexicals(parts.join(' '),
-          WA_MULTIWORD, lex.select { |l| l.is_a?(Lexical) }))
+        wrd = Word.new_lexicals(parts.join(' '),
+          WA_MULTIWORD, lex.select { |l| l.is_a?(Lexical) })
+        wrd.token = @buffer[pos - 1].token
+
+        forward(wrd)
       end
 
-      def check_multiword_key(len)
-        return [] if valid_tokens_in_buffer < len
+      def check_multiword(len, lst = nil)
+        return unless process_buffer?(len)
 
-        seq = []
+        seq, mul, sep = [], [], ' '
 
         @buffer.each { |obj|
           next seq << [obj] unless obj.is_a?(WordForm)
@@ -195,17 +220,18 @@ class Lingo
         }
 
         if @combine
-          mul = []
-
           seq.shift.product(*seq) { |key|
-            @mul_dic.select(key.join(' '), mul)
+            @mul_dic.select(key.join(sep), mul)
             break unless @all || mul.empty?
           } && mul.uniq!
-
-          mul
         else
-          @mul_dic.select(seq.map! { |i,| i }.join(' '))
+          @mul_dic.select(seq.map! { |i,| i }.join(sep), mul)
         end
+
+        lst.push(seq.size).concat(mul.map { |r| r.is_a?(Lexical) ?
+          r.form.count(sep) + 1 : r }).sort!.reverse!.uniq! if lst
+
+        mul unless mul.empty?
       end
 
     end

data/lib/lingo/attendee/sequencer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -43,7 +43,9 @@ class Lingo
     #
     # === Mögliche Verlinkung
     # Erwartet:: Daten vom Typ *Word* z.B. von Wordsearcher, Decomposer, Ocr_variator, Multiworder
-    # Erzeugt:: Daten vom Typ *Word* (mit Attribut WA_SEQUENCE). Je erkannter Mehrwortgruppe wird ein zusätzliches Word-Objekt in den Datenstrom eingefügt. Z.B. für Ocr_variator, Sequencer, Noneword_filter, Vector_filter
+    # Erzeugt:: Daten vom Typ *Word* (mit Attribut WA_SEQUENCE). Je erkannter Mehrwortgruppe wird
+    # ein zusätzliches Word-Objekt in den Datenstrom eingefügt. Z.B. für Ocr_variator, Sequencer,
+    # Vector_filter
     #
     # === Parameter
     # Kursiv dargestellte Parameter sind optional (ggf. mit Angabe der Voreinstellung).
@@ -95,15 +97,20 @@ class Lingo
 
     class Sequencer < BufferedAttendee
 
+      UNK = %w[#]
+      NUM = %w[0]
+
+      CLS = /[[:alpha:]#{NUM.join}]/o
+
       def init
-        @stopper = get_array('stopper', DEFAULT_SKIP)
+        @stopper = get_ary('stopper', DEFAULT_SKIP)
                      .push(WA_UNKNOWN, WA_UNKMULPART)
 
         @mwc = get_key('multiword', LA_MULTIWORD)
         @cls = []
 
         @seq = get_key('sequences').map { |str, fmt|
-          @cls.concat(cls = (str = str.downcase).scan(/[[:alpha:]]/))
+          @cls.concat(cls = (str = str.downcase).scan(CLS))
 
           (str =~ /\W/ ? [Regexp.new(str), nil] : [str, cls]).push(
             fmt == true ? '|' : fmt ? fmt.gsub(/\d+/, '%\&$s') : nil)
@@ -124,7 +131,7 @@ class Lingo
 
       def process_buffer
         flush(@buffer.size < 2 ? @buffer : begin
-          arg, cls, mwc, unk = [[], buf = [], map = [], @seq], @cls, @mwc, %w[#]
+          arg = [[], buf = [], map = [], @seq]
 
           iter, skip, rewind = @buffer.each_with_index, 0, lambda {
             iter.rewind; skip.times { iter.next }; skip = 0
@@ -142,16 +149,17 @@ class Lingo
               rewind.call
             end
 
-            att = obj.is_a?(Word) && !obj.unknown? ? obj.attrs(false) : unk
+            att = (tok = obj.is_a?(Token)) ? obj.number? ? NUM : UNK :
+              obj.is_a?(Word) && !obj.unknown? ? obj.compound_attrs : UNK
 
-            if (att &= cls).empty?
+            if (att &= @cls).empty?
               find_seq(*arg)
               rewind.call if skip > 0
             else
-              if n = obj.multiword_size(mwc)
-                n.times { iter.next }
-                skip = idx + 1
-              end
+              obj.each_lex(@mwc) { |lex|
+                lex.form.count(' ').succ.times { iter.next }
+                break skip = idx + 1
+              } unless tok
 
               buf << obj
               map << att
@@ -178,18 +186,18 @@ class Lingo
             while pos = q.index(str, pos || 0)
               _str, _cls = [$&, $&.chars] unless cls
 
-              args.clear
+              _tok = nil; args.clear
 
               _cls.each_with_index { |wc, i|
-                buf[pos + i].lexicals.find { |l|
-                  args[i] = l.form if l.attr == wc
-                } or break
+                obj = buf[pos + i];_tok ||= obj.token
+
+                args[i] = obj.is_a?(Word) ? obj.lexicals.find { |lex|
+                  break lex.form if lex.attr == wc } : obj.form or break
               } or next
 
-              forms << (
+              forms << [_str, _tok,
                 fmt =~ /\d/ ? fmt.gsub('%0$s', _str) % args :
-                fmt ? "#{_str}:#{args.join(fmt)}" : args.join(' ')
-              )
+                fmt ? "#{_str}:#{args.join(fmt)}" : args.join(' ')]
 
               pos += 1
             end
@@ -197,7 +205,12 @@ class Lingo
         }.clear
 
         forms.uniq!
-        forms.each { |f| mat << Word.new_lexical(f, WA_SEQUENCE, LA_SEQUENCE) }
+
+        forms.each { |s, t, f|
+          wrd = Word.new_lexical(f, WA_SEQUENCE, LA_SEQUENCE)
+          wrd.pattern, wrd.token = s, t
+          mat << wrd
+        }
 
         buf.clear
         mat

data/lib/lingo/attendee/synonymer.rb

@@ -36,7 +36,7 @@ class Lingo
     #
     # === Mögliche Verlinkung
     # Erwartet:: Daten vom Typ *Word* z.B. von Wordsearcher, Decomposer, Ocr_variator, Multiworder
-    # Erzeugt:: Daten vom Typ *Word* (ggf. um Relationen ergänzt) z.B. für Decomposer, Ocr_variator, Multiworder, Sequencer, Noneword_filter, Vector_filter
+    # Erzeugt:: Daten vom Typ *Word* (ggf. um Relationen ergänzt) z.B. für Decomposer, Ocr_variator, Multiworder, Sequencer, Vector_filter
     #
     # === Parameter
     # Kursiv dargestellte Parameter sind optional (ggf. mit Angabe der Voreinstellung).
@@ -75,7 +75,7 @@ class Lingo
       def init
         set_dic
         @com = !get_key('compound-parts', false)
-        @skip = get_array('skip', WA_UNKNOWN, :upcase)
+        @skip = get_ary('skip', WA_UNKNOWN, :upcase)
       end
 
       def control(*)

data/lib/lingo/attendee/text_reader.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -107,10 +107,14 @@ class Lingo
 
     class TextReader < self
 
+      include TextUtils
+
       # TODO: FILE/LIR-FILE (?)
       def init
         get_files
 
+        @encoding = get_enc
+
         @filter   = get_key('filter', false)
         @progress = get_key('progress', false)
 
@@ -124,42 +128,40 @@ class Lingo
       end
 
       def control(cmd, *)
-        if cmd == :TALK
-          command(:LIR) if @lir
-          @files.each { |i| spool(i) }
-
-          command(:EOT)
-          :skip_command
-        end
-      end
+        return unless cmd == :TALK
 
-      private
+        command(:LIR) if @lir
 
-      def spool(path)
-        command(:FILE, path)
+        @files.each { |path|
+          command(:FILE, path)
 
-        io = !stdin?(path) ? open_file(name = path) : begin
-          stdin = lingo.config.stdin.set_encoding(ENC)
-          @progress ? StringIO.new(stdin.read) : stdin
-        end
+          io = stdin?(path) ? open_stdin : open_path(name = path)
 
-        Progress.new(self, @progress && io.size, name) { |progress|
-          pos = 0 unless pos?(io = filter(io, path, progress))
+          Progress.new(self, @progress && io.size, name) { |progress|
+            pos = 0 unless pos?(io = filter(io, path, progress))
 
-          io.each { |line|
-            progress << offset = pos ? pos += line.bytesize : io.pos
+            io.each { |line|
+              progress << offset = pos ? pos += line.bytesize : io.pos
 
-            line =~ @skip ? nil : line =~ @lir ?
-              command(:RECORD, $1 || $&) : begin
-                line.sub!(@cut, '') if @cut
-                forward(line, offset) unless line.empty?
-              end
+              line =~ @skip ? nil : line =~ @lir ?
+                command(:RECORD, $1 || $&) : begin
+                  line.sub!(@cut, '') if @cut
+                  forward(line, offset) unless line.empty?
+                end
+            }
           }
+
+          io.close unless stdin?(path)
+
+          command(:EOF, path)
         }
 
-        command(:EOF, path)
+        command(:EOT)
+        :skip_command
       end
 
+      private
+
       def filter(io, path, progress)
         case @filter == true ? file_type(io, path) : @filter.to_s
           when 'pdftotext' then filter_pdftotext(io, path, progress)
@@ -170,64 +172,51 @@ class Lingo
         end
       end
 
-      def filter_pdftotext(io, path, progress)
-        if cmd = File.which(name = 'pdftotext')
-          with_tempfile(name) { |tempfile|
-            pdf_path = stdin?(path) ? tempfile[:pdf, io] : path
-            system(cmd, '-q', pdf_path, txt_path = tempfile[:txt])
+      def filter_pdftotext(io, path, progress, name = 'pdftotext')
+        cancel_filter(:PDF, name, :command) unless cmd = File.which(name)
 
-            progress.init(File.size(txt_path)) if @progress
-            open_file(txt_path)
-          }
-        else
-          cancel_filter(:PDF, name, :command)
-        end
+        with_tempfile(name) { |tempfile|
+          pdf_path = stdin?(path) ? tempfile[:pdf, io] : path
+          system(cmd, '-q', pdf_path, txt_path = tempfile[:txt])
+
+          progress.init(File.size(txt_path)) if @progress
+          open_path(txt_path)
+        }
       end
 
       def filter_pdf(io)
-        if Object.const_defined?(:PDF) && PDF.const_defined?(:Reader)
-          text_enum(PDF::Reader.new(io).pages)
-        else
-          cancel_filter(:PDF, 'pdf-reader')
-        end
+        Object.const_defined?(:PDF) && PDF.const_defined?(:Reader) ? text_enum(
+          PDF::Reader.new(io).pages) : cancel_filter(:PDF, 'pdf-reader')
       end
 
-      def filter_html(io, xml = false)
-        type = xml ? :XML : :HTML
-
-        if Object.const_defined?(:Nokogiri)
-          text_enum(Nokogiri.send(type, io, nil, ENC).children)
-        else
-          cancel_filter(type, :nokogiri)
-        end
+      def filter_html(io, xml = false, type = xml ? :XML : :HTML)
+        Object.const_defined?(:Nokogiri) ? text_enum(Nokogiri.send(type,
+          io, nil, @encoding).children) : cancel_filter(type, :nokogiri)
       end
 
       def file_type(io, path)
-        if Object.const_defined?(:FileMagic) && io.respond_to?(:rewind)
-          type = FileMagic.fm(:mime, simplified: true).io(io, 256)
-          io.rewind
-          type
-        elsif Object.const_defined?(:MIME) && MIME.const_defined?(:Types)
-          if type = MIME::Types.of(path).first
-            type.content_type
-          else
-            cancel('Filters not available. File type could not be determined.')
-          end
-        else
-          cancel("Filters not available. Please install the `ruby-filemagic' or `mime-types' gem.")
-        end
+        Object.const_defined?(:FileMagic) && io.respond_to?(:pos=) ?
+          FileMagic.fm(:mime, simplified: true).io(io, 256, true) :
+        Object.const_defined?(:MIME) && MIME.const_defined?(:Types) ?
+          (type = MIME::Types.of(path).first) ? type.content_type :
+          cancel_filters('File type could not be determined.') :
+          cancel_filters(please_install(:gem, 'ruby-filemagic', 'mime-types'))
+      end
+
+      def cancel_filters(msg)
+        cancel("Filters not available. #{msg}")
       end
 
       def cancel_filter(type, name, what = :gem)
-        cancel("#{type} filter not available. Please install the `#{name}' #{what}.")
+        cancel("#{type} filter not available. #{please_install(what, name)}")
       end
 
-      def cancel(msg)
-        throw(:cancel, msg)
+      def please_install(what, *names)
+        "Please install the `#{names.join("' or `")}' #{what}."
       end
 
-      def stdin?(path)
-        %w[STDIN -].include?(path)
+      def cancel(msg)
+        throw(:cancel, msg)
       end
 
       def pos?(io)
@@ -235,10 +224,6 @@ class Lingo
       rescue Errno::ESPIPE
       end
 
-      def open_file(path)
-        File.open(path, 'rb', encoding: ENC)
-      end
-
       def with_tempfile(name)
         require 'tempfile'
 
@@ -263,31 +248,17 @@ class Lingo
 
         @files = []
 
-        Array(get_key('files', '-')).each { |path|
-          stdin?(path) ? @files << path :
-            add_files(File.expand_path(path), *args)
-        }
+        Array(get_key('files', '-')).each { |path| stdin?(path) ?
+          @files << path : add_files(File.expand_path(path), *args) }
       end
 
       def add_files(path, glob, recursive = false)
-        entries = Dir[path].sort!
-        raise FileNotFoundError.new(path) if entries.empty?
+        raise FileNotFoundError.new(path) if (entries = Dir[path]).sort!.empty?
 
         entries.each { |entry|
-          if File.directory?(entry)
-            if recursive
-              Find.find(entry) { |match|
-                if File.file?(match) && File.fnmatch?(glob, match)
-                  @files << match
-                end
-              }
-            else
-              add_files(File.join(entry, glob), glob)
-            end
-          else
-            @files << entry
-          end
-        }
+          !File.directory?(entry) ? @files << entry : !recursive ?
+            add_files(File.join(entry, glob), glob) : Find.find(entry) { |match|
+              @files << match if File.file?(match) && File.fnmatch?(glob, match) } }
       end
 
     end