lingo 1.8.5 → 1.8.6

data/lib/lingo/attendee/sequencer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -28,6 +28,7 @@ class Lingo
 
   class Attendee
 
+    #--
     # Der Sequencer ist von seiner Funktion her ähnlich dem Multiworder. Der Multiworder
     # nutzt zur Erkennung von Mehrwortgruppen spezielle Wörterbücher, der Sequencer hingegen
     # definierte Folgen von Wortklassen. Mit dem Sequencer können Indexterme generiert werden,
@@ -90,11 +91,10 @@ class Lingo
     #   out> :./PUNC:
     #   out> *EOL('test.txt')
     #   out> *EOF('test.txt')
+    #++
 
     class Sequencer < BufferedAttendee
 
-      protected
-
       def init
         @stopper = get_array('stopper', DEFAULT_SKIP)
                      .push(WA_UNKNOWN, WA_UNKMULPART)
@@ -114,8 +114,8 @@ class Lingo
         raise MissingConfigError.new(:sequences) if @seq.empty?
       end
 
-      def control(cmd, param)
-        process_buffer if [STR_CMD_RECORD, STR_CMD_EOF].include?(cmd)
+      def control(cmd, *)
+        process_buffer if [:RECORD, :EOF].include?(cmd)
       end
 
       def process_buffer?

data/lib/lingo/attendee/stemmer.rb

@@ -30,8 +30,6 @@ class Lingo
 
     class Stemmer < self
 
-      protected
-
       def init
         extend(Lingo.get_const(get_key('type', 'porter'), self.class))
 
@@ -39,6 +37,9 @@ class Lingo
         @all = get_key('mode', '').downcase == 'all'
       end
 
+      def control(*)
+      end
+
       def process(obj)
         if obj.is_a?(Word) && obj.unknown?
           stem = stem(Unicode.downcase(obj.form), @all)

data/lib/lingo/attendee/synonymer.rb

@@ -28,6 +28,7 @@ class Lingo
 
   class Attendee
 
+    #--
     # Der Synonymer untersucht die von anderen Attendees ermittelten Grundformen eines Wortes
     # und sucht in den angegebenen Wörterbüchern nach Relationen zu anderen Grundformen.
     # Gefundene Relationen erweitern die Liste des Word-Objektes und werden zur späteren
@@ -67,19 +68,17 @@ class Lingo
     #   out> :./PUNC:
     #   out> *EOL('test.txt')
     #   out> *EOF('test.txt')
+    #++
 
     class Synonymer < self
 
-      protected
-
       def init
         set_dic
         @com = !get_key('compound-parts', false)
         @skip = get_array('skip', WA_UNKNOWN, :upcase)
       end
 
-      def control(cmd, param)
-        # can control
+      def control(*)
       end
 
       def process(obj)

data/lib/lingo/attendee/text_reader.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -37,6 +37,7 @@ class Lingo
 
   class Attendee
 
+    #--
     # Der TextReader ist eine klassische Datenquelle. Er liest eine oder mehrere Dateien
     # und gibt sie Zeilenweise in den Ausgabekanal. Der Start bzw. Wechsel einer Datei
     # wird dabei über den Kommandokanal angekündigt, ebenso wie das Ende.
@@ -102,75 +103,71 @@ class Lingo
     #   out> *RECORD('00002')
     #   out> "020: Nicht-konventionelle Thesaurusrelationen als Orientierungshilfen."
     #   out> *EOF('lir.txt')
+    #++
 
     class TextReader < self
 
-      protected
-
-      # TODO: FILE und LIR-FILE (?)
+      # TODO: FILE/LIR-FILE (?)
       def init
         get_files
 
-        @chomp    = get_key('chomp', true)
         @filter   = get_key('filter', false)
         @progress = get_key('progress', false)
 
-        lingo.deprecate('lir-record-pattern', :records, self) if has_key?('lir-record-pattern')
+        if has_key?('lir-record-pattern')
+          lingo.config.deprecate('lir-record-pattern', :records, self)
+        end
 
         @lir  = get_re('records', get_key('lir-record-pattern', nil), %r{^\[(\d+)\.\]})  # DEPRECATE lir-record-pattern
         @cut  = get_re('fields', !!@lir, %r{^.+?:\s*})
         @skip = get_re('skip', nil)
       end
 
-      def control(cmd, param)
-        if cmd == STR_CMD_TALK
-          forward(STR_CMD_LIR, '') if @lir
+      def control(cmd, *)
+        if cmd == :TALK
+          command(:LIR) if @lir
           @files.each { |i| spool(i) }
+
+          command(:EOT)
+          :skip_command
         end
       end
 
       private
 
-      # Gibt eine Datei zeilenweise in den Ausgabekanal
       def spool(path)
-        forward(STR_CMD_FILE, path)
+        command(:FILE, path)
 
-        io = !stdin?(path) ? open_file(name = path) :
-          string_or_io(lingo.config.stdin.set_encoding(ENC))
+        io = !stdin?(path) ? open_file(name = path) : begin
+          stdin = lingo.config.stdin.set_encoding(ENC)
+          @progress ? StringIO.new(stdin.read) : stdin
+        end
 
         Progress.new(self, @progress && io.size, name) { |progress|
-          filter(io, path, progress) { |line, pos|
-            progress << pos
+          pos = 0 unless pos?(io = filter(io, path, progress))
 
-            line.chomp! if @chomp
-            next if line =~ @skip
+          io.each { |line|
+            progress << offset = pos ? pos += line.bytesize : io.pos
 
-            if line =~ @lir
-              forward(STR_CMD_RECORD, $1 || $&)
-            else
-              line.sub!(@cut, '') if @cut
-              forward(line) unless line.empty?
-            end
+            line =~ @skip ? nil : line =~ @lir ?
+              command(:RECORD, $1 || $&) : begin
+                line.sub!(@cut, '') if @cut
+                forward(line, offset) unless line.empty?
+              end
           }
         }
 
-        forward(STR_CMD_EOF, path)
+        command(:EOF, path)
       end
 
       def filter(io, path, progress)
-        block = @progress ?
-          lambda { |line| yield line, io.pos } :
-          lambda { |line| yield line, 0 }
-
-        io = case @filter == true ? file_type(io, path) : @filter.to_s
+        case @filter == true ? file_type(io, path) : @filter.to_s
           when 'pdftotext' then filter_pdftotext(io, path, progress)
           when /html/i     then filter_html(io)
           when /xml/i      then filter_html(io, true)
-          when /pdf/i      then filter_pdf(io, &block); return
+          when /pdf/i      then filter_pdf(io)
           else io
         end
-
-        io.each_line(&block)
       end
 
       def filter_pdftotext(io, path, progress)
@@ -189,7 +186,7 @@ class Lingo
 
       def filter_pdf(io)
         if Object.const_defined?(:PDF) && PDF.const_defined?(:Reader)
-          PDF::Reader.new(io).pages.each { |page| yield page.text }
+          text_enum(PDF::Reader.new(io).pages)
         else
           cancel_filter(:PDF, 'pdf-reader')
         end
@@ -199,8 +196,7 @@ class Lingo
         type = xml ? :XML : :HTML
 
         if Object.const_defined?(:Nokogiri)
-          doc = Nokogiri.send(type, io, nil, ENC)
-          string_or_io(doc.children.map { |x| x.inner_text }.join)
+          text_enum(Nokogiri.send(type, io, nil, ENC).children)
         else
           cancel_filter(type, :nokogiri)
         end
@@ -208,7 +204,7 @@ class Lingo
 
       def file_type(io, path)
         if Object.const_defined?(:FileMagic) && io.respond_to?(:rewind)
-          type = FileMagic.fm(:mime, simplified: true).buffer(io.read(256))
+          type = FileMagic.fm(:mime, simplified: true).io(io, 256)
           io.rewind
           type
         elsif Object.const_defined?(:MIME) && MIME.const_defined?(:Types)
@@ -234,8 +230,9 @@ class Lingo
         %w[STDIN -].include?(path)
       end
 
-      def string_or_io(io)
-        @progress ? StringIO.new(io.is_a?(String) ? io : io.read) : io
+      def pos?(io)
+        io.pos if io.respond_to?(:pos)
+      rescue Errno::ESPIPE
       end
 
       def open_file(path)
@@ -257,6 +254,10 @@ class Lingo
         tempfiles.each(&:unlink)
       end
 
+      def text_enum(collection)
+        Enumerator.new { |y| collection.each { |x| y << x.text } }
+      end
+
       def get_files
         args = [get_key('glob', '*.txt'), get_key('recursive', false)]
 

data/lib/lingo/attendee/text_writer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -28,6 +28,7 @@ class Lingo
 
   class Attendee
 
+    #--
     # Der TextWriter ermöglicht die Umleitung des Datenstroms in eine Textdatei. Dabei werden
     # Objekte, die nicht vom Typ String sind in eine sinnvolle Textrepresentation gewandelt.
     # Der Name der Ausgabedatei wird durch den Namen der Eingabedatei (des Textreaders) bestimmt.
@@ -74,27 +75,26 @@ class Lingo
     #   0.01923 textdatei
     #   0.01923 typ
     #   0.01923 umleitung
+    #++
 
     class TextWriter < self
 
-      protected
-
       def init
         @ext = get_key('ext', 'txt2')
         @lir = get_key('lir-format', false)
 
-        @sep   = @config['sep'] unless @lir
+        @sep = get_key('sep', nil) unless @lir
         @sep &&= @sep.evaluate
         @sep ||= ' '
 
         @no_sep, @no_puts = true, false
       end
 
-      def control(cmd, param)
+      def control(cmd, param = nil, *)
         case cmd
-          when STR_CMD_LIR
+          when :LIR
             @lir = true unless @lir.nil?
-          when STR_CMD_FILE
+          when :FILE
             @no_sep = true
 
             if stdout?(@ext)
@@ -104,20 +104,20 @@ class Lingo
             end
 
             @lir_rec_no, @lir_rec_buf = '', []
-          when STR_CMD_RECORD
+          when :RECORD
             if @lir
               @no_sep = true
 
               flush_lir_buffer
               @lir_rec_no = param
             end
-          when STR_CMD_EOL
+          when :EOL
             @no_sep = true
 
             unless @lir
               @file.puts unless @no_puts
             end
-          when STR_CMD_EOF
+          when :EOF
             flush_lir_buffer if @lir
 
             unless stdout?(@filename)

data/lib/lingo/attendee/tokenizer.rb

@@ -28,6 +28,7 @@ class Lingo
 
   class Attendee
 
+    #--
     # Der Tokenizer zerlegt eine Textzeile in einzelne Token. Dies ist notwendig,
     # damit nachfolgende Attendees die Textdatei häppchenweise verarbeiten können.
     #
@@ -77,6 +78,7 @@ class Lingo
     #   out> :./PUNC:
     #   out> *EOL('test.txt')
     #   out> *EOF('test.txt')
+    #++
 
     class Tokenizer < self
 
@@ -88,14 +90,14 @@ class Lingo
         ['SPAC', /^\s+/],
         ['WIKI', /^=+.+=+|^__[A-Z]+__/],
         ['NUMS', /^[+-]?(?:\d{4,}|\d{1,3}(?:\.\d{3,3})*)(?:\.|(?:,\d+)?%?)/],
-        ['URLS', /^(?:www\.|mailto:|#{PROTO}|\S+?[._]\S+?@\S+?\.)\S+/],
+        ['URLS', /^(?:www\.|mailto:|#{PROTO}|\S+?[._]\S+?@\S+?\.)[^\s<>]+/],
         ['ABRV', /^(?:(?:(?:#{CHAR})+\.)+)(?:#{CHAR})+/],
         ['WORD', /^(?:#{CHAR}|#{DIGIT}|-)+/],
         ['PUNC', /^[!,.:;?¡¿]+/]
       ]
 
       OTHER = [
-        ['OTHR', /^["$#%&'()*+\/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷]/],
+        ['OTHR', /^["$#%&'()*+\/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷„“–]/],
         ['HELP', /^\S+/]
       ]
 
@@ -113,8 +115,8 @@ class Lingo
           RULES.assoc(name)
         end
 
-        def rules(name)
-          RULES.select { |rule,| rule == name }
+        def rules(name = nil)
+          name ? RULES.select { |rule,| rule == name } : RULES.map(&:first)
         end
 
         def delete(*names)
@@ -155,13 +157,14 @@ class Lingo
 
       end
 
-      protected
-
       def init
         @space = get_key('space', false)
         @tags  = get_key('tags',  false)
         @wiki  = get_key('wiki',  false)
 
+        @skip_tags = get_array('skip-tags', '', :downcase)
+        @tags = true unless @skip_tags.empty?
+
         skip = []
         skip << 'HTML' unless @tags
         skip << 'WIKI' unless @wiki
@@ -170,7 +173,7 @@ class Lingo
           hash.delete_if { |name, _| skip.include?(Token.clean(name)) }
         }
 
-        @nest, nest_re = [], []
+        @override, @nest, nest_re = [], [], []
 
         @nests.each { |name, re|
           re.map!.with_index { |r, i| r.is_a?(Regexp) ?
@@ -182,30 +185,30 @@ class Lingo
 
         @nest_re = /^(?<_>.*?)(?:#{nest_re.join('|')})/
 
-        @filename = @linenum = nil
+        reset
       end
 
-      def control(cmd, param)
+      def control(cmd, filename = nil, *)
         case cmd
-          when STR_CMD_FILE then @filename, @linenum = param, 1
-          when STR_CMD_LIR  then @filename, @linenum = nil, nil
-          when STR_CMD_EOL  then @linenum += 1 if @linenum
-          when STR_CMD_EOF  then @nest.clear
+          when :FILE then reset(filename)
+          when :LIR  then reset(nil, nil)
+          when :EOL  then @linenum += 1 if @linenum
+          when :EOF  then @override.clear; @nest.clear
         end
       end
 
-      def process(obj)
-        if obj.is_a?(String)
-          tokenize(obj)
-          forward(STR_CMD_EOL, @filename) if @filename
-        else
-          forward(obj)
-        end
+      def process(line, offset)
+        @offset = offset
+        tokenize(line)
+        command(:EOL, @filename) if @filename
       end
 
       private
 
-      # tokenize("Eine Zeile.")  ->  [:Eine/WORD:, :Zeile/WORD:, :./PUNC:]
+      def reset(filename = nil, linenum = 1)
+        @filename, @linenum, @position, @offset = filename, linenum, -1, 0
+      end
+
       def tokenize(line)
         @nest.empty? ? tokenize_line(line) : tokenize_nest(line)
       rescue => err
@@ -225,8 +228,11 @@ class Lingo
       def tokenize_rule(line, rules = @rules)
         rules.find { |name, expr|
           next unless line =~ expr
-          forward_token($&, name) if name != 'SPAC' || @space
-          yield $'
+
+          rest = $'
+          forward_token($&, name, rest) if name != 'SPAC' || @space
+
+          yield rest
         }
       end
 
@@ -235,13 +241,26 @@ class Lingo
         mdc = @nests[@nest.last].last.match(line)
 
         if mdo && (!mdc || mdo[0].length < mdc[0].length)
-          forward_token(mdo[:_], @nest.last) unless mdo[:_].empty?
-
+          rest = mdo.post_match
           nest = @nests.keys.find { |name| mdo[name] }
-          forward_nest(mdo[nest], mdo.post_match, nest)
+          text = mdo[nest]
+          lead = mdo[:_]
+
+          forward_token(lead, @nest.last, text + rest) unless lead.empty?
+
+          forward_nest(text, nest, rest)
         elsif mdc
-          forward_token(mdc[0], @nest.pop)
-          tokenize(mdc.post_match)
+          rest = mdc.post_match
+          nest = @nest.pop
+          text = mdc[0]
+
+          forward_token(text, nest, rest)
+
+          if overriding?(nest)
+            @override.pop if text.downcase.end_with?("/#{@override.last}>")
+          end
+
+          tokenize(rest)
         else
           forward_token(line, @nest.last)
         end
@@ -250,21 +269,32 @@ class Lingo
       def tokenize_open(line)
         @nests.each { |nest, (open_re, _)|
           next unless line =~ open_re
-          return forward_nest($&, $', nest)
+          return forward_nest($&, nest, $')
         }
 
         tokenize_rule(line, OTHER) { |rest| line = rest }
         tokenize(line)
       end
 
-      def forward_nest(match, rest, nest)
-        forward_token(match, nest)
+      def forward_nest(match, nest, rest)
+        if overriding?(nest)
+          tag = rest[/^[^\s>]*/].downcase
+          @override << tag if @skip_tags.include?(tag)
+        end
+
+        forward_token(match, nest, rest)
+
         @nest << nest
         tokenize(rest)
       end
 
-      def forward_token(*args)
-        forward(Token.new(*args))
+      def forward_token(form, attr, rest = '')
+        forward(Token.new(form, @override.empty? ? attr : 'SKIP',
+          @position += 1, @offset - form.bytesize - rest.bytesize))
+      end
+
+      def overriding?(nest)
+        nest == 'HTML' && !@skip_tags.empty?
       end
 
     end