lingo 1.8.1 → 1.8.2

data/lib/lingo/attendee/text_reader.rb

@@ -24,6 +24,8 @@
 ###############################################################################
 #++
 
+require 'find'
+
 %w[filemagic mime/types hpricot pdf-reader].each { |lib|
   begin
     require lib
@@ -55,19 +57,19 @@ class Lingo
     #                Komma voneinander getrennt, z.B.
     #                  files: 'readme.txt'
     #                  files: 'readme.txt,lingo.cfg'
-    # <b><i>lir-record-pattern</i></b>:: Mit diesem Parameter wird angegeben, woran der Anfang
-    #                                    eines neuen Records erkannt werden kann und wie die
-    #                                    Record-Nummer identifiziert wird. Das Format einer
-    #                                    LIR-Datei ist z.B.
-    #                                      [00001.]
-    #                                      020: ¬Die Aufgabenteilung zwischen Wortschatz und Grammatik.
+    # <b><i>records</i></b>:: Mit diesem Parameter wird angegeben, woran der Anfang
+    #                         eines neuen Records erkannt werden kann und wie die
+    #                         Record-Nummer identifiziert wird. Das Format einer
+    #                         LIR-Datei ist z.B.
+    #                           [00001.]
+    #                           020: ¬Die Aufgabenteilung zwischen Wortschatz und Grammatik.
     #
-    #                                      [00002.]
-    #                                      020: Nicht-konventionelle Thesaurusrelationen als Orientierungshilfen.
-    #                                    Mit der Angabe von
-    #                                      lir-record-pattern: "^\[(\d+)\.\]"
-    #                                    werden die Record-Zeilen erkannt und jeweils die Record-Nummer +00001+,
-    #                                    bzw. +00002+ erkannt.
+    #                           [00002.]
+    #                           020: Nicht-konventionelle Thesaurusrelationen als Orientierungshilfen.
+    #                         Mit der Angabe von
+    #                           records: "^\[(\d+)\.\]"
+    #                         werden die Record-Zeilen erkannt und jeweils die Record-Nummer +00001+,
+    #                         bzw. +00002+ erkannt.
     #
     # === Generierte Kommandos
     # Damit der nachfolgende Datenstrom einwandfrei verarbeitet werden kann, generiert der TextReader
@@ -90,7 +92,7 @@ class Lingo
     # Bei der Verarbeitung einer LIR-Datei mit der Ablaufkonfiguration <tt>t2.cfg</tt>
     #   meeting:
     #     attendees:
-    #       - text_reader: { out: lines,  files: '$(files)', lir-record-pattern: "^\[(\d+)\.\]" }
+    #       - text_reader: { out: lines,  files: '$(files)', records: "^\[(\d+)\.\]" }
     #       - debugger:    { in: lines, prompt: 'out>'}
     # ergibt die Ausgabe mit <tt>lingo -c t2 lir.txt</tt>
     #   out> *LIR-FORMAT('')
@@ -105,112 +107,136 @@ class Lingo
 
       protected
 
-      # TODO: FILE und LIR-FILE
-      # TODO: lir-record-pattern abkürzen
-      # Interpretation der Parameter
+      # TODO: FILE und LIR-FILE (?)
       def init
-        @files = Array(get_key('files', '-'))
-        @rec_pat = Regexp.new(get_key('lir-record-pattern', ''))
-        @is_LIR_file = has_key?('lir-record-pattern')
-        @chomp = get_key('chomp', true)
-        @filter = get_key('filter', false)
+        get_files
+
+        @chomp    = get_key('chomp', true)
+        @filter   = get_key('filter', false)
+        @progress = get_key('progress', false)
+
+        if @lir = get_key('records', get_key('lir-record-pattern', nil))  # DEPRECATE lir-record-pattern
+          @lir = @lir == true ? %r{^\[(\d+)\.\]} : Regexp.new(@lir)
+        end
       end
 
       def control(cmd, param)
-        if cmd==STR_CMD_TALK
-          forward(STR_CMD_LIR, '') if @is_LIR_file
-          @files.each { |filename| spool(filename) }
+        if cmd == STR_CMD_TALK
+          forward(STR_CMD_LIR, '') if @lir
+          @files.each(&method(:spool))
         end
       end
 
       private
 
       # Gibt eine Datei zeilenweise in den Ausgabekanal
-      def spool(filename)
-        unless stdin?(filename)
-          raise FileNotFoundError.new(filename) unless File.exist?(filename)
-
+      def spool(path)
+        unless stdin = stdin?(path)
           inc('Anzahl Dateien')
-          add('Anzahl Bytes', File.stat(filename).size)
+          add('Anzahl Bytes', size = File.size(path))
+
+          size = nil unless @progress
         end
 
-        forward(STR_CMD_FILE, filename)
-
-        filter(filename) { |line|
-          inc('Anzahl Zeilen')
-          line.chomp! if @chomp
-          line.gsub!(/\303\237/, "ß")
-      ### HACK
-          if @is_LIR_file && line =~ @rec_pat
-            forward(STR_CMD_RECORD, $1)
-          else
-            forward(line) if line.size>0
-          end
+        forward(STR_CMD_FILE, path)
+
+        ShowProgress.new(self, size, path) { |progress|
+          filter(path, stdin) { |line, pos|
+            inc('Anzahl Zeilen')
+            progress[pos]
+
+            line.chomp! if @chomp
+
+            if line =~ @lir
+              forward(STR_CMD_RECORD, $1)
+            else
+              forward(line) unless line.empty?
+            end
+          }
         }
 
-        forward(STR_CMD_EOF, filename)
+        forward(STR_CMD_EOF, path)
       end
 
-      def filter(filename, &block)
-        file = stdin?(filename) ?
-          @lingo.config.stdin.set_encoding(ENC) :
-          File.open(filename, 'rb', encoding: ENC)
-
-        file = case @filter == true ? file_type(filename, file) : @filter.to_s
-          when /html/ then filter_html(file)
-          when /xml/  then filter_html(file, true)
-          when /pdf/  then filter_pdf(file, &block) or return
-          else file
-        end if @filter
+      def filter(path, stdin = stdin?(path))
+        io, block = stdin ? [
+          @lingo.config.stdin.set_encoding(ENC),
+          lambda { |line| yield line, 0 }
+        ] : [
+          File.open(path, 'rb', encoding: ENC),
+          lambda { |line| yield line, io.pos }
+        ]
+
+        case @filter == true ? file_type(path, io) : @filter.to_s
+          when /html/i then io = filter_html(io)
+          when /xml/i  then io = filter_html(io, true)
+          when /pdf/i  then filter_pdf(io, &block); return
+        end
 
-        file.each_line(&block)
+        io.each_line(&block) if io
       end
 
-      def filter_pdf(file, &block)
+      def filter_pdf(io, &block)
         if Object.const_defined?(:PDF) && PDF.const_defined?(:Reader)
-          PDFFilter.filter(file, &block)
-          nil
+          PDFFilter.filter(io, &block)
         else
           warn "PDF filter not available. Please install `pdf-reader'."
-          file
         end
       end
 
-      def filter_html(file, xml = false)
+      def filter_html(io, xml = false)
         if Object.const_defined?(:Hpricot)
-          Hpricot(file, xml: xml).inner_text
+          Hpricot(io, xml: xml).inner_text
         else
           warn "#{xml ? 'X' : 'HT'}ML filter not available. Please install `hpricot'."
-          file
+          nil
         end
       end
 
-      def file_type(filename, file)
-        if Object.const_defined?(:FileMagic) && file.respond_to?(:rewind)
-          type = FileMagic.fm(:mime, simplified: true).buffer(file.read(256))
-          file.rewind
-          type
+      def file_type(path, io)
+        if Object.const_defined?(:FileMagic) && io.respond_to?(:rewind)
+          FileMagic.fm(:mime, simplified: true).buffer(io.read(256)).tap {
+            io.rewind
+          }
         elsif Object.const_defined?(:MIME) && MIME.const_defined?(:Types)
-          if type = MIME::Types.of(filename).first
-            type.content_type
-          else
-            warn 'Filters not available. File type could not be determined.'
-            nil
-          end
+          MIME::Types.of(path).first.tap { |type| type ? type.content_type :
+            warn('Filters not available. File type could not be determined.')
+          }
         else
           warn "Filters not available. Please install `ruby-filemagic' or `mime-types'."
           nil
         end
       end
 
-      def stdin?(filename)
-        %w[STDIN -].include?(filename)
+      def stdin?(path)
+        %w[STDIN -].include?(path)
+      end
+
+      def get_files
+        args = [get_key('glob', '*.txt'), get_key('recursive', false)]
+
+        @files = []
+
+        Array(get_key('files', '-')).each { |path|
+          stdin?(path) ? @files << path : add_files(path, *args)
+        }
+
+        @files.map!(&File.method(:expand_path))
+        @files.uniq!
+      end
+
+      def add_files(path, glob, recursive = false)
+        Dir[path].sort!.each { |match|
+          File.directory?(match) ? recursive ? Find.find(match) { |entry|
+            @files << entry if File.file?(entry) && File.fnmatch?(glob, entry)
+          } : add_files(File.join(match, glob), glob) : @files << match
+        }.empty? and raise FileNotFoundError.new(path)
       end
 
       class PDFFilter
 
-        def self.filter(file, &block)
-          PDF::Reader.new.parse(file, new(&block))
+        def self.filter(io, &block)
+          PDF::Reader.new.parse(io, new(&block))
         end
 
         def initialize(&block)

data/lib/lingo/attendee/text_writer.rb

@@ -82,11 +82,15 @@ class Lingo
       def init
         @ext = get_key('ext', 'txt2')
         @lir = get_key('lir-format', false)
-        @sep = @lir ? ' ' : eval("\"#{@config['sep'] || ' '}\"")
+
+        @sep   = @config['sep'] unless @lir
+        @sep &&= @sep.evaluate
+        @sep ||= ' '
+
         @no_sep, @no_puts = true, false
       end
 
-      def control(cmd, par)
+      def control(cmd, param)
         case cmd
           when STR_CMD_LIR
             @lir = true
@@ -94,49 +98,43 @@ class Lingo
             @no_sep = true
 
             if stdout?(@ext)
-              @filename = @ext
-              @file = @lingo.config.stdout
+              @filename, @file = @ext, @lingo.config.stdout
             else
-              @filename = par.sub(/(\.[^.]+)?$/, '.'+@ext)
-              @file = File.new(@filename,'w')
               inc('Anzahl Dateien')
+              @file = File.open(@filename = File.set_ext(param, ".#{@ext}"), 'w')
             end
 
-            @lir_rec_no = ''
-            @lir_rec_buf = Array.new
+            @lir_rec_no, @lir_rec_buf = '', []
           when STR_CMD_RECORD
             @no_sep = true
+
             if @lir
               flush_lir_buffer
-              @lir_rec_no = par
+              @lir_rec_no = param
             end
           when STR_CMD_EOL
             @no_sep = true
+
             unless @lir
-              @file.puts unless @no_puts # unless @sep=="\n"
               inc('Anzahl Zeilen')
+              @file.puts unless @no_puts
             end
           when STR_CMD_EOF
             flush_lir_buffer if @lir
 
             unless stdout?(@filename)
+              add('Anzahl Bytes', @file.size)
               @file.close
-              add('Anzahl Bytes', File.stat(@filename).size)
             end
         end
       end
 
       def process(obj)
-        if @lir
-          @lir_rec_buf << (obj.kind_of?(Token) ? obj.form : obj.to_s)
-        else
-          @file.print @sep unless @no_sep
-          @no_sep=false if @no_sep
-          if obj.is_a?(Word) || obj.is_a?(Token)
-            @file.print obj.form
-          else
-            @file.print obj
-          end
+        obj = obj.form if obj.is_a?(WordForm)
+
+        @lir ? @lir_rec_buf << obj : begin
+          @no_sep ? @no_sep = false : @file.print(@sep)
+          @file.print(obj)
         end
       end
 
@@ -144,12 +142,11 @@ class Lingo
 
       def flush_lir_buffer
         unless @lir_rec_no.empty? || @lir_rec_buf.empty?
-          if @sep =~ /\n/
-            @file.print '*', @lir_rec_no, "\n", @lir_rec_buf.join(@sep), "\n"
-          else
-            @file.print @lir_rec_no, '*', @lir_rec_buf.join(@sep), "\n"
-          end
+          @file.print(*[@lir_rec_no, @lir_rec_buf.join(@sep), "\n"].tap { |buf|
+            @sep =~ /\n/ ? buf.insert(1, "\n").unshift('*') : buf.insert(1, '*')
+          })
         end
+
         @lir_rec_no = ''
         @lir_rec_buf.clear
       end

data/lib/lingo/attendee/tokenizer.rb

@@ -101,10 +101,6 @@ class Lingo
       protected
 
       def init
-        # Regular Expressions für Token-Erkennung einlesen
-        regulars = get_key('regulars', '')
-        raise NoConfigKeyError.new(:regulars) unless regulars
-
         @space = get_key('space', false)
         @tags  = get_key('tags',  true)
         @wiki  = get_key('wiki',  true)
@@ -115,30 +111,22 @@ class Lingo
         @rules << ['WIKI', /^\[\[.+?\]\]/]   unless @wiki
         @rules.unshift(['WIKI', /^=+.+=+$/]) unless @wiki
 
-        # Mit _xxx_ gekennzeichnete Makros anwenden und Expressions ergänzen und umwandeln
-        macros = {}
-
-        regulars.each { |rule|
-          name = rule.keys[0]
-          expr = rule.values[0].gsub(/_(\w+?)_/) {
+        get_key('regulars', []).each_with_object({}) { |rule, macros|
+          expr = rule.values.first.gsub(/_(\w+?)_/) {
             macros[$&] || begin
               Database::Source.const_get("UTF8_#{$1.upcase}")
             rescue NameError
             end
           }
 
-          if name =~ /^_\w+_$/    # is a macro
-            macros[name] = expr if name =~ /^_\w+_$/
+          if (name = rule.keys.first) =~ /^_\w+_$/
+            macros[name] = expr
           else
-            @rules << [name, Regexp.new('^'+expr)]
+            @rules << [name, /^#{expr}/]
           end
         }
 
-        # Der Tokenizer gibt jedes Zeilenende als Information weiter, sofern es sich
-        # nicht um die Verarbeitung einer LIR-Datei handelt. Im Falle einer normalen Datei
-        # wird der Dateiname gespeichert und als Kennzeichen für die Erzeugung von
-        # Zeilenende-Nachrichten herangezogen.
-        @filename = nil
+        @filename = @cont = nil
       end
 
       def control(cmd, param)
@@ -154,12 +142,10 @@ class Lingo
           inc('Anzahl Zeilen')
 
           tokenize(obj) { |form, attr|
-            token = Token.new(form, attr)
-
-            inc('Anzahl Muster '+token.attr)
+            inc("Anzahl Muster #{attr}")
             inc('Anzahl Token')
 
-            forward(token)
+            forward(Token.new(form, attr))
           }
 
           forward(STR_CMD_EOL, @filename) if @filename
@@ -175,7 +161,7 @@ class Lingo
         case @cont
           when 'HTML'
             if textline =~ /^[^<>]*>/
-              yield $~[0], @cont
+              yield $&, @cont
               textline, @cont = $', nil
             else
               yield textline, @cont
@@ -183,7 +169,7 @@ class Lingo
             end
           when 'WIKI'
             if textline =~ /^[^\[\]]*\]\]/
-              yield $~[0], @cont
+              yield $&, @cont
               textline, @cont = $', nil
             else
               yield textline, @cont
@@ -191,12 +177,12 @@ class Lingo
             end
           when nil
             if !@tags && textline =~ /<[^<>]*$/
-              yield $~[0], @cont = 'HTML'
+              yield $&, @cont = 'HTML'
               textline = $`
             end
 
             if !@wiki && textline =~ /\[\[[^\[\]]*$/
-              yield $~[0], @cont = 'WIKI'
+              yield $&, @cont = 'WIKI'
               textline = $`
             end
         end
@@ -204,7 +190,7 @@ class Lingo
         until textline.empty?
           @rules.each { |name, expr|
             if textline =~ expr
-              yield $~[0], name if name != 'SPAC' || @space
+              yield $&, name if name != 'SPAC' || @space
               textline = $'
               break
             end

data/lib/lingo/attendee/variator.rb

@@ -75,67 +75,39 @@ class Lingo
       protected
 
       def init
-        # Parameter verarbeiten
-        @marker  = get_key('marker', '*')
-        @max_var = get_key('max-var', '10000').to_i
-        filter = get_array('check', WA_UNKNOWN)
+        @marker = get_key('marker', '*')
+        @max    = get_key('max-var', max = 10000).to_i
+        @max    = max unless @max > 0
+        @var    = get_key('variations')
 
-        # Daten verarbeiten
-        @var_strings = get_key('variations')
-        raise MissingConfigError.new(:variations) if @var_strings.empty?
+        raise MissingConfigError.new(:variations) if @var.empty?
 
-        # Initialisierungen
         @check = Hash.new(false)
-        filter.each { |s| @check[s.upcase] = true }
+        get_array('check', WA_UNKNOWN).each { |s| @check[s.upcase] = true }
 
         set_dic
         set_gra
-
-        if @max_var.zero?
-          @max_var = 10000
-          @lingo.warn "#{self.class}: max-var is 0, setting to #{@max_var}"
-        end
       end
 
-      def control(cmd, par)
-        # Status wird abgefragt
-        if cmd == STR_CMD_STATUS
-          # Eigenen Status um Status von Dictionary und Grammer erweitern
-          @dic.report.each_pair { | k, v | set( k, v ) }
-          @gra.report.each_pair { | k, v | set( k, v ) }
-        end
+      def control(cmd, param)
+        report_on(cmd, @dic, @gra)
       end
 
       def process(obj)
-        # Zu prüfende Wörter filtern
         if obj.is_a?(Word) && @check[obj.attr]
-          # Statistik für Report
           inc('Anzahl gesuchter Wörter')
 
-          # Erzeuge Variationen einer Wortform
-          variations = [obj.form]
-          @var_strings.each do |switch|
-            from, to = switch
-            variations = variate(variations, from, to)
-          end
-
-          # Prüfe Variation auf bekanntes Wort
-          variations[0...@max_var].each do |var|
-            # Variiertes Wort im Wörterbuch suchen
-            word = @dic.find_word(var)
-            word = @gra.find_compositum(var) if word.unknown?
-            next if word.unknown? || (
-              word.attr == WA_KOMPOSITUM && word.lexicals.any? { |lex|
-                lex.attr[0..0] == LA_TAKEITASIS
+          @var.each_with_object([obj.form]) { |a, v| variate(v, *a) }.
+            tap { |v| v.slice!(@max..-1) }.each { |var|
+            next if (word = find_word(var)).unknown? || (
+              word.attr == WA_COMPOUND && word.lexicals.any? { |lex|
+                lex.attr.start_with?(LA_TAKEITASIS)
               }
             )
 
-            # Das erste erkannte Wort beendet die Suche
             inc('Anzahl gefundener Wörter')
-            word.form = @marker + var
-            forward(word)
-            return
-          end
+            return forward(word.tap { word.form = @marker + var })
+          }
         end
 
         forward(obj)
@@ -146,32 +118,20 @@ class Lingo
       # Variiere die Bestandteile eines Arrays gemäß den Austauschvorgaben.
       #
       # variate( 'Tiieh', 'ieh', 'sch' ) => ['Tiieh', 'Tisch']
-      def variate(variation_list, from, to)
-        # neue Varianten sammeln
-        add_variations = []
-        from_re = Regexp.new(from)
-
-        # alle Wörter in der variation_list permutieren
-        variation_list.each do |wordform|
+      def variate(variations, from, to)
+        add, change, re = [], [from, to], Regexp.new(from)
 
-          # Wortform in Teile zerlegen und anschließend Dimension feststellen
-          wordpart = " #{wordform} ".split( from_re )
-          n = wordpart.size - 1
+        variations.each { |form|
+          parts = " #{form} ".split(re)
 
-          # Austauschketten in Matrix hinterlegen
-          change = [from, to]
-
-          # Austauschketten auf alle Teile anwenden
-          (1..(2**n-1)).each do |i|
-            variation = wordpart[0]
-            # i[x] = Wert des x.ten Bit von Integer i
-            (1..n).each { |j| variation += change[i[j-1]] + wordpart[j]  }
-
-            add_variations << variation.strip
-          end
-        end
+          1.upto(2 ** (n = parts.size - 1) - 1) { |i|
+            var = parts.first
+            1.upto(n) { |j| var += change[i[j - 1]] + parts[j] }
+            add << var.strip
+          }
+        }
 
-        variation_list + add_variations
+        variations.concat(add)
       end
 
     end