lingo 1.8.2 → 1.8.3

data/en.lang

@@ -43,18 +43,16 @@
 #    lingo language definition
 ---
 language:
-
   name: 'Englisch'
 
   dictionary:
-
     databases:
-
       #    Systemwörterbücher
       sys-dic: { name: en/lingo-dic.txt, txt-format: WordClass, separator: '=' }
       sys-syn: { name: en/lingo-syn.txt, txt-format: KeyValue, separator: '=', def-wc: y }
       sys-mul: { name: en/lingo-mul.txt, txt-format: SingleWord, use-lex: 'sys-dic', def-wc: m }
-
+      sys-irr: { name: en/lingo-irr.txt, txt-format: WordClass, separator: '=' }
+      sys-wdn: { name: en/lingo-wdn.txt, txt-format: WordClass, separator: '=' }
       #    Benutzerwörterbücher
       usr-dic: { name: en/user-dic.txt, txt-format: WordClass, separator: '=' }
 
@@ -77,17 +75,6 @@ language:
       - [f, ""]
 
   attendees:
-    tokenizer:
-      regulars:
-        - _char_: '_baslat_|_lat1sp_|_latexa_|_latexb_|_ipaext_'
-        - NUMS:  '[+-]?(\d{4,}|\d{1,3}(\.\d{3,3})*)(\.|(,\d+)?%?)'
-        - URLS:  '((mailto:|(news|http|https|ftp|ftps)://)\S+|^(www(\.\S+)+)|[^\s.]+([\._]\S+)+@\S+(\.\S+)+)'
-        - ABRV:  '(((_char_)+\.)+)(_char_)+'
-        - WORD:  '(_char_|_digit_|\-)+'
-        - PUNC:  '([!,\.:;?]|\xc2\xa1|\xc2\xbf)'
-        - OTHR:  '([\"#$%&\x27()*\+\-/<=>@\[\\\]^_{|}~]|\xc2\xa2|\xc2\xa3|\xc2\xa4|\xc2\xa5|\xc2\xa6|\xc2\xa7|\xc2\xa8|\xc2\xa9|\xc2\xaa|\xc2\xab|\xc2\xac|\xc2\xae|\xc2\xaf|\xc2\xb0|\xc2\xb1|\xc2\xb2|\xc2\xb3|\xc2\xb4|\xc2\xb5|\xc2\xb6|\xc2\xb7|\xc2\xb8|\xc2\xb9|\xc2\xba|\xc2\xbb|\xc2\xbc|\xc2\xbd|\xc2\xbe|\xc3\x97|\xc3\xb7)'
-        - HELP:  '[^ ]*'
-
     variator:
       variations:
         - [ ieh, sch ]

data/lib/lingo/app.rb

@@ -0,0 +1,82 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+require 'shellwords'
+require 'sinatra/base'
+
+class Lingo
+
+  class App < Sinatra::Base
+
+    class << self
+
+      def init_app(file, *args, &block)
+        set :root, File.chomp_ext(file)
+        parse_options(*args, &block)
+      end
+
+      def parse_options(lingo_options = false)
+        argv, banner = [], "Usage: #{$0} [-h|--help] [sinatra-options]"
+        while arg = ARGV.shift and arg != '--'; argv << arg; end
+
+        if lingo_options || block_given?
+          banner << ' [-- lingo-options]'
+
+          opts = ENV["LINGO_#{name.split('::').last.upcase}_OPTS"]
+          ARGV.unshift(*Shellwords.shellsplit(opts)) if opts
+
+          ARGV.unshift(*lingo_options) if lingo_options.is_a?(Array)
+        end
+
+        OptionParser.new(banner, 16) { |o|
+          o.on('-p port',   'set the port (default is 4567)')                { |v| set :port, Integer(v) }
+          o.on('-o addr',   'set the host (default is 0.0.0.0)')             { |v| set :bind, v }
+          o.on('-e env',    'set the environment (default is development)')  { |v| set :environment, v.to_sym }
+          o.on('-s server', 'specify rack server/handler (default is thin)') { |v| set :server, v }
+          o.on('-x',        'turn on the mutex lock (default is off)')       {     set :lock, true }
+        }.parse!(argv)
+
+        ARGV.unshift(*yield) if block_given?
+      end
+
+      def rackup(name)
+        file = File.join(File.dirname(__FILE__), name, 'config.ru')
+        file if File.readable?(file)
+      end
+
+    end
+
+    def to_json(q, r)
+      q, r = 'q', 'Required parameter -- Input string' unless q
+
+      content_type :json
+      { q => r }.to_json
+    end
+
+  end
+
+end

data/lib/lingo/attendee/abbreviator.rb

@@ -68,7 +68,7 @@ class Lingo
     #   out> *EOL('test.txt')
     #   out> *EOF('test.txt')
 
-    class Abbreviator < BufferedAttendee
+    class Abbreviator < self
 
       protected
 
@@ -77,36 +77,32 @@ class Lingo
       end
 
       def control(cmd, param)
-        report_on(cmd, @dic)
-        process_buffer
+        send_abbr(nil) if [STR_CMD_RECORD, STR_CMD_EOF].include?(cmd)
       end
 
-      private
-
-      def process_buffer?
-        form_at(-1, Token) == CHAR_PUNCT
-      end
-
-      def process_buffer
-        if @buffer.size < 2
-          forward_buffer
-          return
-        end
-
-        if form = form_at(-2, Token)
-          inc('Anzahl gesuchter Abkürzungen')
-
-          if (abbr = find_word(form)).identified?
-            inc('Anzahl gefundener Abkürzungen')
-
-            abbr.form += CHAR_PUNCT
-
-            @buffer[-2] = abbr
-            @buffer.delete_at(-1)
+      def process(obj)
+        if obj.is_a?(Token)
+          if obj.form == CHAR_PUNCT
+            if @abbr && (abbr = find_word(form = @abbr.form)).identified?
+              form << CHAR_PUNCT unless form.end_with?(CHAR_PUNCT)
+              send_abbr(abbr)
+            else
+              send_abbr(@abbr)
+              forward(obj)
+            end
+          else
+            send_abbr(@abbr, obj)
           end
+        else
+          send_abbr(obj)
         end
+      end
+
+      private
 
-        forward_buffer
+      def send_abbr(abbr, obj = nil)
+        @abbr = obj
+        forward(abbr) if abbr
       end
 
     end

data/lib/lingo/attendee/debugger.rb

@@ -96,16 +96,20 @@ class Lingo
       end
 
       def control(cmd, param)
-        if cmd != STR_CMD_STATUS && eval(@cmd_eval)
-          warn "#{@prompt} #{AgendaItem.new(cmd, param).inspect}"
-        end
+        debug(AgendaItem.new(cmd, param), @cmd_eval)
       end
 
       def process(obj)
-        warn "#{@prompt} #{obj.inspect}" if eval(@obj_eval)
+        debug(obj, @obj_eval)
         forward(obj)
       end
 
+      private
+
+      def debug(obj, cond)
+        warn "#{@prompt} #{obj.inspect}" if eval(cond)
+      end
+
     end
 
   end

data/lib/lingo/attendee/decomposer.rb

@@ -80,7 +80,6 @@ class Lingo
       end
 
       def control(cmd, param)
-        report_on(cmd, @gra)
       end
 
       def process(obj)

data/lib/lingo/attendee/dehyphenizer.rb

@@ -89,7 +89,7 @@ class Lingo
         if ab.all? { |i| i.is_a?(Word) } && a.form[-1, 1] == h && !(
           (c = b.get_class(/./).first) && @skip.include?(c.attr)
         )
-          a, b = ab.map!(&:form)
+          a, b = ab.map! { |i| i.form }
 
           word = dehyphenize(a.chomp(h) + b)
           word = dehyphenize(a          + b) unless dehyphenized?(word)
@@ -106,7 +106,7 @@ class Lingo
       private
 
       def dehyphenize(form)
-        find_word(form, &:identified?)
+        find_word(form) { |i| i.identified? }
       end
 
       def dehyphenized?(word)

data/lib/lingo/attendee/multi_worder.rb

@@ -113,7 +113,7 @@ class Lingo
       end
 
       def control(cmd, param)
-        control_multi(cmd, @mul_dic)
+        control_multi(cmd)
       end
 
       def process_buffer
@@ -177,29 +177,36 @@ class Lingo
       def check_multiword_key(len)
         return [] if valid_tokens_in_buffer < len
 
-        seq = @buffer.map { |obj|
+        seq = []
+
+        @buffer.each { |obj|
           next [obj] unless obj.is_a?(WordForm)
           next if (form = obj.form) == CHAR_PUNCT
 
           w = find_word(form, @lex_dic, @lex_gra)
           l = w.lexicals
 
-          (w.attr == WA_COMPOUND ? [l.first] : l.empty? ? [w] : l.dup).tap { |i|
-            i.concat(@syn_dic.find_synonyms(w)) if @syn_dic
-            i.map! { |j| j.form.downcase }.uniq!
-          }
-        }
+          i = w.attr == WA_COMPOUND ? [l.first] : l.empty? ? [w] : l.dup
 
-        seq.compact!
-        seq.slice!(len..-1)
+          @syn_dic.find_synonyms(w, i) if @syn_dic
+          i.map! { |j| Unicode.downcase(j.form) }.uniq!
+
+          seq << i
+
+          break unless seq.length < len
+        }
 
         if @combine
-          [].tap { |mul| seq.shift.product(*seq) { |key|
-            mul.concat(@mul_dic.select(key.join(' ')))
+          mul = []
+
+          seq.shift.product(*seq) { |key|
+            @mul_dic.select(key.join(' '), mul)
             break unless @all_keys || mul.empty?
-          } && mul.uniq! }
+          } && mul.uniq!
+
+          mul
         else
-          @mul_dic.select(seq.map!(&:first).join(' '))
+          @mul_dic.select(seq.map! { |i,| i }.join(' '))
         end
       end
 

data/lib/lingo/attendee/noneword_filter.rb

@@ -87,9 +87,7 @@ class Lingo
 
       def process(obj)
         if obj.is_a?(Word) && obj.unknown?
-          inc('Anzahl nicht erkannter Wörter')
-
-          non = obj.form.downcase
+          non = Unicode.downcase(obj.form)
           @sort ? @nonewords << non : forward(non)
         end
       end
@@ -97,11 +95,8 @@ class Lingo
       private
 
       def send_nonewords
-        @nonewords.sort!
         @nonewords.uniq!
-
-        add('Objekte gefiltert', @nonewords.size)
-        @nonewords.each(&method(:forward)).clear
+        flush(@nonewords.sort!)
       end
 
     end

data/lib/lingo/attendee/sequencer.rb

@@ -97,11 +97,15 @@ class Lingo
 
       def init
         @stopper = get_array('stopper', DEFAULT_SKIP, :upcase)
+        @classes = []
 
         @seq = get_key('sequences').map { |string, format|
-          [string = string.downcase, string.split(//), format]
+          @classes.concat(classes = string.downcase!.chars.to_a)
+          [string, classes, format]
         }
 
+        @classes.uniq!
+
         raise MissingConfigError.new(:sequences) if @seq.empty?
       end
 
@@ -115,42 +119,62 @@ class Lingo
       end
 
       def process_buffer
-        insert_sequences if @buffer.size > 1
-        forward_buffer
+        matches = []
+
+        if @buffer.size > 1
+          buf, map, seq, cls, unk = [], [], @seq, @classes, %w[#]
+
+          @buffer.each { |obj|
+            att = obj.is_a?(Word) && !obj.unknown? ? obj.attrs(false) : unk
+
+            (att &= cls).empty? ? find_seq(buf, map, seq, matches) : begin
+              buf << obj
+              map << att
+            end
+          }
+
+          find_seq(buf, map, seq, matches)
+        end
+
+        flush(@buffer.concat(matches))
       end
 
       private
 
-      def insert_sequences
-        matches, buf, seq = Hash.new { |h, k| h[k] = [] }, @buffer, @seq
+      def find_seq(buf, map, seq, matches)
+        return if buf.empty?
 
-        map = buf.map { |obj|
-          obj.is_a?(Word) && !obj.unknown? ? obj.attrs(false) : ['#']
-        }
+        match = Hash.new { |h, k| h[k] = [] }
+
+        map.replace(map.shift.product(*map))
+        map.map! { |i| i.join }
+        map.uniq!
 
-        map.shift.product(*map).map!(&:join).tap(&:uniq!).each { |q|
+        map.each { |q|
           seq.each { |string, classes, format|
             while pos = q.index(string, pos || 0)
-              inc('Anzahl erkannter Sequenzen')
-
-              fmt = format.dup
+              form = format.dup
 
               classes.each_with_index { |wc, i|
                 buf[pos + i].lexicals.find { |l|
-                  fmt.gsub!(i.succ.to_s, l.form) if l.attr == wc
+                  form.gsub!(i.succ.to_s, l.form) if l.attr == wc
                 } or break
               } or next
 
-              matches[pos] << fmt
-
-              pos += 1
+              match[pos += 1] << form
             end
           }
         }
 
-        matches.sort.each { |pos, forms| forms.tap(&:uniq!).each { |form|
-          @inserts << [pos, Word.new_lexical(form, WA_SEQUENCE, LA_SEQUENCE)]
-        } }
+        match.each_value { |forms|
+          forms.uniq!
+          forms.each { |form|
+            matches << Word.new_lexical(form, WA_SEQUENCE, LA_SEQUENCE)
+          }
+        }
+
+        buf.clear
+        map.clear
       end
 
     end

data/lib/lingo/attendee/stemmer/porter.rb

@@ -290,7 +290,7 @@ class Lingo
               case rule
                 when RULE_RE
                   cond, repl, goto = $1, $3, $4
-                  stem = word[/(.+)#{$2.downcase}$/, 1] or next
+                  stem = word[/(.+)#{Unicode.downcase($2)}$/, 1] or next
                 when GOTO_RE
                   goto = $1
                   break
@@ -324,7 +324,7 @@ class Lingo
               found, word = true, begin
                 stem[0...Integer(repl)]
               rescue ArgumentError
-                stem << repl.downcase
+                stem << Unicode.downcase(repl)
               end
 
               break

data/lib/lingo/attendee/stemmer.rb

@@ -41,7 +41,7 @@ class Lingo
 
       def process(obj)
         if obj.is_a?(Word) && obj.unknown?
-          stem = stem(obj.form.downcase, @all)
+          stem = stem(Unicode.downcase(obj.form), @all)
           obj.add_lexicals([Lexical.new(stem, @wc)]) if stem
         end
 

data/lib/lingo/attendee/synonymer.rb

@@ -78,19 +78,11 @@ class Lingo
       end
 
       def control(cmd, param)
-        report_on(cmd, @dic)
       end
 
       def process(obj)
         if obj.is_a?(Word) && !@skip.include?(obj.attr)
-          inc('Anzahl gesuchter Wörter')
-
-          unless (syn = @dic.find_synonyms(obj)).empty?
-            inc('Anzahl erweiteter Wörter')
-
-            obj.add_lexicals(syn.tap(&:uniq!))
-            add('Anzahl gefundener Synonyme', syn.size)
-          end
+          obj.add_lexicals(@dic.find_synonyms(obj))
         end
 
         forward(obj)

data/lib/lingo/attendee/text_reader.rb

@@ -115,15 +115,17 @@ class Lingo
         @filter   = get_key('filter', false)
         @progress = get_key('progress', false)
 
-        if @lir = get_key('records', get_key('lir-record-pattern', nil))  # DEPRECATE lir-record-pattern
-          @lir = @lir == true ? %r{^\[(\d+)\.\]} : Regexp.new(@lir)
-        end
+        @lingo.deprecate('lir-record-pattern', :records, self) if has_key?('lir-record-pattern')
+
+        @lir  = get_re('records', get_key('lir-record-pattern', nil), %r{^\[(\d+)\.\]})  # DEPRECATE lir-record-pattern
+        @cut  = get_re('fields', !!@lir, %r{^.+?:\s*})
+        @skip = get_re('skip', nil)
       end
 
       def control(cmd, param)
         if cmd == STR_CMD_TALK
           forward(STR_CMD_LIR, '') if @lir
-          @files.each(&method(:spool))
+          @files.each { |i| spool(i) }
         end
       end
 
@@ -132,24 +134,22 @@ class Lingo
       # Gibt eine Datei zeilenweise in den Ausgabekanal
       def spool(path)
         unless stdin = stdin?(path)
-          inc('Anzahl Dateien')
-          add('Anzahl Bytes', size = File.size(path))
-
-          size = nil unless @progress
+          size = File.size(path) if @progress
         end
 
         forward(STR_CMD_FILE, path)
 
         ShowProgress.new(self, size, path) { |progress|
           filter(path, stdin) { |line, pos|
-            inc('Anzahl Zeilen')
             progress[pos]
 
             line.chomp! if @chomp
+            next if line =~ @skip
 
             if line =~ @lir
               forward(STR_CMD_RECORD, $1)
             else
+              line.sub!(@cut, '') if @cut
               forward(line) unless line.empty?
             end
           }
@@ -159,13 +159,13 @@ class Lingo
       end
 
       def filter(path, stdin = stdin?(path))
-        io, block = stdin ? [
-          @lingo.config.stdin.set_encoding(ENC),
-          lambda { |line| yield line, 0 }
-        ] : [
-          File.open(path, 'rb', encoding: ENC),
+        io = stdin ?
+          @lingo.config.stdin.set_encoding(ENC) :
+          File.open(path, 'rb', encoding: ENC)
+
+        block = stdin || !@progress ?
+          lambda { |line| yield line, 0 } :
           lambda { |line| yield line, io.pos }
-        ]
 
         case @filter == true ? file_type(path, io) : @filter.to_s
           when /html/i then io = filter_html(io)
@@ -195,13 +195,16 @@ class Lingo
 
       def file_type(path, io)
         if Object.const_defined?(:FileMagic) && io.respond_to?(:rewind)
-          FileMagic.fm(:mime, simplified: true).buffer(io.read(256)).tap {
-            io.rewind
-          }
+          type = FileMagic.fm(:mime, simplified: true).buffer(io.read(256))
+          io.rewind
+          type
         elsif Object.const_defined?(:MIME) && MIME.const_defined?(:Types)
-          MIME::Types.of(path).first.tap { |type| type ? type.content_type :
-            warn('Filters not available. File type could not be determined.')
-          }
+          if type = MIME::Types.of(path).first
+            type.content_type
+          else
+            warn 'Filters not available. File type could not be determined.'
+            nil
+          end
         else
           warn "Filters not available. Please install `ruby-filemagic' or `mime-types'."
           nil
@@ -220,17 +223,27 @@ class Lingo
         Array(get_key('files', '-')).each { |path|
           stdin?(path) ? @files << path : add_files(path, *args)
         }
-
-        @files.map!(&File.method(:expand_path))
-        @files.uniq!
       end
 
       def add_files(path, glob, recursive = false)
-        Dir[path].sort!.each { |match|
-          File.directory?(match) ? recursive ? Find.find(match) { |entry|
-            @files << entry if File.file?(entry) && File.fnmatch?(glob, entry)
-          } : add_files(File.join(match, glob), glob) : @files << match
-        }.empty? and raise FileNotFoundError.new(path)
+        entries = Dir[path].sort!
+        raise FileNotFoundError.new(path) if entries.empty?
+
+        entries.each { |entry|
+          if File.directory?(entry)
+            if recursive
+              Find.find(entry) { |match|
+                if File.file?(match) && File.fnmatch?(glob, match)
+                  @files << File.expand_path(match)
+                end
+              }
+            else
+              add_files(File.join(entry, glob), glob)
+            end
+          else
+            @files << File.expand_path(entry)
+          end
+        }
       end
 
       class PDFFilter

data/lib/lingo/attendee/text_writer.rb

@@ -100,7 +100,6 @@ class Lingo
             if stdout?(@ext)
               @filename, @file = @ext, @lingo.config.stdout
             else
-              inc('Anzahl Dateien')
               @file = File.open(@filename = File.set_ext(param, ".#{@ext}"), 'w')
             end
 
@@ -116,14 +115,12 @@ class Lingo
             @no_sep = true
 
             unless @lir
-              inc('Anzahl Zeilen')
               @file.puts unless @no_puts
             end
           when STR_CMD_EOF
             flush_lir_buffer if @lir
 
             unless stdout?(@filename)
-              add('Anzahl Bytes', @file.size)
               @file.close
             end
         end
@@ -142,9 +139,9 @@ class Lingo
 
       def flush_lir_buffer
         unless @lir_rec_no.empty? || @lir_rec_buf.empty?
-          @file.print(*[@lir_rec_no, @lir_rec_buf.join(@sep), "\n"].tap { |buf|
-            @sep =~ /\n/ ? buf.insert(1, "\n").unshift('*') : buf.insert(1, '*')
-          })
+          buf = [@lir_rec_no, @lir_rec_buf.join(@sep), "\n"]
+          @sep =~ /\n/ ? buf.insert(1, "\n").unshift('*') : buf.insert(1, '*')
+          @file.print(*buf)
         end
 
         @lir_rec_no = ''