lingo 1.8.4.2 → 1.8.5

data/en/user-dic.txt

@@ -1,5 +1,2 @@
-#   
-#   Musterwörterbuch user-dic.txt als Vorlage
-#   für eigene Benutzerwörterbücher
-#
-#
+# Musterwörterbuch user-dic.txt als Vorlage
+# für eigene Benutzerwörterbücher

data/en.lang

@@ -40,62 +40,62 @@
 #           definiert sein, da es sonst noch nicht existiert!
 #
 
-#    lingo language definition
 ---
 language:
   name: 'Englisch'
 
   dictionary:
     databases:
-      #    Systemwörterbücher
+      # System dictionaries
       sys-dic: { name: en/lingo-dic.txt, txt-format: WordClass, separator: '=' }
-      sys-syn: { name: en/lingo-syn.txt, txt-format: KeyValue, separator: '=', def-wc: y }
-      sys-mul: { name: en/lingo-mul.txt, txt-format: SingleWord, use-lex: 'sys-dic', def-wc: m }
+      sys-syn: { name: en/lingo-syn.txt, txt-format: KeyValue,  separator: '=',    def-wc: y }
+      sys-mul: { name: en/lingo-mul.txt, txt-format: SingleWord, use-lex: sys-dic, def-wc: m }
       sys-irr: { name: en/lingo-irr.txt, txt-format: WordClass, separator: '=' }
       sys-wdn: { name: en/lingo-wdn.txt, txt-format: WordClass, separator: '=' }
-      #    Benutzerwörterbücher
-      usr-dic: { name: en/user-dic.txt, txt-format: WordClass, separator: '=' }
+
+      # User dictionaries
+      usr-dic: { name: en/user-dic.txt,  txt-format: WordClass, separator: '=' }
 
     compound:
-      min-word-size: "7"
-      min-part-size: "3"
-      max-parts: "5"
-      min-avg-part-size: "4"
-      append-wordclass: "+"
+      min-word-size:     '7'
+      min-part-size:     '3'
+      max-parts:         '5'
+      min-avg-part-size: '4'
+      append-wordclass:  '+'
       skip-sequences: [ xx ]
 
     suffix:
-      #    Suffixliste, Stand: *****ENGLISCH****
-      #    Suffixklasse:   s = Substantiv, a = Adjektiv, v = Verb, e = Eigenwort, f = Fugung
-      #    Suffixe je Klasse:   "<suffix>['/'<ersetzung>][ <suffix>['/'<ersetzung>]]"
-      - [s, " es s ves/f ves/fe ies/y "]
-      - [a, " er est r st ier/y iest/y "]
-      - [v, " d ed en es ing s ing/e "]
-      - [e, " s "]
-      - [f, ""]
+      # Suffixliste, Stand: *****ENGLISCH****
+      # Suffixklasse: s = Substantiv, a = Adjektiv, v = Verb, e = Eigenwort, f = Fugung
+      # Suffixe je Klasse: "<suffix>['/'<ersetzung>][ <suffix>['/'<ersetzung>]]"
+      - [s, 'es s ves/f ves/fe ies/y']
+      - [a, 'er est r st ier/y iest/y']
+      - [v, 'd ed en es ing s ing/e']
+      - [e, 's']
+      - [f, '']
 
   attendees:
     variator:
       variations:
-        - [ ieh, sch ]
-        - [ fec, see ]
-        - [ it,  st  ]
-        - [ fch, sch ]
-        - [ fp,  sp  ]
-        - [ f,   s   ]
-        - [ c,   e   ]
-        - [ ffc, sse ]
-        - [ ff,  ss  ]
-        - [ e,   c   ]
-        - [ ni,  m   ]
-        - [ feh, sch ]
-        - [ lt,  st  ]
-        - [ il,  st  ]
-        - [ ftc, ste ]
-        - [ ft,  st  ]
-        - [ fl,  st  ]
-        - [ li,  h   ]
-        - [ i,   s   ]
+        - [ieh, sch]
+        - [fec, see]
+        - [it,  st ]
+        - [fch, sch]
+        - [fp,  sp ]
+        - [f,   s  ]
+        - [c,   e  ]
+        - [ffc, sse]
+        - [ff,  ss ]
+        - [e,   c  ]
+        - [ni,  m  ]
+        - [feh, sch]
+        - [lt,  st ]
+        - [il,  st ]
+        - [ftc, ste]
+        - [ft,  st ]
+        - [fl,  st ]
+        - [li,  h  ]
+        - [i,   s  ]
 
     sequencer:
-      sequences: [ [AS, "2, 1"], [AK, "2, 1"], [AAK, "3, 1 2"], [AAS, "3, 1 2"] ]
+      sequences: [ [AS, '2, 1'], [AK, '2, 1'], [AAK, '3, 1 2'], [AAS, '3, 1 2'] ]

data/lib/lingo/app.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -27,19 +27,19 @@
 require 'json'
 require 'optparse'
 require 'shellwords'
-require 'sinatra/base'
-require 'sinatra/cookies'
+require 'sinatra/bells'
 
 class Lingo
 
-  class App < Sinatra::Base
+  class App < Sinatra::Bells
 
     class << self
 
       def init_app(file, *args, &block)
-        set :root, File.chomp_ext(file)
+        set_root(file)
         parse_options(*args, &block)
-        helpers Sinatra::Cookies
+
+        get('/about') { to_json(self.class, version: VERSION) }
       end
 
       def parse_options(lingo_options = false)
@@ -63,7 +63,11 @@ class Lingo
           o.on('-x',        'turn on the mutex lock (default is off)')       {     set :lock, true }
         }.parse!(argv)
 
+        abort banner unless argv.empty?
+
         ARGV.unshift(*yield) if block_given?
+      rescue OptionParser::ParseError => err
+        abort "#{err}\n#{banner}"
       end
 
       def rackup(name)

data/lib/lingo/attendee/abbreviator.rb

@@ -74,6 +74,7 @@ class Lingo
 
       def init
         set_dic
+        @abbr = nil
       end
 
       def control(cmd, param)

data/lib/lingo/attendee/decomposer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -80,6 +80,7 @@ class Lingo
       end
 
       def control(cmd, param)
+        # can control
       end
 
       def process(obj)

data/lib/lingo/attendee/multi_worder.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -86,7 +86,7 @@ class Lingo
         @combine = get_key('combine', false)
         @all     = @combine.is_a?(String) && @combine.downcase == 'all'
 
-        lex_src, lex_mod, d = nil, nil, @lingo.dictionary_config['databases']
+        lex_src, lex_mod, d = nil, nil, lingo.dictionary_config['databases']
 
         (mul_src = get_array('source')).each { |src|
           s, m = d[src].values_at('use-lex', 'lex-mode')
@@ -119,9 +119,8 @@ class Lingo
       def process_buffer
         unless form_at(0) == CHAR_PUNCT
           unless (res = check_multiword_key(3)).empty?
-            len = res.map { |r|
-              r.is_a?(Lexical) ? r.form.split(' ').size : r[/^\*(\d+)/, 1].to_i
-            }.sort!.reverse!
+            len = res.map { |r| r.is_a?(Lexical) ? r.form.count(' ') + 1 : r }
+            len.sort!.reverse!
 
             unless (max = len.first) > 3
               create_and_forward_multiword(3, res)
@@ -201,7 +200,7 @@ class Lingo
 
           seq.shift.product(*seq) { |key|
             @mul_dic.select(key.join(' '), mul)
-            break unless @all_keys || mul.empty?
+            break unless @all || mul.empty?
           } && mul.uniq!
 
           mul

data/lib/lingo/attendee/stemmer.rb

@@ -42,7 +42,7 @@ class Lingo
       def process(obj)
         if obj.is_a?(Word) && obj.unknown?
           stem = stem(Unicode.downcase(obj.form), @all)
-          obj.add_lexicals([Lexical.new(stem, @wc)]) if stem
+          obj << Lexical.new(stem, @wc) if stem
         end
 
         forward(obj)

data/lib/lingo/attendee/synonymer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -74,15 +74,17 @@ class Lingo
 
       def init
         set_dic
+        @com = !get_key('compound-parts', false)
         @skip = get_array('skip', WA_UNKNOWN, :upcase)
       end
 
       def control(cmd, param)
+        # can control
       end
 
       def process(obj)
         if obj.is_a?(Word) && !@skip.include?(obj.attr)
-          obj.add_lexicals(@dic.find_synonyms(obj))
+          obj.add_lexicals(@dic.find_synonyms(obj, [], @com))
         end
 
         forward(obj)

data/lib/lingo/attendee/text_reader.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -26,7 +26,7 @@
 
 require 'find'
 
-%w[filemagic mime/types hpricot pdf-reader].each { |lib|
+%w[filemagic mime/types nokogiri nuggets/file/which pdf-reader].each { |lib|
   begin
     require lib
   rescue LoadError
@@ -115,7 +115,7 @@ class Lingo
         @filter   = get_key('filter', false)
         @progress = get_key('progress', false)
 
-        @lingo.deprecate('lir-record-pattern', :records, self) if has_key?('lir-record-pattern')
+        lingo.deprecate('lir-record-pattern', :records, self) if has_key?('lir-record-pattern')
 
         @lir  = get_re('records', get_key('lir-record-pattern', nil), %r{^\[(\d+)\.\]})  # DEPRECATE lir-record-pattern
         @cut  = get_re('fields', !!@lir, %r{^.+?:\s*})
@@ -135,16 +135,12 @@ class Lingo
       def spool(path)
         forward(STR_CMD_FILE, path)
 
-        if stdin?(path)
-          io = @lingo.config.stdin.set_encoding(ENC)
-          io = StringIO.new(io.read) if @progress
-        else
-          io, name = File.open(path, 'rb', encoding: ENC), path
-        end
+        io = !stdin?(path) ? open_file(name = path) :
+          string_or_io(lingo.config.stdin.set_encoding(ENC))
 
-        ShowProgress.new(self, @progress && io.size, name) { |progress|
-          filter(io) { |line, pos|
-            progress[pos]
+        Progress.new(self, @progress && io.size, name) { |progress|
+          filter(io, path, progress) { |line, pos|
+            progress << pos
 
             line.chomp! if @chomp
             next if line =~ @skip
@@ -161,38 +157,56 @@ class Lingo
         forward(STR_CMD_EOF, path)
       end
 
-      def filter(io)
+      def filter(io, path, progress)
         block = @progress ?
           lambda { |line| yield line, io.pos } :
           lambda { |line| yield line, 0 }
 
-        case @filter == true ? file_type(path, io) : @filter.to_s
-          when /html/i then io = filter_html(io)
-          when /xml/i  then io = filter_html(io, true)
-          when /pdf/i  then filter_pdf(io, &block); return
+        io = case @filter == true ? file_type(io, path) : @filter.to_s
+          when 'pdftotext' then filter_pdftotext(io, path, progress)
+          when /html/i     then filter_html(io)
+          when /xml/i      then filter_html(io, true)
+          when /pdf/i      then filter_pdf(io, &block); return
+          else io
         end
 
-        io.each_line(&block) if io
+        io.each_line(&block)
       end
 
-      def filter_pdf(io, &block)
+      def filter_pdftotext(io, path, progress)
+        if cmd = File.which(name = 'pdftotext')
+          with_tempfile(name) { |tempfile|
+            pdf_path = stdin?(path) ? tempfile[:pdf, io] : path
+            system(cmd, '-q', pdf_path, txt_path = tempfile[:txt])
+
+            progress.init(File.size(txt_path)) if @progress
+            open_file(txt_path)
+          }
+        else
+          cancel_filter(:PDF, name, :command)
+        end
+      end
+
+      def filter_pdf(io)
         if Object.const_defined?(:PDF) && PDF.const_defined?(:Reader)
-          PDFFilter.filter(io, &block)
+          PDF::Reader.new(io).pages.each { |page| yield page.text }
         else
-          warn "PDF filter not available. Please install `pdf-reader'."
+          cancel_filter(:PDF, 'pdf-reader')
         end
       end
 
       def filter_html(io, xml = false)
-        if Object.const_defined?(:Hpricot)
-          Hpricot(io, xml: xml).inner_text
+        type = xml ? :XML : :HTML
+
+        if Object.const_defined?(:Nokogiri)
+          doc = Nokogiri.send(type, io, nil, ENC)
+          string_or_io(doc.children.map { |x| x.inner_text }.join)
         else
-          warn "#{xml ? 'X' : 'HT'}ML filter not available. Please install `hpricot'."
-          nil
+          cancel_filter(type, :nokogiri)
         end
       end
 
-      def file_type(path, io)
+      def file_type(io, path)
         if Object.const_defined?(:FileMagic) && io.respond_to?(:rewind)
           type = FileMagic.fm(:mime, simplified: true).buffer(io.read(256))
           io.rewind
@@ -201,26 +215,56 @@ class Lingo
           if type = MIME::Types.of(path).first
             type.content_type
           else
-            warn 'Filters not available. File type could not be determined.'
-            nil
+            cancel('Filters not available. File type could not be determined.')
           end
         else
-          warn "Filters not available. Please install `ruby-filemagic' or `mime-types'."
-          nil
+          cancel("Filters not available. Please install the `ruby-filemagic' or `mime-types' gem.")
         end
       end
 
+      def cancel_filter(type, name, what = :gem)
+        cancel("#{type} filter not available. Please install the `#{name}' #{what}.")
+      end
+
+      def cancel(msg)
+        throw(:cancel, msg)
+      end
+
       def stdin?(path)
         %w[STDIN -].include?(path)
       end
 
+      def string_or_io(io)
+        @progress ? StringIO.new(io.is_a?(String) ? io : io.read) : io
+      end
+
+      def open_file(path)
+        File.open(path, 'rb', encoding: ENC)
+      end
+
+      def with_tempfile(name)
+        require 'tempfile'
+
+        tempfiles = []
+
+        yield lambda { |ext, io = nil|
+          tempfiles << temp = Tempfile.new([name, ".#{ext}"])
+          temp.write(io.read) if io
+          temp.close
+          temp.path
+        }
+      ensure
+        tempfiles.each(&:unlink)
+      end
+
       def get_files
         args = [get_key('glob', '*.txt'), get_key('recursive', false)]
 
         @files = []
 
         Array(get_key('files', '-')).each { |path|
-          stdin?(path) ? @files << path : add_files(path, *args)
+          stdin?(path) ? @files << path :
+            add_files(File.expand_path(path), *args)
         }
       end
 
@@ -233,42 +277,18 @@ class Lingo
             if recursive
               Find.find(entry) { |match|
                 if File.file?(match) && File.fnmatch?(glob, match)
-                  @files << File.expand_path(match)
+                  @files << match
                 end
               }
             else
               add_files(File.join(entry, glob), glob)
             end
           else
-            @files << File.expand_path(entry)
+            @files << entry
           end
         }
       end
 
-      class PDFFilter
-
-        def self.filter(io, &block)
-          PDF::Reader.new.parse(io, new(&block))
-        end
-
-        def initialize(&block)
-          @block = block
-        end
-
-        def show_text(string, *params)
-          @block[string << '|']
-        end
-
-        alias_method :super_show_text,                 :show_text
-        alias_method :move_to_next_line_and_show_text, :show_text
-        alias_method :set_spacing_next_line_show_text, :show_text
-
-        def show_text_with_positioning(params, *)
-          params.each { |param| show_text(param) if param.is_a?(String) }
-        end
-
-      end
-
     end
 
     # For backwards compatibility.

data/lib/lingo/attendee/text_writer.rb

@@ -98,7 +98,7 @@ class Lingo
             @no_sep = true
 
             if stdout?(@ext)
-              @filename, @file = @ext, @lingo.config.stdout
+              @filename, @file = @ext, lingo.config.stdout
             else
               @file = File.open(@filename = File.set_ext(param, ".#{@ext}"), 'w')
             end