lingo 1.8.4.2 → 1.8.5

data/lib/lingo/language/grammar.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -46,7 +46,7 @@ class Lingo
       def initialize(config, lingo)
         @dic, @suggestions = Dictionary.new(config, lingo), []
 
-        lingo.deprecate(:compositum, :compound, self) if lingo.dictionary_config.has_key?('compositum')
+        lingo.deprecate(:compositum, :compound, self) if lingo.dictionary_config.key?('compositum')
 
         cfg = lingo.dictionary_config['compound'] ||
               lingo.dictionary_config['compositum']  # DEPRECATE compositum
@@ -79,16 +79,9 @@ class Lingo
       # find_compound arbeitet in verschiedenen Leveln, da die Methode auch rekursiv aufgerufen wird. Ein Level größer 1
       # entspricht daher einem rekursiven Aufruf
       def find_compound(str, level = 1, tail = false)
-        return permute_compound([[], [], ''], str, level, tail) if level != 1
-
-        (@_compound ||= {})[str] ||= permute_compound(
-          com = Word.new(str, WA_UNKNOWN), str, level, tail
-        ) { |lex|
-          com.attr = WA_COMPOUND
-          com.lexicals = lex.each { |l|
-            l.attr += @append_wc unless l.attr == LA_COMPOUND
-          }
-        }
+        level == 1 ? (@_compound ||= {})[str] ||=
+          permute_compound(Word.new(str, WA_UNKNOWN), str, level, tail) :
+          permute_compound([[], [], ''],              str, level, tail)
       end
 
       private
@@ -115,7 +108,9 @@ class Lingo
             }
           end
 
-          block_given? ? yield(lex) : ret = res if !lex.empty? &&
+          level > 1 ? ret = res : ret.identify(lex.each { |l|
+            l.attr += @append_wc unless l.attr == LA_COMPOUND
+          }, WA_COMPOUND) if !lex.empty? &&
             sta.size              <= @max_parts         &&
             sta.min               >= @min_part_size     &&
             str.length / sta.size >= @min_avg_part_size &&
@@ -131,16 +126,16 @@ class Lingo
       def test_compound(fstr, infix, bstr, level = 1, tail = false)
         sta, seq, empty = [fstr.length, bstr.length], %w[? ?], [[], [], '']
 
-        if !Lingo.sort!(blex = @dic.select_with_suffix(bstr)).empty?
+        if !(blex = @dic.select_with_suffix(bstr)).empty?
           # 1. Word w/ suffix
           bform, seq[1] = tail ? bstr : blex.first.form, blex.first.attr
-        elsif tail && !Lingo.sort!(blex = @dic.select_with_infix(bstr)).empty?
+        elsif tail && !(blex = @dic.select_with_infix(bstr)).empty?
           # 2. Word w/ infix, unless tail part
           bform, seq[1] = bstr, blex.first.attr
         elsif infix == '-'
           blex, bsta, bseq = find_compound(bstr, level + 1, tail)
 
-          if !Lingo.sort!(blex).empty?
+          if !blex.empty?
             # 3. Compound
             bform, seq[1], sta[1..-1] = blex.first.form, bseq, bsta
           else
@@ -151,13 +146,13 @@ class Lingo
           return empty
         end
 
-        if !Lingo.sort!(flex = @dic.select_with_infix(fstr)).empty?
+        if !(flex = @dic.select_with_infix(fstr)).empty?
           # 1. Word w/ infix
           fform, seq[0] = fstr, flex.first.attr
         else
           flex, fsta, fseq = find_compound(fstr, level + 1, true)
 
-          if !Lingo.sort!(flex).empty?
+          if !flex.empty?
             # 2. Compound
             fform, seq[0], sta[0..0] = flex.first.form, fseq, fsta
           elsif infix == '-'
@@ -168,14 +163,15 @@ class Lingo
           end
         end
 
-        { flex => fform, blex => bform }.each { |a, f|
-          a.each { |l| l.src ||= f }
+        forms = [[flex, fform], [blex, bform]].each { |ary|
+          ary.shift.each { |lex| lex.src ||= ary.first }
         }
 
-        flex.concat(blex).delete_if { |l| l.attr == LA_COMPOUND }.
-          push(Lexical.new(fform + infix + bform, LA_COMPOUND))
+        flex.concat(blex).delete_if { |lex| lex.attr == LA_COMPOUND }
 
-        [Lingo.sort!(flex), sta, seq.join]
+        [forms.shift.product(*forms).map { |front, back|
+          Lexical.new(front + infix + back, LA_COMPOUND)
+        }.concat(flex), sta, seq.join]
       end
 
     end

data/lib/lingo/language/lexical.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -36,6 +36,10 @@ class Lingo
 
     class Lexical < WordForm
 
+      def attr?(attr)
+        self.attr.start_with?(attr)
+      end
+
       def <=>(other)
         return 1 unless other.is_a?(self.class)
 

data/lib/lingo/language/lexical_hash.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -34,6 +34,8 @@ class Lingo
 
     class LexicalHash
 
+      KEY_REF_RE = %r{\A#{Regexp.escape(Database::KEY_REF)}(\d+)\z}o
+
       def self.open(*args)
         yield lexical_hash = new(*args)
       ensure
@@ -41,7 +43,6 @@ class Lingo
       end
 
       def initialize(id, lingo)
-        @wc  = lingo.database_config(id).fetch('def-wc', LA_UNKNOWN)
         @src = Database.open(id, lingo)
       end
 
@@ -50,22 +51,16 @@ class Lingo
       end
 
       def [](key)
-        rec = @src[key = Unicode.downcase(key)] or return
+        rec = @src[Unicode.downcase(key)] or return
 
         res = rec.map { |str|
-          case str
-            when /^\*\d+$/           then str
-            when /^#(.)$/            then Lexical.new(key, $1)
-            when /^([^#]+?)\s*#(.)$/ then Lexical.new($1,  $2)
-            when /^([^#]+)$/         then Lexical.new($1, @wc)
-            else                          str
+          str =~ KEY_REF_RE ? $1.to_i : begin
+            k, *w = str.split('#')
+            Lexical.new(k.strip, w)
           end
         }
 
-        res.compact!
-        res.sort!
         res.uniq!
-
         res
       end
 

data/lib/lingo/language/token.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -37,6 +37,15 @@ class Lingo
 
     class Token < WordForm
 
+      def self.clean(attr)
+        attr.sub(/:.*/, '')
+      end
+
+      def initialize(*)
+        super
+        @attr = self.class.clean(attr)
+      end
+
       def word?
         attr == TA_WORD
       end

data/lib/lingo/language/word.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -43,6 +43,20 @@ class Lingo
           new_lexicals(form, attr, Lexical.new(form, lex_attr))
         end
 
+        def new_compound_head(lex, attr = WA_UNSET)
+          form, head_lex = nil, []
+
+          lex.reverse_each { |l|
+            src =  l.src ||= l.form
+            form ||= src
+            form  != src ? break : head_lex.unshift(l.dup)
+          }
+
+          head_lex.each { |l| l.attr = l.attr[/\w+/] }.uniq!
+
+          new_lexicals(form, attr, head_lex) if form
+        end
+
       end
 
       # Exakte Representation der originären Zeichenkette, so wie sie im Satz
@@ -63,6 +77,8 @@ class Lingo
         @lexicals = []
       end
 
+      attr_writer :lexicals
+
       def lexicals(compound_parts = true)
         if !compound_parts && attr == WA_COMPOUND
           @lexicals.select { |lex| lex.attr == LA_COMPOUND }
@@ -71,31 +87,30 @@ class Lingo
         end
       end
 
-      def lexicals=(lex)
-        if lex.is_a?(Array)
-          @lexicals = Lingo.sort(lex).uniq
-        else
-          raise TypeError, "wrong argument type #{lex.class} (expected Array)"
-        end
+      def add_lexicals(lex)
+        @lexicals.concat(lex - @lexicals)
       end
 
-      def add_lexicals(lex)
-        unless lex.empty?
-          @lexicals.concat(lex).uniq!
-          Lingo.sort!(@lexicals)
-        end
+      def attr?(*attr)
+        !(attrs & attr).empty?
       end
 
       def attrs(compound_parts = true)
         lexicals(compound_parts).map { |i| i.attr }
       end
 
-      def parts
-        1
+      def genders(compound_parts = true)
+        lexicals(compound_parts).map { |i| i.gender }
       end
 
-      def min_part_size
-        form.length
+      def identify(lex, wc = nil)
+        return self if lex.empty?
+
+        self.lexicals = lex
+        self.attr = wc ||= attr?(LA_COMPOUND) ? WA_COMPOUND : WA_IDENTIFIED
+        self.head = self.class.new_compound_head(lex) if wc == WA_COMPOUND
+
+        self
       end
 
       # Gibt genau die Grundform der Wortklasse zurück, die der RegExp des Übergabe-Parameters
@@ -103,11 +118,8 @@ class Lingo
       def get_class(wc_re)
         wc_re = Regexp.new(wc_re) unless wc_re.is_a?(Regexp)
 
-        unless lexicals.empty?
+        lexicals.empty? ? attr =~ wc_re ? [self] : [] :
           lexicals.select { |lex| lex.attr =~ wc_re }
-        else
-          attr =~ wc_re ? [self] : []
-        end
       end
 
       def norm
@@ -126,9 +138,9 @@ class Lingo
         lex = get_class(wc_re).first and lex.form.count(' ') + 1
       end
 
-      def <<(*other)
-        other.flatten!
-        lexicals.concat(other)
+      def <<(*lex)
+        lex.flatten!
+        @lexicals.concat(lex)
         self
       end
 

data/lib/lingo/language/word_form.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2013 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -36,12 +36,13 @@ class Lingo
 
       include Comparable
 
-      attr_accessor :form, :attr, :src
-
       def initialize(form, attr = WA_UNSET, src = nil)
+        attr, @gender = attr
         @form, @attr, @src = form || '', attr || '', src
       end
 
+      attr_accessor :form, :attr, :gender, :src, :head
+
       def unknown?
         [WA_UNKNOWN, WA_UNKMULPART].include?(attr)
       end
@@ -55,7 +56,7 @@ class Lingo
       end
 
       def to_a
-        [form, attr]
+        [form, attr, gender].compact
       end
 
       def to_s

data/lib/lingo/{show_progress.rb → progress.rb}

@@ -26,53 +26,66 @@
 
 class Lingo
 
-  class ShowProgress
+  class Progress
 
-    def initialize(obj, max, name = nil, doit = true, text = 'progress', nl = true)
-      return yield self unless max && doit
+    def initialize(obj, max, name = nil, doit = true, text = 'progress', nl = true, &block)
+      @doit, @out = doit, obj.lingo.config.stderr
 
-      @out = obj.instance_variable_get(:@lingo).config.stderr
+      if max && doit
+        format = ' [%3d%%]'
+        length = (format % 0).length
 
-      # To get the length of the formatted string we have
-      # to actually substitute the placeholder.
-      fmt = ' [%3d%%]'
-      len = (fmt % 0).length
+        erase = "\b" * length
+        @format = format + erase
 
-      # Now we know how far to "go back" to
-      # overwrite the formatted string...
-      back = "\b" * len
+        print name, ': ' if name
+        print text
 
-      @fmt = fmt       + back
-      @clr = ' ' * len + back
+        init(max)
 
-      print name, ': ' if name
+        msg = %w[done aborted]
+      end
+
+      suc = false
+
+      res = catch(:cancel) {
+        int = trap(:INT) { throw(:cancel) }
 
-      @rat, @cnt, @next = max / 100.0, 0, 0
-      print text
-      step
+        begin
+          yield self
+        ensure
+          trap(:INT, &int)
+        end
 
-      yield self
+        suc = true
+        nil
+      }
 
-      print "#{@clr} done."
-      print "\n" if nl
+      print ' ' * length + erase, ' ', msg[suc ? 0 : 1], '.' if msg
+      print "\n" if msg && res
+
+      print Array(res).join("\n") if res
+      print "\n" if nl && (msg || res)
     end
 
-    def [](value)
-      if defined?(@cnt)
-        @cnt = value
-        step if @cnt >= @next
+    def init(max, doit = @doit)
+      if max && doit
+        @ratio, @next = max / 100.0, 0
+        self << @count = 0
       end
     end
 
-    private
+    def <<(value)
+      if defined?(@count) && (@count = value) >= @next
+        percent = @count / @ratio
+        @next = (percent + 1) * @ratio
 
-    def step
-      percent = @cnt / @rat
-      @next = (percent + 1) * @rat
-
-      print @fmt % percent if percent.finite?
+        print @format % percent if percent.finite?
+      end
     end
 
+    private
+
     def print(*args)
       @out.print(*args)
     end

data/lib/lingo/srv/lingosrv.cfg

@@ -10,5 +10,5 @@ meeting:
     - sequencer:     { stopper: 'PUNC,OTHR' }
     - synonymer:     { skip: '?,t', source: sys-syn }
 
-    - vector_filter: { debug: 'true', prompt: '' }
+    - vector_filter: { debug: 'true', prompt: '', preamble: false }
     - text_writer:   { ext: STDOUT, sep: "\n" }

data/lib/lingo/srv.rb

@@ -42,6 +42,11 @@ class Lingo
     get('/')  { doit }
     post('/') { doit }
 
+    post('/raw') {
+      content_type :text
+      LINGO.talk(request.body, true)
+    }
+
     def doit
       to_json(q = params[:q], case q
         when String then talk(q)
@@ -53,14 +58,14 @@ class Lingo
       r = LINGO.talk(q) unless q.empty?
       return r unless r && SRC_SEP
 
-      r.inject(Hash.new { |h, k| h[k] = [] }) { |h, s|
-        a, b = s.split(SRC_SEP, 2); h[b] << a; h
+      s = Hash.new { |h, k| h[k] = [] }
+
+      r.each { |t|
+        a, b = t.split(SRC_SEP, 2)
+        s[b] << a
       }
-    end
 
-    if ENV['LINGO_NO_SORT']
-      alias_method :_talk, :talk
-      def talk(q) _talk(q).to_a end
+      ENV['LINGO_NO_SORT'] ? s.to_a : s
     end
 
   end

data/lib/lingo/version.rb

@@ -4,13 +4,13 @@ class Lingo
 
     MAJOR = 1
     MINOR = 8
-    TINY  = 4
+    TINY  = 5
 
     class << self
 
       # Returns array representation.
       def to_a
-        [MAJOR, MINOR, TINY] << '2'
+        [MAJOR, MINOR, TINY]
       end
 
       # Short-cut for version string.

data/lib/lingo/web/lingoweb.cfg

@@ -10,5 +10,5 @@ meeting:
     - sequencer:     { stopper: 'PUNC,OTHR' }
     - synonymer:     { skip: '?,t', source: sys-syn }
 
-    - vector_filter: { debug: 'true', prompt: '' }
+    - vector_filter: { debug: 'true', prompt: '', preamble: false }
     - text_writer:   { ext: STDOUT, sep: "\n" }

data/lib/lingo/web/views/index.erb

@@ -41,18 +41,18 @@
     <form action="<%= url_for '/' %>" method="post">
       <div>
         <fieldset><legend><strong><%= t 'Input', 'Eingabe', 'ввод данных' %></strong></legend>
-          <textarea name="q" rows="20" cols="50"><%= @q %></textarea>
+          <textarea name="q" rows="20" cols="50"><%=h @q %></textarea>
         </fieldset>
 
         <fieldset><legend><strong><%= t 'Output', 'Ausgabe', 'вывод данных' %></strong></legend>
-          <textarea readonly="readonly" rows="20" cols="50"><%= @r %></textarea>
+          <textarea readonly="readonly" rows="20" cols="50"><%=h @r %></textarea>
         </fieldset>
 
         <br />
 
         <strong><%= t 'Language', 'Sprache', 'язык' %></strong> = <select name="l">
         <% for l in L %>
-          <option value="<%= l %>"<%= ' selected="selected"' if l == @l %>><%= l %></option>
+          <option value="<%=h l %>"<%= ' selected="selected"' if l == @l %>><%=h l %></option>
         <% end %>
         </select>
 
@@ -144,7 +144,7 @@
   <div id="footer">
     <em>powered by</em> <a href="http://lex-lingo.de">Lingo</a>
     <em>and</em> <a href="http://www.sinatrarb.com">Sinatra</a>
-    -- <strong>v<%= Lingo::VERSION %></strong>
+    — <strong>v<%=h Lingo::VERSION %></strong>
   </div>
 </body>
 </html>

data/lib/lingo/web.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -50,8 +50,8 @@ class Lingo
 
     LINGO = Hash.new { |h, k| h[k] = Lingo.call(cfg, ['-l', k]) }
 
-    CFG, s = '', StringScanner.new('')
-    c = lambda { |n| %Q{<span style="color:#{n}">#{s.matched}</span>} }
+    CFG, s, h = '', StringScanner.new(''), ERB::Util.method(:h)
+    c = lambda { |n| %Q{<span style="color:#{n}">#{h[s.matched]}</span>} }
 
     File.foreach(cfg) { |line|
       s.string = line.chomp
@@ -79,13 +79,11 @@ class Lingo
     }
 
     before do
-      @hl = if v = params[:hl] || cookies[:hl] || env['HTTP_ACCEPT_LANGUAGE']
+      @hl = if v = params[:hl] || env['HTTP_ACCEPT_LANGUAGE']
         v = v.split(',').map { |l| l.split('-').first.strip }
         (v & HL).first
       end || HL.first
 
-      cookies[:hl] = @hl unless cookies[:hl] == @hl
-
       @q = params[:q]
       @l = params[:l] || @hl
       @l = L.first unless L.include?(@l)

data/lib/lingo.rb

@@ -65,14 +65,6 @@ class Lingo
 
   class << self
 
-    if ENV['LINGO_NO_SORT']
-      def sort!(x) x       end
-      def sort(x)  x.dup   end
-    else
-      def sort!(x) x.sort! end
-      def sort(x)  x.sort  end
-    end
-
     def talk(*args)
       new(*args).talk
     end
@@ -242,18 +234,18 @@ class Lingo
 
   def config
     @config ||= Config.new(*@config_args)
+  rescue => err
+    raise ConfigLoadError.new(err)
   end
 
   def dictionary_config
     @dictionary_config ||= config['language/dictionary']
-  rescue => err
-    raise ConfigLoadError.new(err)
   end
 
   def database_config(id)
     dictionary_config['databases'][id].tap { |cfg|
       raise NoDatabaseConfigError.new(id) unless cfg
-      raise InvalidDatabaseConfigError.new(id) unless cfg.has_key?('name')
+      raise InvalidDatabaseConfigError.new(id) unless cfg.key?('name')
     }
   end
 
@@ -327,7 +319,7 @@ require_relative 'lingo/error'
 require_relative 'lingo/debug'
 require_relative 'lingo/config'
 require_relative 'lingo/agenda_item'
-require_relative 'lingo/show_progress'
+require_relative 'lingo/progress'
 require_relative 'lingo/database'
 require_relative 'lingo/language'
 require_relative 'lingo/attendee'

data/lingo.cfg

@@ -52,7 +52,7 @@ meeting:
     ########################################
     # Datenstrom anzeigen
     #
-#   - debugger:        { eval: 'true', ceval: 'cmd!="EOL"', prompt: 'lex:) ' }
+#   - debugger:        { eval: 'true', ceval: 'obj.cmd!="EOL"', prompt: 'lex:) ' }
 
 
     ########################################

data/lir.cfg

@@ -57,7 +57,7 @@ meeting:
     ########################################
     # Datenstrom anzeigen
     #
-#   - debugger:        { eval: 'true', ceval: 'cmd!="EOL"', prompt: 'lex:) ' }
+#   - debugger:        { eval: 'true', ceval: 'obj.cmd!="EOL"', prompt: 'lex:) ' }
 
 
     ########################################