lingo 1.8.5 → 1.8.6

data/test/test_helper.rb

@@ -3,7 +3,7 @@
 require 'test/unit'
 require 'lingo'
 
-class LingoTestCase <  Test::Unit::TestCase
+class LingoTestCase < Test::Unit::TestCase
 
   unless const_defined?(:TEST_FILE)
     TEST_FILE = 'test/de/test.txt'
@@ -20,12 +20,18 @@ class LingoTestCase <  Test::Unit::TestCase
     [a || '', b || '', *c]
   end
 
+  def li(t, o)
+    ["#{t}\r\n", o]
+  end
+
   def ai(t)
-    Lingo::AgendaItem.new(*split(t))
+    i = t.split('|')
+    i.unshift(i.shift.to_sym)
   end
 
   def tk(t)
-    Lingo::Language::Token.new(*split(t, /\|(?=[A-Z])/))
+    a, b, *c = split(t, /\|(?=[A-Z\d])/)
+    Lingo::Language::Token.new(a, b, *c.map(&:to_i))
   end
 
   def lx(t)
@@ -55,8 +61,15 @@ class AttendeeTestCase < LingoTestCase
     @lingo.reset
 
     list = [{ @attendee => cfg }]
-    list.unshift 'TestSpooler' => { 'out' => 'input',  'input'  => input       } if input
-    list.push    'TestDumper'  => { 'in'  => 'output', 'output' => output = [] } if expect
+
+    list.unshift('TestSpooler' => {
+      'out'    => 'input',
+      'input'  => input,
+      'pos'    => @attendee == 'Tokenizer' }) if input
+
+    list.push('TestDumper' => {
+      'in'     => 'output',
+      'output' => output = [] }) if expect
 
     @lingo.invite(list)
     @lingo.start
@@ -74,32 +87,31 @@ class Lingo
 
     class TestSpooler < self
 
-      protected
-
       def init
-        @input = get_key('input')
+        @input, @pos = get_key('input'), get_key('pos', nil) && 0
       end
 
-      def control(cmd, param)
-        @input.each { |i| forward(i) } if cmd == STR_CMD_TALK
+      def control(cmd)
+        if cmd == :TALK
+          @input.each { |i| i.is_a?(Array) ? command(*i) :
+            @pos ? forward(i, @pos += i.bytesize) : forward(i) }
+        end
       end
 
     end
 
     class TestDumper < self
 
-      protected
-
       def init
         @output = get_key('output')
       end
 
-      def control(cmd, param)
-        @output << AgendaItem.new(cmd, param)
+      def control(*args)
+        @output.push(args)
       end
 
-      def process(obj)
-        @output << obj
+      def process(obj, *rest)
+        @output.push(rest.empty? ? obj : rest.unshift(obj))
       end
 
     end
@@ -115,4 +127,5 @@ class Lingo
     end
 
   end
+
 end

data/test/ts_language.rb

@@ -23,8 +23,6 @@ class TestLexicalHash < LingoTestCase
     $stderr = old_stderr
   end
 
-  # TODO: Crypt testen...
-
   def test_cache
     lh('sys-dic') { |ds|
       assert_equal([lx('regen|s|m'), lx('regen|s|n'), lx('regen|v'), lx('rege|a')], ds['regen'])
@@ -38,11 +36,9 @@ class TestLexicalHash < LingoTestCase
 
     lh(id) { |ds| assert_equal([lx('substantiv|s')], ds['substantiv']) }
 
-    # Keine Store-Datei vorhanden, nur Text vorhanden
     File.delete(*Dir["#{Lingo.find(:store, txt_file)}.*"])
     lh(id) { |ds| assert_equal([lx('substantiv|s')], ds['substantiv']) }
 
-    # Store vorhanden, aber Text ist neuer
     lh(id) { |ds| assert_equal([lx('substantiv|s')], ds['substantiv']) }
   end
 
@@ -59,8 +55,8 @@ class TestLexicalHash < LingoTestCase
       assert_equal([lx('abelscher ring ohne nullteiler|m')], ds['abelscher ring ohne nullteiler'])
       assert_equal([4], ds['abelscher ring ohne'])
       assert_equal([lx('alleinreisende frau|m')], ds['alleinreisend frau'])
-      assert_equal([lx('abschaltbarer leistungshalbleiter|m')], ds['abschaltbar leistungshalbleiter'])
-      assert_equal(nil, ds['abschaltbarer leistungshalbleiter'])
+      assert_equal([lx('abschaltbarer leistungshalbleiter|m')], ds['abschaltbarer leistungshalbleiter'])
+      assert_equal(nil, ds['abschaltbar leistungshalbleiter'])
     }
   end
 
@@ -113,12 +109,6 @@ class TestDictionary < LingoTestCase
   end
 
   def test_params
-    # Keine Sprach-Konfiguration angegeben
-    #assert_raise(RuntimeError) {
-    #  Lingo::Language::Dictionary.new({ 'source' => %w[sys-dic] }, @lingo)
-    #}
-
-    # Falsche Parameter angegeben (Pflichtparameter ohne Defaultwert)
     assert_raise(ArgumentError) {
       Lingo::Language::Dictionary.new({ 'course' => %w[sys-dic] }, @lingo)
     }
@@ -142,39 +132,27 @@ class TestDictionary < LingoTestCase
 
   def test_select_two_sources_mode_first
     ld('source' => %w[sys-dic tst-dic], 'mode' => 'first') { |dic|
-      # in keiner Quelle vorhanden
       assert_equal([], dic.select('hasennasen'))
-      # nur in erster Quelle vorhanden
       assert_equal([lx('knaller|s')], dic.select('knaller'))
-      # nur in zweiter Quelle vorhanden
       assert_equal([lx('super indexierungssystem|m')], dic.select('lex-lingo'))
-      # in beiden Quellen vorhanden
       assert_equal([lx('a-dur|s|m'), lx('a-dur|s|n')], dic.select('a-dur'))
     }
   end
 
   def test_select_two_sources_mode_first_flipped
     ld('source' => %w[tst-dic sys-dic], 'mode' => 'first') { |dic|
-      # in keiner Quelle vorhanden
       assert_equal([], dic.select('hasennasen'))
-      # nur in erster Quelle vorhanden
       assert_equal([lx('knaller|s')], dic.select('knaller'))
-      # nur in zweiter Quelle vorhanden
       assert_equal([lx('super indexierungssystem|m')], dic.select('lex-lingo'))
-      # in beiden Quellen vorhanden
       assert_equal([lx('b-dur|s')], dic.select('a-dur'))
     }
   end
 
   def test_select_two_sources_mode_all
     ld('source' => %w[sys-dic tst-dic], 'mode' => 'all') { |dic|
-      # in keiner Quelle vorhanden
       assert_equal([], dic.select('hasennasen'))
-      # nur in erster Quelle vorhanden
       assert_equal([lx('knaller|s')], dic.select('knaller'))
-      # nur in zweiter Quelle vorhanden
       assert_equal([lx('super indexierungssystem|m')], dic.select('lex-lingo'))
-      # in beiden Quellen vorhanden
       assert_equal([lx('a-dur|s|m'), lx('a-dur|s|n'), lx('b-dur|s')], dic.select('a-dur'))
       assert_equal([lx('aas|s|n'), lx('aas|s')], dic.select('aas'))
     }
@@ -182,15 +160,11 @@ class TestDictionary < LingoTestCase
 
   def test_select_two_sources_mode_default
     ld('source' => %w[sys-dic tst-dic]) { |dic|
-      # in keiner Quelle vorhanden
       assert_equal([], dic.select('hasennasen'))
-      # nur in erster Quelle vorhanden
       assert_equal([lx('knaller|s')], dic.select('knaller'))
-      # nur in zweiter Quelle vorhanden
       assert_equal([lx('super indexierungssystem|m')], dic.select('lex-lingo'))
       assert_equal([lx('wirkungsort|s'), lx('wirkung|s+'), lx('ort|s+')], dic.select('wirkungsort'))
       assert_equal([lx('zettelkatalog|k'), lx('zettel|s+'), lx('katalog|s+')], dic.select('zettelkatalog'))
-      # in beiden Quellen vorhanden
       assert_equal([lx('a-dur|s|m'), lx('a-dur|s|n'), lx('b-dur|s')], dic.select('a-dur'))
       assert_equal([lx('aas|s|n'), lx('aas|s')], dic.select('aas'))
     }
@@ -207,7 +181,7 @@ class TestDictionary < LingoTestCase
 
   def test_infix_lexicals
     ld('source' => %w[sys-dic]) { |dic|
-      assert_equal( [lx('information|f'), lx('informationsen|f')], ax(dic, 'informations', :infix))
+      assert_equal([lx('information|f')], ax(dic, 'informations', :infix))
     }
   end
 
@@ -222,7 +196,7 @@ class TestDictionary < LingoTestCase
 
   def test_select_with_infix
     ld('source' => %w[sys-dic]) { |dic|
-      assert_equal( [lx('information|f'), lx('informationsen|f')], ax(dic, 'informations', :infix))
+      assert_equal([lx('information|f')], ax(dic, 'informations', :infix))
     }
   end
 
@@ -250,10 +224,6 @@ class TestGrammar < LingoTestCase
     @lingo = Lingo.new
   end
 
-  def test_params
-    # Die gleichen Fälle wie bei Dictionary, daher nicht notwendig
-  end
-
   def test_cache
     lg { |gra|
       assert_equal(
@@ -294,7 +264,7 @@ class TestGrammar < LingoTestCase
       )
 
       assert_equal(
-        wd('benutzerforschung|KOM', 'benutzerforschung|k', 'benutzen|v+', 'erforschung|s+'),
+        wd('benutzerforschung|KOM', 'benutzerforschung|k', 'benutzer|s+', 'forschung|s+'),
         gra.find_compound('benutzerforschung')
       )
 
@@ -318,64 +288,53 @@ class TestGrammar < LingoTestCase
         gra.find_compound('titelbestandsbestände')
       )
 
-      # hinterer Teil ist ein Wort mit Suffix
       assert_equal(
         wd('hasenbraten|KOM', 'hasenbraten|k', 'hase|s+', 'braten|v+'),
         gra.find_compound('hasenbraten')
       )
 
-      # hinterer Teil ist ein Wort mit Infix ohne Schwanz
       assert_equal(
         wd('nasenlaufen|KOM', 'nasenlaufen|k', 'nase|s+', 'laufen|v+'),
         gra.find_compound('nasenlaufen')
       )
 
-      # hinterer Teil ist ein Wort mit Infix mit Schwanz
       assert_equal(
         wd('nasenlaufens|KOM', 'nasenlaufen|k', 'nase|s+', 'laufen|v+'),
         gra.find_compound('nasenlaufens')
       )
 
-      # hinterer Teil ist ein Kompositum nach Bindestrich
       assert_equal(
         wd('arrafat-nachfolgebedarf|KOM', 'arrafat-nachfolgebedarf|k', 'arrafat|x+', 'nachfolge|s+', 'bedarf|s+'),
         gra.find_compound('arrafat-nachfolgebedarf')
       )
 
-      # hinterer Teil ist ein TakeItAsIs nach Bindestrich
       assert_equal(
         wd('nachfolge-arrafat|KOM', 'nachfolge-arrafat|k', 'nachfolge|s+', 'arrafat|x+'),
         gra.find_compound('nachfolge-arrafat')
       )
 
-      # vorderer Teil ist ein Wort mit Suffix => siehe Hasenbraten
-      # vorderer Teil ist ein Kompositum
       assert_equal(
         wd('morgenonkelmantel|KOM', 'morgenonkelmantel|k', 'morgen|w+', 'morgen|s+', 'onkel|s+', 'mantel|s+'),
         gra.find_compound('morgenonkelmantel')
       )
 
-      # vorderer Teil ist ein TakeItAsIs vor Bindestrich / bindestrichversion
       assert_equal(
         wd('arrafat-nachfolger|KOM', 'arrafat-nachfolger|k', 'arrafat|x+', 'nachfolger|s+'),
         gra.find_compound('arrafat-nachfolger')
       )
 
-      # bindestrichversion zwei-teilig
       assert_equal(
         wd('cd-rom-technologie|KOM', 'cd-rom-technologie|k', 'cd-rom|s+|f', 'cd-rom|s+|m', 'technologie|s+|f'),
         gra.find_compound('cd-rom-technologie')
       )
 
-      # bindestrichversion drei-teilig
       assert_equal(
         wd('albert-ludwigs-universität|KOM', 'albert-ludwigs-universität|k', 'albert|e+', 'ludwig|e+', 'universität|s+'),
         gra.find_compound('albert-ludwigs-universität')
       )
 
-      # normal mit suggestion
       assert_equal(
-        wd('benutzerforschung|KOM', 'benutzerforschung|k', 'benutzen|v+', 'erforschung|s+'),
+        wd('benutzerforschung|KOM', 'benutzerforschung|k', 'benutzer|s+', 'forschung|s+'),
         gra.find_compound('benutzerforschung')
       )
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lingo
 version: !ruby/object:Gem::Version
-  version: 1.8.5
+  version: 1.8.6
 platform: ruby
 authors:
 - John Vorhauer
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-02 00:00:00.000000000 Z
+date: 2015-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cyclops
@@ -17,34 +17,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.0'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: nuggets
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
@@ -119,16 +113,22 @@ dependencies:
   name: hen
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -144,7 +144,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -189,27 +189,34 @@ files:
 - bin/lingoctl
 - bin/lingosrv
 - bin/lingoweb
-- de.lang
-- de/lingo-abk.txt
-- de/lingo-dic.txt
-- de/lingo-mul.txt
-- de/lingo-syn.txt
-- de/test_dic.txt
-- de/test_gen.txt
-- de/test_mu2.txt
-- de/test_mul.txt
-- de/test_sgw.txt
-- de/test_syn.txt
-- de/user-dic.txt
-- en.lang
-- en/lingo-dic.txt
-- en/lingo-irr.txt
-- en/lingo-mul.txt
-- en/lingo-syn.txt
-- en/lingo-wdn.txt
-- en/user-dic.txt
+- config/lingo-call.cfg
+- config/lingo.cfg
+- config/lir.cfg
+- dict/de/lingo-abk.txt
+- dict/de/lingo-dic.txt
+- dict/de/lingo-mul.txt
+- dict/de/lingo-syn.txt
+- dict/de/test_dic.txt
+- dict/de/test_gen.txt
+- dict/de/test_mu2.txt
+- dict/de/test_mul.txt
+- dict/de/test_sgw.txt
+- dict/de/test_syn.txt
+- dict/de/user-dic.txt
+- dict/en/lingo-dic.txt
+- dict/en/lingo-irr.txt
+- dict/en/lingo-mul.txt
+- dict/en/lingo-syn.txt
+- dict/en/lingo-wdn.txt
+- dict/en/user-dic.txt
+- dict/ru/lingo-dic.txt
+- dict/ru/lingo-mul.txt
+- dict/ru/lingo-syn.txt
+- dict/ru/user-dic.txt
+- lang/de.lang
+- lang/en.lang
+- lang/ru.lang
 - lib/lingo.rb
-- lib/lingo/agenda_item.rb
 - lib/lingo/app.rb
 - lib/lingo/attendee.rb
 - lib/lingo/attendee/abbreviator.rb
@@ -249,6 +256,7 @@ files:
 - lib/lingo/database/source/single_word.rb
 - lib/lingo/database/source/word_class.rb
 - lib/lingo/debug.rb
+- lib/lingo/deferred_attendee.rb
 - lib/lingo/error.rb
 - lib/lingo/language.rb
 - lib/lingo/language/char.rb
@@ -271,16 +279,6 @@ files:
 - lib/lingo/web/public/lingo.png
 - lib/lingo/web/public/lingoweb.css
 - lib/lingo/web/views/index.erb
-- lingo-call.cfg
-- lingo.cfg
-- lingo.rb
-- lir.cfg
-- ru.lang
-- ru/lingo-dic.txt
-- ru/lingo-mul.txt
-- ru/lingo-syn.txt
-- ru/user-dic.txt
-- spec/spec_helper.rb
 - test/attendee/ts_abbreviator.rb
 - test/attendee/ts_decomposer.rb
 - test/attendee/ts_multi_worder.rb
@@ -304,13 +302,19 @@ files:
 - test/ref/artikel.seq
 - test/ref/artikel.syn
 - test/ref/artikel.vec
+- test/ref/artikel.vef
 - test/ref/artikel.ven
 - test/ref/artikel.ver
+- test/ref/artikel.vet
 - test/ref/lir.mul
 - test/ref/lir.non
 - test/ref/lir.seq
 - test/ref/lir.syn
 - test/ref/lir.vec
+- test/ref/lir.vef
+- test/ref/lir.ven
+- test/ref/lir.ver
+- test/ref/lir.vet
 - test/test_helper.rb
 - test/ts_database.rb
 - test/ts_language.rb
@@ -324,51 +328,34 @@ licenses:
 metadata: {}
 post_install_message: |2+
 
-  lingo-1.8.5 [2014-10-02]:
+  lingo-1.8.6 [2015-02-09]:
 
-  * Dictionary values (projections) are no longer sorted; hence, order of
-    definition affects processing.
-  * Lexicals in Lingo::Language::Word are no longer sorted; in particular,
-    compound parts keep their original order.
-  * Lexicals in Lingo::Language::Word are no longer cleaned from duplicates.
-  * Compiled dictionaries are updated whenever the Lingo version or their
-    configuration changes, not only when the source file's size or modification
-    time changes.
-  * Lingo::Attendee::Synonymer learned <tt>compound-parts</tt> option to also
-    generate synonyms for compound parts when set to +true+.
-  * Lingo::Attendee::TextReader learned better PDF-to-text conversion using the
-    +pdftotext+ command; specify <tt>filter: pdftotext</tt> in the config.
-  * Lingo::Attendee::VectorFilter learned +dict+ option to print words in
-    dictionary format (viz. Lingo::Database::Source::WordClass).
-  * Lingo::Attendee::VectorFilter learned +preamble+ option to print current
-    configuration to the beginning of the log file (<tt>debug: 'true'</tt>);
-    set <tt>preamble: false</tt> to disable.
-  * Multiword dictionaries compiled from base forms can now generate inflected
-    adjectives based on the gender of the head noun; set <tt>inflect: true</tt>
-    in the dictionary config.
-  * Lingo::Database::Source::WordClass supports gender information being encoded
-    in the dictionary as well as shorthand notation for multiple word
-    classes/genders.
-  * Lingo::Database::Source::WordClass supports compounds being encoded in the
-    dictionary (appending <tt>+</tt> to their parts' word classes is
-    recommended).
-  * Lingo::Database::Source removes leading and trailing whitespace from
-    dictionary lines.
-  * Lingo::Database::Crypter uses OpenSSL to encrypt/decrypt dictionaries.
-    Note: Can't decrypt dictionaries encrypted with the old scheme anymore.
-  * Lingo::Attendee::Tokenizer learned subset of MediaWiki syntax.
-  * Eliminated pathological behaviour of the +URLS+ rule in
-    Lingo::Attendee::Tokenizer.
-  * Fixed regression introduced in 1.8.2 where <tt>combine: all</tt> would no
-    longer work in Lingo::Attendee::MultiWorder.
-  * Updated and extended Russian dictionaries. (Yulia Dorokhova, Thomas Müller)
-  * +lingoctl+ no longer overwrites existing files without confirmation.
-  * +lingoctl+ learned +archive+ command.
-  * Dictionary cleanup.
+  * Lingo::Attendee::VectorFilter learned +pos+ option to print position and
+    byte offset with each word.
+  * Lingo::Attendee::VectorFilter learned +tfidf+ option to sort results based
+    on their tf–idf[https://en.wikipedia.org/wiki/Tf–idf] score; the document
+    frequencies are calculated over the "corpus" of all files processed during
+    a single program invocation.
+  * Lingo::Attendee::VectorFilter learned +tokens+ option to filter on
+    Lingo::Language::Token in addition to Lingo::Language::Word.
+  * Lingo::Attendee::VectorFilter no longer supports +debug+ (as well as
+    +prompt+ and +preamble+); use Lingo::Attendee::DebugFilter instead.
+  * Lingo::Attendee::TextReader no longer removes line endings; option +chomp+
+    is obsolete.
+  * Lingo::Attendee::TextReader passes byte offset to the following attendee.
+  * Lingo::Attendee::Tokenizer records token's byte offset.
+  * Lingo::Attendee::Tokenizer records token's sequence position.
+  * Lingo::Attendee::Tokenizer learned <tt>skip-tags</tt> option to skip over
+    specified tags' contents.
+  * Lingo::Attendee subclasses warn when invalid or obsolete options or names
+    are used.
+  * Changed German infix substitution +/en+ to +ch/chen+ in order to prevent
+    overly aggressive identifications.
+  * Internal refactoring and API changes.
 
 rdoc_options:
 - "--title"
-- lingo Application documentation (v1.8.5)
+- lingo Application documentation (v1.8.6)
 - "--charset"
 - UTF-8
 - "--line-numbers"
@@ -389,7 +376,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: The full-featured automatic indexing system