lingo 1.9.0.pre1 → 1.9.0

data/test/attendee/ts_text_reader.rb

@@ -206,7 +206,7 @@ class TestAttendeeTextReader < AttendeeTestCase
   end
 
   def test_article_pdf
-    meet({ 'files' => file = 'test/article.pdf', 'filter' => true }, nil, [
+    meet({ 'files' => file = 'test/article.pdf', 'filter' => 'pdf' }, nil, [
       ai("FILE|#{path = File.expand_path(file)}"),
 ["                        Klaus Lepsky: Ist automatische Normierung mögich?\n", 75],
       [" ──────────────────────────────────────────────────────────────────────\n", 287],
@@ -356,7 +356,7 @@ class TestAttendeeTextReader < AttendeeTestCase
   end
 
   def test_article_xml
-    meet({ 'files' => file = 'test/article.xml', 'filter' => true }, nil, [
+    meet({ 'files' => file = 'test/article.xml', 'filter' => 'xml' }, nil, [
       ai("FILE|#{path = File.expand_path(file)}"),
       ["\n", 1],
       ["\t\n", 3],
@@ -477,7 +477,7 @@ class TestAttendeeTextReader < AttendeeTestCase
   end
 
   def test_article_html
-    meet({ 'files' => file = 'test/article.html', 'filter' => true }, nil, [
+    meet({ 'files' => file = 'test/article.html', 'filter' => 'html' }, nil, [
       ai("FILE|#{path = File.expand_path(file)}"),
       ["\n", 1],
       ["test/article-html.html\n", 24],
@@ -540,6 +540,6 @@ class TestAttendeeTextReader < AttendeeTestCase
       ai("EOF|#{path}"),
       ai('EOT|')
     ])
-  end
+  end unless RUBY_ENGINE == 'jruby'
 
 end

data/test/attendee/ts_text_writer.rb

@@ -30,7 +30,15 @@ class TestAttendeeTextWriter < AttendeeTestCase
 
     assert_equal([
       "Dies,ist,eine,Zeile,.\n", "Dies,ist,eine,zweite,Zeile,.\n"
-    ], File.readlines('test/test.tst', encoding: Lingo::ENC))
+    ], readlines('test.tst'))
+  end
+
+  def test_format
+    meet({ 'ext' => '%c-%l.tst', 'sep' => ',' }, @input)
+
+    assert_equal([
+      "Dies,ist,eine,Zeile,.\n", "Dies,ist,eine,zweite,Zeile,.\n"
+    ], readlines('test.lingo-de.tst'))
   end
 
   def test_complex
@@ -38,7 +46,7 @@ class TestAttendeeTextWriter < AttendeeTestCase
 
     assert_equal([
       "Dies-ist-eine-Zeile-.\n", "Dies-ist-eine-zweite-Zeile-.\n"
-    ], File.readlines('test/test.yip', encoding: Lingo::ENC))
+    ], readlines('test.yip'))
   end
 
   def test_crlf
@@ -46,7 +54,7 @@ class TestAttendeeTextWriter < AttendeeTestCase
 
     assert_equal([
       "Dies\n", "ist\n", "eine\n", "Zeile\n", ".\n", "Dies\n", "ist\n", "eine\n", "zweite\n", "Zeile\n", ".\n"
-    ], File.readlines('test/test.txt2', encoding: Lingo::ENC))
+    ], readlines('test.txt2'))
   end
 
   def test_lir_file
@@ -73,7 +81,7 @@ cen. 056: Die intellektuelle Erschließung des Internet befindet sich in einer K
 FG-Projekt GERHARD.\n",
       "00239*020: Information Retrieval und Dokumentmanagement im Multimedia-Zeitalter. 056: \"Das Buch ist ein praxisbezogenes VADEMECUM\
  für alle, die in einer Welt der Datennetze Wissen/Informationen sammeln.\n"
-    ], File.readlines('test/lir.vec', encoding: Lingo::ENC))
+    ], readlines('lir.vec'))
   end
 
   def test_nonewords
@@ -83,7 +91,13 @@ FG-Projekt GERHARD.\n",
 
     assert_equal([
       "Nonwörter\n", "Nonsense"
-    ], File.readlines('test/text.non', encoding: Lingo::ENC))
+    ], readlines('text.non'))
+  end
+
+  private
+
+  def readlines(file)
+    File.readlines("test/#{file}", encoding: Lingo::ENCODING)
   end
 
 end

data/test/test_helper.rb

@@ -3,6 +3,8 @@
 require 'test/unit'
 require 'lingo'
 
+warn 'NOTICE: Skipping slow tests...' if ENV['LINGO_DISABLE_SLOW_TESTS']
+
 class LingoTestCase < Test::Unit::TestCase
 
   unless const_defined?(:TEST_FILE)

data/test/ts_database.rb

@@ -2,7 +2,7 @@
 
 require_relative 'test_helper'
 
-class TestDatabase < LingoTestCase
+class DatabaseTestCase < LingoTestCase
 
   def setup
     @lingo = Lingo.new
@@ -64,6 +64,214 @@ Wort2=
     EOT
   end
 
+  def write(config, input)
+    FileUtils.mkdir_p(File.dirname(TEST_FILE))
+    File.open(TEST_FILE, 'w', encoding: Lingo::ENCODING) { |f| f.write(input) }
+
+    yield set_config('tst', config.merge('name' => TEST_FILE))
+  ensure
+    cleanup_store
+  end
+
+  def set_config(id, config)
+    "_test_#{id}_".tap { |i| @lingo.config["language/dictionary/databases/#{i}"] = config }
+  end
+
+end
+
+class TestSource < DatabaseTestCase
+
+  def test_dump_singleword
+    compare({
+      'txt-format' => 'SingleWord'
+    }, @singleword)
+  end
+
+  def test_dump_singleword_defwc
+    compare({
+      'txt-format' => 'SingleWord',
+      'def-wc'     => '*'
+    }, @singleword)
+  end
+
+  def test_dump_singleword_defmulwc
+    compare({
+      'txt-format' => 'SingleWord',
+      'def-mul-wc' => 'm'
+    }, @singleword)
+  end
+
+  def test_dump_singleword_uselex
+    compare({
+      'txt-format' => 'SingleWord',
+      'use-lex'    => set_config('lex',
+        'name'       => 'de/lingo-dic.txt',
+        'txt-format' => 'WordClass',
+        'separator'  => '='
+      )
+    }, @singleword)
+  end
+
+  def test_dump_singleword_inflect
+    compare({
+      'txt-format' => 'SingleWord',
+      'use-lex'    => set_config('lex',
+        'name'       => 'de/lingo-dic.txt',
+        'txt-format' => 'WordClass',
+        'separator'  => '='
+      ),
+      'inflect'    => true
+    }, @singleword_inflect)
+  end
+
+  def test_dump_singleword_inflect_s
+    compare({
+      'txt-format' => 'SingleWord',
+      'use-lex'    => set_config('lex',
+        'name'       => 'de/lingo-dic.txt',
+        'txt-format' => 'WordClass',
+        'separator'  => '='
+      ),
+      'inflect'    => 's'
+    }, @singleword_inflect)
+  end
+
+  def test_dump_singleword_inflect_e
+    compare({
+      'txt-format' => 'SingleWord',
+      'use-lex'    => set_config('lex',
+        'name'       => 'de/lingo-dic.txt',
+        'txt-format' => 'WordClass',
+        'separator'  => '='
+      ),
+      'inflect'    => 'e'
+    }, @singleword_inflect)
+  end
+
+  def test_dump_singleword_hyphenate
+    compare({
+      'txt-format' => 'SingleWord',
+      'use-lex'    => set_config('lex',
+        'name'       => 'de/lingo-dic.txt',
+        'txt-format' => 'WordClass',
+        'separator'  => '='
+      ),
+      'hyphenate'  => true
+    }, @singleword)
+  end
+
+  def test_dump_singleword_crypt
+    compare({
+      'txt-format' => 'SingleWord',
+      'crypt'      => true
+    }, @singleword)
+  end
+
+  def test_dump_keyvalue
+    compare({
+      'txt-format' => 'KeyValue'
+    }, @keyvalue.gsub(' * ', '*'))
+  end
+
+  def test_dump_keyvalue_separator
+    compare({
+      'txt-format' => 'KeyValue',
+      'separator'  => '*'
+    }, @keyvalue.gsub(' * ', '*'))
+  end
+
+  def test_dump_keyvalue_defwc
+    compare({
+      'txt-format' => 'KeyValue',
+      'separator'  => '*',
+      'def-wc'     => 's'
+    }, @keyvalue.gsub(' * ', '*'))
+  end
+
+  def test_dump_wordclass
+    compare({
+      'txt-format' => 'WordClass',
+      'separator'  => '='
+    }, %q{
+      Wort1=Projektion1 #h
+      Wort2=Projektion2 #i
+      Wort3=Projektion3 #e
+      Wort1=Projektion4 #e
+      Wort1=#s
+      Wort2=
+      Wort4.illegal
+      Wort4=still illegal
+      Wort4=still illegal#s!
+      Wort4=now we're talking #s+
+    })
+  end
+
+  def test_dump_wordclass_gender
+    compare({
+      'txt-format' => 'WordClass'
+    }, %q{
+      substantiv,substantiv #a|s.n
+      mehr,mehr #s|w.n mehren #v
+      wort,wort #s.n
+      gruppe,gruppe #s.f
+      modul,modul #s.m|n
+      nocken,nock #s.f|m|n nocke #s.f nocken #s.m
+      albern,albern #a|v
+      fortuna,fortuna #e|s.f
+    })
+  end
+
+  def test_dump_wordclass_gender_noncompact
+    compare({
+      'txt-format' => 'WordClass'
+    }, %q{
+      substantiv,substantiv #a substantiv #s.n
+      mehr,mehr #s.n mehr #w mehren #v
+      wort,wort #s.n
+      gruppe,gruppe #s.f
+      modul,modul #s.m modul #s.n
+      nocken,nock #s.f nock #s.m nock #s.n nocke #s.f nocken #s.m
+      albern,albern #a albern #v
+      fortuna,fortuna #e.f fortuna #s.f
+    }, nil, nil, false)
+  end
+
+  def test_dump_multivalue
+    compare({
+      'txt-format' => 'MultiValue',
+      'separator'  => ';'
+    }, %q{
+      Hasen;Nasen;Vasen;Rasen
+      Gold;Edelmetall;Mehrwert
+      Rasen;Gras;Grüne Fläche
+      Rasen;Rennen;Wettrennen
+    })
+  end
+
+  def test_dump_multikey
+    compare({
+      'txt-format' => 'MultiKey'
+    }, %q{
+      Hasen;Nasen;Vasen;Rasen
+      Gold;Edelmetall;Mehrwert
+    })
+  end
+
+  def compare(config, input, *args)
+    write(config, input) { |id|
+      src, dump, lines = Lingo::Database::Source.from_id(id, @lingo), [], []
+
+      src.each_lexical { |key, lex| dump << src.dump_line(key, lex, *args) }
+      src.each_line { |line,| lines << line }
+
+      assert_equal dump, lines
+    }
+  end
+
+end
+
+class TestDatabase < DatabaseTestCase
+
   def test_singleword
     compare({
       'txt-format' => 'SingleWord'
@@ -482,22 +690,13 @@ Wort2=
   end
 
   def compare(config, input, output = nil)
-    FileUtils.mkdir_p(File.dirname(TEST_FILE))
-    File.open(TEST_FILE, 'w', encoding: Lingo::ENC) { |f| f.write(input) }
+    err = nil
 
-    id, err = set_config('tst', config.merge('name' => TEST_FILE)), nil
-
-    Lingo::Database.open(id, @lingo) { |db| begin
-      block_given? ? yield(db) : assert_equal(output, db.to_h
-        .tap { |h| h.delete(Lingo::Database::SYS_KEY) }); rescue => err; end }
+    write(config, input) { |id| Lingo::Database.open(id, @lingo) { |db| begin
+      block_given? ? yield(db) : assert_equal(output, db.to_h.tap { |h|
+        h.delete(Lingo::Database::SYS_KEY) }); rescue => err; end } }
 
     raise err if err
-  ensure
-    cleanup_store
-  end
-
-  def set_config(id, config)
-    "_test_#{id}_".tap { |i| @lingo.config["language/dictionary/databases/#{i}"] = config }
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lingo
 version: !ruby/object:Gem::Version
-  version: 1.9.0.pre1
+  version: 1.9.0
 platform: ruby
 authors:
 - John Vorhauer
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-02 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cyclops
@@ -31,42 +31,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.4'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: sinatra-bells
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: unicode
   requirement: !ruby/object:Gem::Requirement
@@ -129,14 +129,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: hen
   requirement: !ruby/object:Gem::Requirement
@@ -146,7 +146,7 @@ dependencies:
         version: '0.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.3
+        version: 0.8.5
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -156,7 +156,7 @@ dependencies:
         version: '0.8'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.3
+        version: 0.8.5
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -247,6 +247,7 @@ files:
 - lang/ru.lang
 - lib/lingo.rb
 - lib/lingo/app.rb
+- lib/lingo/array_utils.rb
 - lib/lingo/attendee.rb
 - lib/lingo/attendee/abbreviator.rb
 - lib/lingo/attendee/analysis_filter.rb
@@ -370,20 +371,31 @@ licenses:
 metadata: {}
 post_install_message: |2+
 
-  lingo-1.9.0 [unreleased]:
+  lingo-1.9.0 [2016-09-13]:
 
+  * <b>Dropped support for Ruby 1.9.</b>
   * Removed support for deprecated options and attendee names (+old+ → +new+):
-    * Lingo::Language::Grammar : +compositum+ → +compound+
-    * Lingo::Attendee::TextReader : +lir-record-pattern+ → +records+
-    * Lingo::Config : +multiworder+ → +multi_worder+, +objectfilter+ →
-      +object_filter+, +textreader+ → +text_reader+, +textwriter+ →
-      +text_writer+, +vectorfilter+ → +vector_filter+, +wordsearcher+ →
-      +word_searcher+
+    * Lingo::Language::Grammar<b></b>:
+      +compositum+ → +compound+
+    * Lingo::Attendee::TextReader<b></b>:
+      +lir-record-pattern+ → +records+
+    * Lingo::Config<b></b>:
+      +multiworder+ → +multi_worder+,
+      +objectfilter+ → +object_filter+,
+      +textreader+ → +text_reader+,
+      +textwriter+ → +text_writer+,
+      +vectorfilter+ → +vector_filter+,
+      +wordsearcher+ → +word_searcher+
+  * Lingo::Attendee::TextWriter learned format directives for +ext+ option
+    (currently supported are: <tt>%c</tt> = config name, <tt>%l</tt> = language
+    name, <tt>%d</tt> = current date, <tt>%t</tt> = current time).
+  * Lingo::Attendee::Sequencer remembers word form of sequences.
+  * Updated and extended English system dictionary and suffix list.
   * Fixed errors with XML input (issue #15 by Thomas Berger).
 
 rdoc_options:
 - "--title"
-- lingo Application documentation (v1.9.0.pre1)
+- lingo Application documentation (v1.9.0)
 - "--charset"
 - UTF-8
 - "--line-numbers"
@@ -396,15 +408,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.6
 signing_key: 
 specification_version: 4
 summary: The full-featured automatic indexing system