lingo 1.8.5 → 1.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 43119723e15432c990503ec61938ee96b15a1e53
-  data.tar.gz: 6674993f107c30acf34ea43eff24272356663b5a
+  metadata.gz: e4cc870c8c1b49c580841a934b5906ed6ddf75e4
+  data.tar.gz: 1ecb26c708daa4bfa09f4aa76f6d7e17f1a72683
 SHA512:
-  metadata.gz: 95bff3ff2eadfc3267da83a1d3a5398cc3b9f4eff13ae1f6682c24054fd5baf7e20ad03b241a2d5731f8c3636dd87d7f937bdb1a4f022ab883e40c5ad4407ec1
-  data.tar.gz: b7c7a505f4f61f70a59e8154d204a9ac87fe84fda453e1449fd66ca70e82bda8378070f4b0f7d763f7c8191f19e789c382b65135f8951c9917d7e7ca204d504d
+  metadata.gz: f2f0abed6198a7fcf0ff4f44aa442266f38c44646c7f4e8ef894886c453ce1654edd217c675f12e6b7d828c43ac461abb64d92aef20015249dbdf6f9efc03a3f
+  data.tar.gz: cb0be6e46a16639a384bab3507dc3b2bd4465736d1d7e0189d3930d1252e247fff4421364d860bd2cdd12f26b4f4445192a87998bea017bb1f285c8e0bda7639

data/ChangeLog

@@ -2,6 +2,31 @@
 
 = Revision history for Lingo
 
+== 1.8.6 [2015-02-09]
+
+* Lingo::Attendee::VectorFilter learned +pos+ option to print position and
+  byte offset with each word.
+* Lingo::Attendee::VectorFilter learned +tfidf+ option to sort results based
+  on their tf–idf[https://en.wikipedia.org/wiki/Tf–idf] score; the document
+  frequencies are calculated over the "corpus" of all files processed during
+  a single program invocation.
+* Lingo::Attendee::VectorFilter learned +tokens+ option to filter on
+  Lingo::Language::Token in addition to Lingo::Language::Word.
+* Lingo::Attendee::VectorFilter no longer supports +debug+ (as well as
+  +prompt+ and +preamble+); use Lingo::Attendee::DebugFilter instead.
+* Lingo::Attendee::TextReader no longer removes line endings; option +chomp+
+  is obsolete.
+* Lingo::Attendee::TextReader passes byte offset to the following attendee.
+* Lingo::Attendee::Tokenizer records token's byte offset.
+* Lingo::Attendee::Tokenizer records token's sequence position.
+* Lingo::Attendee::Tokenizer learned <tt>skip-tags</tt> option to skip over
+  specified tags' contents.
+* Lingo::Attendee subclasses warn when invalid or obsolete options or names
+  are used.
+* Changed German infix substitution +/en+ to +ch/chen+ in order to prevent
+  overly aggressive identifications.
+* Internal refactoring and API changes.
+
 == 1.8.5 [2014-10-02]
 
 * Dictionary values (projections) are no longer sorted; hence, order of

data/README

@@ -35,7 +35,7 @@
 
 == VERSION
 
-This documentation refers to Lingo version 1.8.5
+This documentation refers to Lingo version 1.8.6
 
 
 == DESCRIPTION
@@ -58,7 +58,7 @@ is a minimal configuration example to analyse this README file:
   meeting:
     attendees:
       - text_reader: { files: 'README' }
-      - debugger:    { eval: 'true', ceval: 'cmd!="EOL"', prompt: '<debug>: ' }
+      - debugger:    { eval: 'true', ceval: 'cmd!="EOL"', prompt: '<debug>:  ' }
 
 Lingo is told to invite two attendees and wants them to talk to each other,
 hence the name Lingo (= the technical language).
@@ -187,7 +187,7 @@ of context to external files.
 _Example_:
 
   # keep line endings
-  - text_reader:     { files: $(files), chomp: false }
+  - text_reader:     { files: $(files) }
   # keep whitespace
   - tokenizer:       { space: true }
   # do processing...
@@ -545,7 +545,7 @@ Travis CI::     https://travis-ci.org/lex-lingo/lingo
 === Background and Theory
 
 * Gödert, W.; Lepsky, K.; Nagelschmidt, M.: <em>{Informationserschließung und Automatisches Indexieren: ein Lehr- und Arbeitsbuch}[http://dx.doi.org/10.1007/978-3-642-23513-9]</em>. (German) Berlin etc.: Springer, 2012.
-* Lepsky, K.; Vorhauer, J.: <em>{Lingo: ein open source System für die automatische Indexierung deutschsprachiger Dokumente}[http://dx.doi.org/10.1515/ABITECH.2006.26.1.18]</em>. (German) In: ABI Technik 26 (1), 2006. pp 18-29.
+* Lepsky, K.; Vorhauer, J.: <em>{Lingo – ein open source System für die automatische Indexierung deutschsprachiger Dokumente}[http://dx.doi.org/10.1515/ABITECH.2006.26.1.18]</em>. (German) In: ABI Technik 26 (1), 2006. pp 18-29.
 * Nohr, H.: <em>{Grundlagen der automatischen Indexierung: ein Lehrbuch}[http://logos-verlag.de/cgi-bin/buch/isbn/0121]</em>. (German) Berlin: Logos, 2005.
 * Hausser, R.: <em>{Grundlagen der Computerlinguistik. Mensch-Maschine-Kommunikation in natürlicher Sprache}[http://zbmath.org/?q=an:0956.68141]</em>. (German) Berlin etc.: Springer, 2000.
 * Allen, J.: <em>{Natural language understanding}[http://zbmath.org/?q=an:0851.68106]</em>. (English) Redwood City, CA: Benjamin/Cummings, 1995.
@@ -559,6 +559,8 @@ Travis CI::     https://travis-ci.org/lex-lingo/lingo
 * Bredack, J.: <em>{Terminologieextraktion von Mehrwortgruppen in kunsthistorischen Fachtexten}[http://ixtrieve.fh-koeln.de/lehre/bredack-2013.pdf]</em>. (German) Köln: Fachhochschule Köln, 2013.
 * Maylein, L.; Langenstein, A.: <em>{Neues vom Relevanz-Ranking im HEIDI-Katalog der Universitätsbibliothek Heidelberg}[http://b-i-t-online.de/heft/2013-03-fachbeitrag-maylein.pdf]</em>. (German) In: b.i.t.online 16 (3), 2013. pp 190-200.
 * Gödert, W.: <em>{Detecting multiword phrases in mathematical text corpora}[http://arxiv.org/abs/1210.0852]</em>. (English) arXiv:1210.0852 [cs.CL], 2012.
+* Jersek, T.: <em>{Automatische DDC-Klassifizierung mit Lingo: Vorgehensweise und Ergebnisse}[http://www.citeulike.org/user/klaus-lepsky/article/12476139]</em>. (German) Köln: Fachhochschule Köln, 2012.
+* Glaesener, L.: <em>{Automatisches Indexieren einer informationswissenschaftlichen Datenbank mit Mehrwortgruppen}[http://www.citeulike.org/user/klaus-lepsky/article/12476133]</em>. (German) Köln: Fachhochschule Köln, 2012.
 * Schiffer, R.: <em>{Automatisches Indexieren technischer Kongressschriften}[http://ixtrieve.fh-koeln.de/lehre/schiffer-2007.pdf]</em>. (German) Köln: Fachhochschule Köln, 2007.
 
 
@@ -582,7 +584,7 @@ Lingo is based on a collective development by Klaus Lepsky and John Vorhauer.
 == LICENSE AND COPYRIGHT
 
 Copyright (C) 2005-2007 John Vorhauer
-Copyright (C) 2007-2014 John Vorhauer, Jens Wille
+Copyright (C) 2007-2015 John Vorhauer, Jens Wille
 
 Lingo is free software: you can redistribute it and/or modify it under the
 terms of the GNU Affero General Public License as published by the Free

data/Rakefile

@@ -1,33 +1,20 @@
 # encoding: utf-8
 
-__DIR__ = File.expand_path('..', __FILE__)
-
-require 'rake/clean'
-require 'nuggets/ruby'
-require File.join(__DIR__, %w[lib lingo version])
-
-PACKAGE_NAME = 'lingo'
-PACKAGE_PATH = File.join(__DIR__, 'pkg', "#{PACKAGE_NAME}-#{Lingo::VERSION}")
-
-if RUBY_PLATFORM =~ /msdos|mswin|djgpp|mingw|windows/i
-  ZIP_COMMANDS = ['zip', '7z a']  # for hen's gem task
-end
-
-task default: :spec
-task package: [:checkdoc, 'test:all', :clean]
+require_relative 'lib/lingo/version'
 
 begin
   require 'hen'
 
   Hen.lay! {{
     gem: {
-      name:         PACKAGE_NAME,
+      name:         'lingo',
       version:      Lingo::VERSION,
       summary:      'The full-featured automatic indexing system',
       authors:      ['John Vorhauer', 'Jens Wille'],
       email:        ['lingo@vorhauer.de', 'jens.wille@gmail.com'],
       license:      'AGPL-3.0',
       homepage:     'http://lex-lingo.de',
+
       description:  <<-EOT,
 Lingo is an open source indexing system for research and teachings.
 The main functions of Lingo are:
@@ -39,23 +26,32 @@ The main functions of Lingo are:
 * generic identification of phrases/word sequences based on patterns
   of word classes
       EOT
+
       extra_files:  FileList[
-        'lingo.rb', 'lingo{,-call}.cfg', 'lir.cfg',
-        '{de,en,ru}.lang', '{de,en,ru}/{lingo-*,user-dic,test_*}.txt',
-        'txt/{artikel{,-en,-ru},lir}.txt', 'lib/lingo/{srv,web}/**/{,.}*'
+        'lib/lingo/{srv,web}/**/{,.}*',
+        'config/*.cfg',
+        'dict/*/*.txt',
+        'lang/*.lang',
+        'txt/*.txt'
       ].to_a,
-      required_ruby_version: '>= 1.9.3',
+
       dependencies: {
-        'cyclops'       => ['~> 0.0', '>= 0.0.4'],
-        'nuggets'       => '~> 1.0',
+        'cyclops'       => '~> 0.1',
+        'nuggets'       => '~> 1.1',
         'rubyzip'       => '~> 1.1',
         'sinatra-bells' => '~> 0.0',
         'unicode'       => '~> 0.4'
       },
+
       development_dependencies: {
         'diff-lcs' => '~> 1.2',
         'open4'    => '~> 1.3'
-      }
+      },
+
+      required_ruby_version: '>= 1.9.3'
+    },
+    test: {
+      pattern: %w[test/ts_*.rb test/attendee/ts_*.rb]
     }
   }}
 rescue LoadError => err
@@ -71,40 +67,22 @@ CLEAN.include(
 
 CLOBBER.include('store')
 
-task :checkdoc do
-  docfile = File.join(__DIR__, 'doc', 'index.html')
-  abort "Please run `rake doc' first." unless File.exists?(docfile)
-end
-
 desc 'Run ALL tests'
-task 'test:all' => [:test, 'test:txt', 'test:lir']
-
-Rake::TestTask.new(:test) do |t|
-  t.test_files = FileList.new('test/ts_*.rb', 'test/attendee/ts_*.rb')
-end
+task 'test:all' => %w[test test:txt test:lir]
 
 desc 'Test against reference file (TXT)'
-task 'test:txt' do
-  test_ref('artikel', 'lingo')
-end
+task('test:txt') { test_ref('artikel', 'lingo') }
 
 desc 'Test against reference file (LIR)'
-task 'test:lir' do
-  test_ref('lir')
-end
-
-desc 'Run all tests on packaged distribution'
-task 'test:remote' => [:package] do
-  chdir(PACKAGE_PATH) { system('rake test:all') } || abort
-end
+task('test:lir') { test_ref('lir') }
 
-unless (benchmarks = Dir[File.join(__DIR__, 'bench', '*_bench.rb')]).empty?
+unless (benchmarks = Dir[File.expand_path('../bench/*_bench.rb', __FILE__)]).empty?
   desc 'Run all benchmarks'
   task :bench
 
   benchmarks.each { |benchmark|
     bench = File.basename(benchmark, '_bench.rb')
-    task :bench => benchtask = "bench:#{bench}"
+    task bench: benchtask = "bench:#{bench}"
 
     desc "Run #{bench} benchmark"
     task(benchtask) { system(File.ruby, benchmark) }
@@ -113,18 +91,43 @@ end
 
 def test_ref(name, cfg = name)
   require 'diff/lcs'
-  require 'diff/lcs/ldiff'
+  require 'diff/lcs/hunk'
+  require 'nuggets/ruby'
+
+  jruby = RUBY_ENGINE == 'jruby'
+  jruby_lir = jruby && name == 'lir'
+
+  cmd = %W[bin/lingo -c #{cfg} txt/#{name}.txt]
+  buf, diff = ["Command failed: #{cmd.join(' ')}"], 0
+
+  Process.ruby(*cmd, I: :lib, &jruby ?
+    lambda { |_, _, o, e| buf << e.read; buf << o.read } :
+    lambda { |_, _, o, e| IO.interact({}, { o => buf, e => buf }) }
+  ).success? or abort buf.join("\n\n")
+
+  Dir["test/ref/#{name}.*"].sort.each { |ref|
+    unless File.exist?(txt = ref.sub(/test\/ref/, 'txt'))
+      puts "?? #{txt}"
+    else
+      puts "## #{txt}"
+
+      data = [ref, txt].map { |file|
+        File.readlines(file).each { |line|
+          line.chomp!
+          line.gsub!(/(\d+\.\d+)\d/, '\1') if jruby_lir
+        }
+      }
 
-  cmd = %W[lingo.rb -c #{cfg} txt/#{name}.txt]
-  diff, msg = 0, ["Command failed: #{cmd.join(' ')}"]
+      diffs, fld = Diff::LCS.diff(*data), 0
 
-  Process.ruby(*cmd) { |_, _, o, e|
-    IO.interact({}, { o => msg, e => msg })
-  }.success? or abort msg.join("\n\n")
+      diffs.empty? ? next : diffs.each { |piece|
+        dlh = Diff::LCS::Hunk.new(*data, piece, 0, fld)
+        fld = dlh.file_length_difference
+        puts dlh.diff(:old)
+      }
+    end
 
-  Dir["test/ref/#{name}.*"].each { |ref|
-    puts "## #{org = ref.sub(/test\/ref/, 'txt')}"
-    diff += Diff::LCS::Ldiff.run(ARGV.clear << '-a' << org << ref)
+    diff += 1
   }
 
   exit diff + 1 unless diff.zero?

data/{lingo-call.cfg → config/lingo-call.cfg}

@@ -8,4 +8,4 @@ meeting:
     - multi_worder:  { source: sys-mul }
     - sequencer:     { }
     - synonymer:     { source: sys-syn, skip: '?,t' }
-    - debugger:      { prompt: '', ceval: 'false' }
+    - debugger:      { prompt: '', ceval: 'false', preamble: false }

data/{lingo.cfg → config/lingo.cfg}

@@ -52,7 +52,7 @@ meeting:
     ########################################
     # Datenstrom anzeigen
     #
-#   - debugger:        { eval: 'true', ceval: 'obj.cmd!="EOL"', prompt: 'lex:) ' }
+#   - debugger:        { eval: 'true', ceval: 'cmd!=:EOL', prompt: 'lex:) ' }
 
 
     ########################################
@@ -60,7 +60,7 @@ meeting:
     #
 
     # Erstelle Datei mit Endung .log für Datenstrom
-    - vector_filter:   { in: syn, debug: 'true', prompt: 'lex:) ' }
+    - debug_filter:    { in: syn, prompt: 'lex:) ' }
     - text_writer:     { ext: log, sep: "\n" }
 
     # Erstelle Datei mit Endung .non für nicht erkannte Wörter
@@ -83,6 +83,14 @@ meeting:
     - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: term_rel }
     - text_writer:     { ext: ver, sep: "\n" }
 
+    # Erstelle Datei mit Endung .vef für erkannte Indexterme mit TFIDF-Gewichtung
+    - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: term_rel, tfidf: true }
+    - text_writer:     { ext: vef, sep: "\n" }
+
+    # Erstelle Datei mit Endung .vet für erkannte Indexterme mit Positionen
+    - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: false, pos: true }
+    - text_writer:     { ext: vet, sep: "\n" }
+
     # Erstelle Datei mit Endung .mul für erkannte Mehrwortgruppen
     - vector_filter:   { in: syn, lexicals: m }
     - text_writer:     { ext: mul, sep: "\n" }

data/{lir.cfg → config/lir.cfg}

@@ -57,7 +57,7 @@ meeting:
     ########################################
     # Datenstrom anzeigen
     #
-#   - debugger:        { eval: 'true', ceval: 'obj.cmd!="EOL"', prompt: 'lex:) ' }
+#   - debugger:        { eval: 'true', ceval: 'cmd!=:EOL', prompt: 'lex:) ' }
 
 
     ########################################
@@ -65,7 +65,7 @@ meeting:
     #
 
     # Erstelle Datei mit Endung .log für Datenstrom
-    - vector_filter:   { in: syn, debug: 'true', prompt: 'lex:) ' }
+    - debug_filter:    { in: syn, prompt: 'lex:) ' }
     - text_writer:     { ext: log, sep: "\n" }
 
     # Erstelle Datei mit Endung .non für nicht erkannte Wörter
@@ -88,6 +88,14 @@ meeting:
     - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: term_rel }
     - text_writer:     { ext: ver, sep: '|' }
 
+    # Erstelle Datei mit Endung .vef für erkannte Indexterme mit TFIDF-Gewichtung
+    - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: term_rel, tfidf: true }
+    - text_writer:     { ext: vef, sep: '|' }
+
+    # Erstelle Datei mit Endung .vet für erkannte Indexterme mit Positionen
+    - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: false, pos: true }
+    - text_writer:     { ext: vet, sep: '|' }
+
     # Erstelle Datei mit Endung .mul für erkannte Mehrwortgruppen
     - vector_filter:   { in: syn, lexicals: m }
     - text_writer:     { ext: mul, sep: '|' }

data/{de.lang → lang/de.lang}

@@ -79,7 +79,7 @@ language:
       - [a, 'este ste ster sten stes ester estes esten e em en er ere eren erer eres es erem']
       - [v, 'e/en en/en est/en et/en st/en t/en te/en ten/en eten/en ete/en etest/en s']
       - [e, 's']
-      - [f, 's n e en es er ch/che /en']
+      - [f, 's n e en es er ch/che ch/chen']
 
     inflect:
       a:  # adjectives

data/lib/lingo.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -29,6 +29,8 @@ require 'stringio'
 require 'pathname'
 require 'fileutils'
 require 'nuggets/file/ext'
+require 'nuggets/hash/nest'
+require 'nuggets/hash/seen'
 require 'nuggets/env/user_home'
 require 'nuggets/string/camelscore'
 
@@ -200,12 +202,10 @@ class Lingo
     end
 
     def walk(path, options, legacy = true)
-      dirs = [options[:dir].to_s]
+      dirs, seen = [options[:dir].to_s], Hash.seen
       dirs << '' if legacy
       dirs.uniq!
 
-      seen = Hash.new { |h, k| h[k] = true; false }
-
       path.each { |d|
         next if seen[d = File.expand_path(d)]
         dirs.each { |i| yield File.join(d, i) } or break
@@ -265,13 +265,13 @@ class Lingo
   end
 
   def invite(list = config['meeting/attendees'])
-    supplier   = Hash.new { |h, k| h[k] = [] }
-    subscriber = Hash.new { |h, k| h[k] = [] }
+    supplier   = Hash.nest { [] }
+    subscriber = Hash.nest { [] }
 
     last_link, auto_link = '', 0
 
     list.each { |hash|
-      name = hash.keys.first.camelcase
+      name = (name_key = hash.keys.first).camelcase
 
       cfg = (config["language/attendees/#{name.downcase}"] || {})
         .merge(hash.values.first).update('name' => name)
@@ -284,41 +284,40 @@ class Lingo
 
       @attendees << attendee = Attendee.const_get(name).new(cfg, self)
 
+      unless name == (real = attendee.class.name.split('::').last)
+        config.deprecate(name_key, real.underscore, attendee, :name)
+      end
+
       { 'in' => subscriber, 'out' => supplier }.each { |key, target|
         cfg[key].split(SEP_RE).each { |ch| target[ch] << attendee }
       }
     }
 
     supplier.each { |ch, attendees| attendees.each { |att|
-      att.add_subscriber(subscriber[ch])
+      att.subscribers.concat(subscriber[ch])
     } }
   end
 
   def start
-    @attendees.first.listen(AgendaItem.new(Attendee::STR_CMD_TALK))
+    @attendees.first.control(:TALK)
   end
 
   def reset(close = true)
     dictionaries.each { |i| i.close } if close
     @dictionaries, @attendees = [], []
-    @lexical_hash = Hash.new { |h, k| h[k] = Language::LexicalHash.new(k, self) }
+    @lexical_hash = Hash.nest { |k| Language::LexicalHash.new(k, self) }
   end
 
   def warn(*msg)
     config.warn(*msg)
   end
 
-  def deprecate(old, new, obj = self)
-    config.deprecate(old, new, obj)
-  end
-
 end
 
 require_relative 'lingo/call'
 require_relative 'lingo/error'
 require_relative 'lingo/debug'
 require_relative 'lingo/config'
-require_relative 'lingo/agenda_item'
 require_relative 'lingo/progress'
 require_relative 'lingo/database'
 require_relative 'lingo/language'