lingo 1.8.0 → 1.8.1

data/info/gpl-hdr.txt

@@ -1,27 +1,24 @@
 #--
-# LINGO ist ein Indexierungssystem mit Grundformreduktion, Kompositumzerlegung,
-# Mehrworterkennung und Relationierung.
-#
-# Copyright (C) 2005-2007 John Vorhauer
-# Copyright (C) 2007-2011 John Vorhauer, Jens Wille
-#
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Affero General Public License as published by the Free
-# Software Foundation; either version 3 of the License, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Affero General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin St, Fifth Floor, Boston, MA 02110, USA
-#
-# For more information visit http://www.lex-lingo.de or contact me at
-# welcomeATlex-lingoDOTde near 50°55'N+6°55'E.
-#
-# Lex Lingo rules from here on
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
 #++
 

data/lib/lingo.rb

@@ -1,55 +1,35 @@
 # encoding: utf-8
 
 #--
-# LINGO ist ein Indexierungssystem mit Grundformreduktion, Kompositumzerlegung,
-# Mehrworterkennung und Relationierung.
-#
-# Copyright (C) 2005-2007 John Vorhauer
-# Copyright (C) 2007-2011 John Vorhauer, Jens Wille
-#
-# This program is free software; you can redistribute it and/or modify it under
-# the terms of the GNU Affero General Public License as published by the Free
-# Software Foundation; either version 3 of the License, or (at your option)
-# any later version.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
-# details.
-#
-# You should have received a copy of the GNU Affero General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin St, Fifth Floor, Boston, MA 02110, USA
-#
-# For more information visit http://www.lex-lingo.de or contact me at
-# welcomeATlex-lingoDOTde near 50°55'N+6°55'E.
-#
-# Lex Lingo rules from here on
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
 #++
 
 require 'stringio'
 require 'benchmark'
+require 'nuggets/file/ext'
 require 'nuggets/env/user_home'
 require 'nuggets/numeric/duration'
-
-require_relative 'lingo/config'
-require_relative 'lingo/attendees'
-require_relative 'lingo/attendee/abbreviator'
-require_relative 'lingo/attendee/debugger'
-require_relative 'lingo/attendee/decomposer'
-require_relative 'lingo/attendee/dehyphenizer'
-require_relative 'lingo/attendee/multiworder'
-require_relative 'lingo/attendee/noneword_filter'
-require_relative 'lingo/attendee/objectfilter'
-require_relative 'lingo/attendee/variator'
-require_relative 'lingo/attendee/sequencer'
-require_relative 'lingo/attendee/synonymer'
-require_relative 'lingo/attendee/textreader'
-require_relative 'lingo/attendee/textwriter'
-require_relative 'lingo/attendee/tokenizer'
-require_relative 'lingo/attendee/vector_filter'
-require_relative 'lingo/attendee/wordsearcher'
-require_relative 'lingo/version'
+require 'nuggets/string/camelscore'
 
 class Lingo
 
@@ -65,13 +45,22 @@ class Lingo
   # The search path for Lingo dictionary and configuration files.
   PATH = ENV['LINGO_PATH'] || [CURR, HOME, BASE].join(File::PATH_SEPARATOR)
 
+  ENV['LINGO_PLUGIN_PATH'] ||= File.join(HOME, 'plugins')
+
+  # Map of file types to their standard location and file extension.
   FIND_OPTIONS = {
     config: { dir: 'config', ext: 'cfg'  },
     dict:   { dir: 'dict',   ext: 'txt'  },
     lang:   { dir: 'lang',   ext: 'lang' },
-    store:  { dir: 'store',  ext: nil    }
+    store:  { dir: 'store',  ext:  nil   },
+    sample: { dir: 'txt',    ext: 'txt'  }
   }
 
+  # Default encoding
+  ENC = 'UTF-8'.freeze
+
+  STRING_SEPARATOR_RE = %r{[; ,|]}
+
   class << self
 
     def talk(*args)
@@ -82,10 +71,6 @@ class Lingo
       Call.new(['-c', cfg, *args]).call(&block)
     end
 
-    def error(msg)
-      abort(msg)
-    end
-
     def list(type, options = {})
       options = options_for(type, options)
       path    = path_for(options)
@@ -130,10 +115,12 @@ class Lingo
     def find_file(file, path, options)
       pn = Pathname.new(file_with_ext(file, options)).cleanpath
 
-      walk(path, options) { |dir|
-        pn2 = pn.expand_path(dir)
-        pn = pn2 and break if pn2.exist?
-      } if pn.relative?
+      if pn.relative?
+        walk(path, options) { |dir|
+          pn2 = pn.expand_path(dir)
+          pn = pn2 and break if pn2.exist?
+        }
+      end
 
       realpath_for(pn, path)
     end
@@ -142,18 +129,14 @@ class Lingo
       base = basename(:dict, find(:dict, file, path))
 
       walk(path.reverse, options, false) { |dir|
-        Pathname.new(dir).ascend { |r|
-          break true if r.file?
-
-          return File.join(dir, base).tap { |s|
-            s.chomp!(File.extname(s))
-          } if r.writable?
-
-          break true if r.exist?
+        Pathname.new(dir).ascend { |i|
+          break  true                                 if i.file?
+          return File.chomp_ext(File.join(dir, base)) if i.writable?
+          break  true                                 if i.exist?
         }
       }
 
-      raise 'No writable store found in search path'
+      raise NoWritableStoreError.new(file, path)
     end
 
     def options_for(type, options = {})
@@ -190,6 +173,11 @@ class Lingo
       pn.realpath(path.first).to_s
     end
 
+    def require_optional(lib)
+      require lib unless ENV["LINGO_NO_#{lib.upcase}"]
+    rescue LoadError
+    end
+
   end
 
   attr_reader :dictionaries, :report_status, :report_time
@@ -205,10 +193,19 @@ class Lingo
 
   def dictionary_config
     @dictionary_config ||= config['language/dictionary']
+  rescue => err
+    raise ConfigLoadError.new(err)
   end
 
   def database_config(id)
-    dictionary_config['databases'][id]
+    dictionary_config['databases'][id].tap { |cfg|
+      raise NoDatabaseConfigError.new(id) unless cfg
+      raise InvalidDatabaseConfigError.new(id) unless cfg.has_key?('name')
+    }
+  end
+
+  def lexical_hash(src)
+    @lexical_hash[src]
   end
 
   def talk
@@ -226,7 +223,7 @@ class Lingo
 
     list.each { |hash|
       # {'attendee' => {'name'=>'Attendee', 'in'=>'nase', 'out'=>'ohr', 'param'=>'hase'}}
-      cfg = hash.values.first.merge('name' => hash.keys.first.capitalize)
+      cfg = hash.values.first.merge('name' => hash.keys.first.camelcase)
 
       %w[in out].each { |key| (cfg[key] ||= '').downcase! }
 
@@ -240,16 +237,16 @@ class Lingo
       attendee = Attendee.const_get(cfg['name']).new(cfg, self)
       @attendees << attendee
 
-      cfg['in'].split(STRING_SEPERATOR_PATTERN).each { |interest|
+      cfg['in'].split(STRING_SEPARATOR_RE).each { |interest|
         subscriber[interest] << attendee
       }
-      cfg['out'].split(STRING_SEPERATOR_PATTERN).each { |theme|
+      cfg['out'].split(STRING_SEPARATOR_RE).each { |theme|
         supplier[theme] << attendee
       }
     }
 
-    supplier.each { |channel, attendees| attendees.each { |att|
-      att.add_subscriber(subscriber[channel])
+    supplier.each { |channel, attendees| attendees.each { |attendee|
+      attendee.add_subscriber(subscriber[channel])
     } }
   end
 
@@ -257,65 +254,39 @@ class Lingo
     @report_status, @report_time = report_status, report_time
 
     time = Benchmark.realtime {
-      @attendees.first.listen(AgendaItem.new(STR_CMD_TALK))
+      @attendees.first.listen(AgendaItem.new(Attendee::STR_CMD_TALK))
     }
 
     if report_status || report_time
-      config.stderr.puts "Require protocol...\n#{separator = '-' * 61}"
-      @attendees.first.listen(AgendaItem.new(STR_CMD_STATUS))
-      config.stderr.puts "#{separator}\nThe duration of the meeting was #{time.to_hms(2)}"
+      warn "Require protocol...\n#{separator = '-' * 61}"
+      @attendees.first.listen(AgendaItem.new(Attendee::STR_CMD_STATUS))
+      warn "#{separator}\nThe duration of the meeting was #{time.to_hms(2)}"
     end
   end
 
   def reset(close = true)
     dictionaries.each(&:close) if close
     @dictionaries, @attendees = [], []
+    @lexical_hash = Hash.new { |h, k| h[k] = Language::LexicalHash.new(k, self) }
   end
 
-  class Call < Lingo
-
-    def initialize(args = [])
-      super(args, StringIO.new, StringIO.new, StringIO.new)
-    end
-
-    def call
-      invite
-
-      if block_given?
-        begin
-          yield self
-        ensure
-          reset
-        end
-      else
-        self
-      end
-    end
-
-    def talk(str)
-      config.stdin.reopen(str)
-
-      start
-
-      %w[stdout stderr].flat_map { |key|
-        io = config.send(key).tap(&:rewind)
-        io.readlines.each(&:chomp!).tap {
-          io.truncate(0)
-          io.rewind
-        }
-      }.tap { |res|
-        if block_given?
-          res.map!(&Proc.new)
-        else
-          res.sort!
-          res.uniq!
-        end
-      }
-    end
-
+  def warn(*msg)
+    config.stderr.puts(*msg)
   end
 
 end
 
+require_relative 'lingo/call'
+require_relative 'lingo/error'
+require_relative 'lingo/config'
+require_relative 'lingo/core_ext'
+require_relative 'lingo/cachable'
+require_relative 'lingo/reportable'
+require_relative 'lingo/agenda_item'
+require_relative 'lingo/database'
+require_relative 'lingo/language'
+require_relative 'lingo/attendee'
+require_relative 'lingo/version'
+
 require 'nuggets/util/pluggable'
 Util::Pluggable.load_plugins_for(Lingo)

data/lib/lingo/agenda_item.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  class AgendaItem
+
+    include Comparable
+
+    attr_reader :cmd, :param
+
+    def initialize(cmd, param = nil)
+      @cmd, @param = cmd || '', param || ''
+    end
+
+    def <=>(other)
+      other.is_a?(self.class) ? to_a <=> other.to_a : 1
+    end
+
+    def to_a
+      [cmd, param]
+    end
+
+    def inspect
+      "*#{cmd.upcase}('#{param}')"
+    end
+
+  end
+
+end

data/lib/lingo/attendee.rb

@@ -0,0 +1,261 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  # Lingo ist als universelles Indexierungssystem entworfen worden. Seine Stärke liegt in der einfachen Konfigurierbarkeit für
+  # spezifische Aufgaben und in der schnelle Entwicklung weiterer Funktionen durch systematischen Kapselung der Komplexität auf
+  # kleine Verarbeitungseinheiten. Die kleinste Verarbeitungseinheit wird Attendee genannt. Um ein gewünschtes Verarbeitungsergebnis
+  # zu bekommen, werden die benötigten Attendees einfach in einer Reihe hinter einander geschaltet. Ein einfaches Beispiel hierfür ist
+  # eine direkte Verbindung zwischen einem Textreader, einem Tokenizer und einem Textwriter. Alle drei Klassen sind von der Klasse
+  # Attendee abgeleitet.
+  #
+  # Der Textreader liest beispielsweise Zeilen aus einer Textdatei und leitet sie weiter an den Tokenizer. Der Tokenizer zerlegt eine
+  # Textzeile in einzelne Wörter und gibt diese weiter an den Textwriter, der diese in eine (andere) Datei schreibt. Über vielfältige
+  # Konfigurationsmöglichkeiten kann das Verhalten der Attendees an die eigenen Bedürfnisse angepasst werden.
+  #
+  # Die Verkettung einzelner Attendees findet über die Schnittstellen +listen+ und +talk+ statt. An +listen+ können beliebige Objekte
+  # zur Ver- und Bearbeitung übergeben werden. Nach der Verarbeitung werden sie mittels +talk+ an die verketteten Attendees weiter
+  # gegeben. Objekte der Klasse AgendaItem dienen dabei der Steuerung der Verarbeitung und sind nicht Bestandteil der normalen
+  # Verarbeitung. Beispiele für AgendaItems sind die Kommandos TALK (Aufforderung zum Start der Verarbeitung), WARN (zur Ausgabe von
+  # Warnungen eines Attendees) und EOL (End of Line, Ende einer Textzeile nach Zerlegung in einzelne Wörter). Eine vollständige
+  # Übersicht benutzer AgendaItems (oder auf Stream Commands) steht in lib/const.rb mit dem Prefix STR_CMD_.
+  #
+  # Um die Entwicklung von neuen Attendees zu beschleunigen, wird durch die Vererbung sind bei wird die gesammte sind in der Regel nur
+  # drei abstrakte Methoden zu implementieren: +init+, +control+ und +process+. Die Methode +init+ wird bei der Instanziierung eines
+  # Objektes einmalig aufgerufen. Sie dient der Vorbereitung der Verarbeitung, z.B. durch das Öffnen und Bereitstellen von
+  # Wörterbüchern zur linguistischen Analyse. An die Methode +control+ werden alle eingehenden AgendaItems weitergeleitet. Dort erfolgt
+  # die Verarbeitungssteuerung, also z.B. bei STR_CMD_FILE das Öffnen einer Datei und bei STR_CMD_EOF respektive das Schließen. Die
+  # echte Verarbeitung von Daten findet daher durch die Methode +process+ statt.
+  #
+  # was macht attendee
+  # - verkettung der attendees anhand von konfigurationsinformationen
+  # - bereitstellung von globalen und spezifischen konfigurationsinformationen
+  # - behandlung von bestimmten übergreifenden Kommandos, z.B. STR_CMD_TALK, STR_CMD_STATUS
+  # - separierung und routing von kommando bzw. datenobjekten
+  #
+  # was macht die abgeleitet klasse
+  # - verarbeitet und/oder transformiert datenobjekte
+  # - wird gesteuert durch kommandos
+  # - schreibt verarbeitungsstatistiken
+
+  class Attendee
+
+    include Language
+    include Reportable
+
+    STR_CMD_TALK   = 'TALK'
+    STR_CMD_STATUS = 'STATUS'
+    STR_CMD_LIR    = 'LIR-FORMAT'
+    STR_CMD_FILE   = 'FILE'
+    STR_CMD_EOL    = 'EOL'
+    STR_CMD_RECORD = 'RECORD'
+    STR_CMD_EOF    = 'EOF'
+
+    STA_NUM_COMMANDS = 'Received Commands'
+    STA_NUM_OBJECTS  = 'Received Objects '
+    STA_TIM_COMMANDS = 'Time to control  '
+    STA_TIM_OBJECTS  = 'Time to process  '
+
+    def initialize(config, lingo)
+      @lingo = lingo
+
+      init_reportable
+
+      # Make sure config exists
+      lingo.dictionary_config
+
+      @config, @subscriber = config, []
+
+      init if self.class.method_defined?(:init)
+
+      @can_control = self.class.method_defined?(:control)
+      @can_process = self.class.method_defined?(:process)
+
+      @skip_command, @timer = false, nil
+    end
+
+    def add_subscriber(subscriber)
+      @subscriber.concat(subscriber)
+    end
+
+    def listen(obj)
+      unless obj.is_a?(AgendaItem)
+        @can_process ? stat_timer(:objects) { process(obj) } : forward(obj)
+      else
+        args = obj.to_a
+        stat_timer(:commands) { control(*args) } if @can_control
+
+        case obj.cmd
+          when STR_CMD_TALK
+            nil
+          when STR_CMD_STATUS
+            report_time
+            report_status
+
+            forward(*args)
+          else
+            forward(*args) unless skip_command!
+        end
+      end
+    end
+
+    def talk(obj)
+      charge_timer { @subscriber.each { |attendee| attendee.listen(obj) } }
+    end
+
+    private
+
+    def sta_for(key)
+      %w[NUM TIM].map { |i| self.class.const_get("STA_#{i}_#{key.upcase}") }
+    end
+
+    def stat_timer(key)
+      n, t = sta_for(key)
+      inc(n)
+
+      return yield unless @lingo.report_time
+
+      @timer = Time.new
+      res = yield
+      add(t, Time.new - @timer)
+      res
+    end
+
+    def charge_timer
+      return yield unless @lingo.report_time
+
+      res = nil
+      @timer += Benchmark.realtime { res = yield }
+      res
+    end
+
+    def report_time
+      return unless @lingo.report_time
+
+      msg = 'Perf: %-15s ' <<
+            '=> %7d commands in %s (%s/cmd)' <<
+            ',  %8d objects in %s (%s/obj)'
+
+      arg = [@config['name']]
+
+      %w[commands objects].each { |k|
+        n, t = sta_for(k).map(&method(:get))
+        arg << n
+
+        arg.concat([1, n].map { |m|
+          s = m.zero? ? 0.0 : t / m.to_f
+
+          '%9.3f %-2s' %
+            if s < 0.001
+              [s * 1000.0 ** 2, 'µs']
+            elsif s < 1.0
+              [s * 1000.0,      'ms']
+            elsif s < 60.0
+              [s,               's']
+            elsif s < 60.0 ** 2
+              [s / 60.0,        'm']
+            else
+              [s / 60.0 ** 2,   'h']
+            end
+        })
+      }
+
+      @lingo.warn msg % arg
+    end
+
+    def report_status
+      return unless @lingo.report_status
+
+      msg = "Attendee <%s> was connected from '%s' to '%s' reporting..."
+
+      @lingo.warn msg % @config.values_at(*%w[name in out]),
+        nil, report.sort.map { |k, v| " #{k} = #{v}" }, nil
+    end
+
+    def skip_command
+      @skip_command = true
+    end
+
+    def skip_command!
+      @skip_command.tap { @skip_command &&= false }
+    end
+
+    def forward(obj, param = nil)
+      talk(param ? AgendaItem.new(obj, param) : obj)
+    end
+
+    def has_key?(key)
+      @config && @config.has_key?(key)
+    end
+
+    def get_key(key, default = nodefault = Object.new)
+      raise MissingConfigError.new(key) if nodefault && !has_key?(key)
+      @config.fetch(key, default)
+    end
+
+    def get_array(key, default = nil)
+      get_key(key, default).split(STRING_SEPARATOR_RE)
+    end
+
+    def dictionary(src, mod)
+      Language::Dictionary.new({ 'source' => src, 'mode' => mod }, @lingo)
+    end
+
+    def grammar(src, mod)
+      Language::Grammar.new({ 'source' => src, 'mode' => mod }, @lingo)
+    end
+
+    def set_dic
+      @dic = dictionary(get_array('source'), get_key('mode', 'all'))
+    end
+
+    def set_gra
+      @gra = grammar(get_array('source'), get_key('mode', 'all'))
+    end
+
+  end
+
+end
+
+require_relative 'buffered_attendee'
+
+require_relative 'attendee/abbreviator'
+require_relative 'attendee/debugger'
+require_relative 'attendee/decomposer'
+require_relative 'attendee/dehyphenizer'
+require_relative 'attendee/multi_worder'
+require_relative 'attendee/noneword_filter'
+require_relative 'attendee/object_filter'
+require_relative 'attendee/variator'
+require_relative 'attendee/sequencer'
+require_relative 'attendee/synonymer'
+require_relative 'attendee/text_reader'
+require_relative 'attendee/text_writer'
+require_relative 'attendee/formatter'
+require_relative 'attendee/tokenizer'
+require_relative 'attendee/vector_filter'
+require_relative 'attendee/word_searcher'