lingo 1.9.0.pre1 → 1.9.0

data/lib/lingo/database/source/multi_key.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -40,25 +40,27 @@ class Lingo
 
       class MultiKey < self
 
-        DEFAULT_SEPARATOR = ';'
+        DEFAULT_SEPARATOR = ';'.freeze
 
-        def initialize(id, lingo)
+        def initialize(*)
           super
-          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep ||= DEFAULT_SEPARATOR)}#{@wrd})*$/
+          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep)}#{@wrd})*$/
         end
 
         def set(db, key, val)
-          key += "##{@def}" if @def
+          key = lexical(key, @def) if @def
           val.each { |v| db[v] = [key] }
         end
 
-        private
-
-        def convert_line(line, key, val)
-          values = line.split(@sep).each { |i| i.strip! }
+        def parse_line(line, key, val)
+          values = line.split(@sep).each(&:strip!)
           [values.shift, values]
         end
 
+        def dump_line(key, val, sep = @sep, *)
+          val.map(&:form).unshift(key).join(sep)
+        end
+
       end
 
     end

data/lib/lingo/database/source/multi_value.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -38,22 +38,24 @@ class Lingo
 
       class MultiValue < self
 
-        DEFAULT_SEPARATOR = ';'
+        DEFAULT_SEPARATOR = ';'.freeze
 
-        def initialize(id, lingo)
+        def initialize(*)
           super
-          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep ||= DEFAULT_SEPARATOR)}#{@wrd})*$/
+          @pat = /^#{@wrd}(?:#{Regexp.escape(@sep)}#{@wrd})*$/
         end
 
         def set(db, key, val)
-          values = val.map { |v| @def ? "#{v}##{@def}" : v }
+          values = val.map { |v| @def ? lexical(v, @def) : v }
           val.each { |v| db[v] = values }
         end
 
-        private
+        def parse_line(line, key, val)
+          [nil, line.split(@sep).each(&:strip!)]
+        end
 
-        def convert_line(line, key, val)
-          [nil, line.split(@sep).each { |i| i.strip! }]
+        def dump_line(key, val, sep = @sep, *)
+          val.map(&:form).join(sep)
         end
 
       end

data/lib/lingo/database/source/single_word.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -38,16 +38,20 @@ class Lingo
 
       class SingleWord < self
 
-        def initialize(id, lingo)
-          super(id, lingo, Language::LA_NOUN)
+        DEFAULT_DEF_WC = Language::LA_NOUN
+
+        def initialize(*)
+          super
           @pat = /^(#{@wrd})$/
           @mul = @config.fetch('def-mul-wc', @def).downcase
         end
 
-        private
+        def parse_line(line, key, val)
+          [k = key.strip, [lexical(k, k.include?(' ') ? @mul : @def)]]
+        end
 
-        def convert_line(line, key, val)
-          [k = key.strip, %W[#{k}##{k.include?(' ') ? @mul : @def}]]
+        def dump_line(key, val, *)
+          key
         end
 
       end

data/lib/lingo/database/source/word_class.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -38,38 +38,67 @@ class Lingo
 
       class WordClass < self
 
-        DEFAULT_SEPARATOR = ','
+        include ArrayUtils
 
-        GENDER_SEPARATOR  = '.'
+        DEFAULT_SEPARATOR = ','.freeze
 
-        def initialize(id, lingo)
+        GENDER_SEPARATOR  = '.'.freeze
+
+        VALUE_SEPARATOR = '|'.freeze
+
+        WC_SEPARATOR = '#'.freeze
+
+        SCAN_RE = /(\S.*?)\s*#{WC_SEPARATOR}(\S+)/o
+
+        def initialize(*)
           super
 
           gen = Regexp.escape(GENDER_SEPARATOR)
-          sep = Regexp.escape(@sep ||= DEFAULT_SEPARATOR)
+          val = Regexp.escape(VALUE_SEPARATOR)
+          sep = Regexp.escape(@sep)
 
-          w, a = '\w%1$s(?:\|\w%1$s)*', '[+]?'
-          wc   = "##{w % a}(?:#{gen}#{w % ''})?"
+          w, a = "\\w%1$s(?:#{val}\\w%1$s)*", '[+]?'
+          wc   = "#{WC_SEPARATOR}#{w % a}(?:#{gen}#{w % ''})?"
 
           @pat = /^(#{@wrd})#{sep}((?:#{@wrd}#{wc})+)$/
         end
 
-        private
-
-        def convert_line(line, key, val)
+        def parse_line(line, key, val)
           values = []
 
-          val.strip.scan(/(\S.*?)\s*#(\S+)/) { |k, v|
-            v, f = v.split('.')
+          val.strip.scan(SCAN_RE) { |k, v|
+            v, f = v.split(GENDER_SEPARATOR)
+            f = f ? f.split(VALUE_SEPARATOR) : [nil]
 
-            v.split('|').product(f ? f.split('|') : [nil]) { |w, g|
-              values << "#{k}##{w}##{g}"
+            combinations(v.split(VALUE_SEPARATOR), f) { |w, g|
+              values << lexical(k, w, g)
             }
           }
 
           [key.strip, values]
         end
 
+        def dump_line(key, val, key_sep = nil, val_sep = nil, compact = true, *)
+          "#{key}#{key_sep || @sep}#{dump_values(val, compact).join(val_sep || ' ')}"
+        end
+
+        def dump_values(val, compact = true)
+          join = lambda { |v|
+            v.compact!; v.uniq!; v.sort!; v.join(VALUE_SEPARATOR) }
+
+          if compact
+            values = Hash.new { |h, k| h[k] = [[], []] }; val.each { |lex|
+              a, g = values[lex.form]; a << lex.attr; g << lex.gender }
+          else
+            values = val.map { |lex| [lex.form, [[lex.attr], [lex.gender]]] }
+          end
+
+          values.sort.map { |form, (attrs, genders)|
+            res = "#{form} #{WC_SEPARATOR}#{join[attrs]}"
+            genders.any? ? "#{res}#{GENDER_SEPARATOR}#{join[genders]}" : res
+          }
+        end
+
       end
 
     end

data/lib/lingo/debug.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -67,7 +67,7 @@ class Lingo
           html:  :GraphHtmlPrinter,
           stack: :CallStackPrinter
         }.each { |ext, name|
-          File.open("#{base}.#{ext}", 'a+', encoding: ENC) { |f|
+          File.open("#{base}.#{ext}", 'a+', encoding: ENCODING) { |f|
             RubyProf.const_get(name).new(result).print(f)
           }
         }

data/lib/lingo/error.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -38,7 +38,9 @@ class Lingo
 
   end
 
-  class NoWritableStoreError < LingoError
+  class StoreError < LingoError; end
+
+  class NoWritableStoreError < StoreError
 
     attr_reader :file, :path
 
@@ -52,7 +54,9 @@ class Lingo
 
   end
 
-  class BackendNotFoundError < LingoError
+  class BackendError < LingoError; end
+
+  class BackendNotFoundError < BackendError
 
     attr_reader :file
 
@@ -66,7 +70,7 @@ class Lingo
 
   end
 
-  class BackendNotAvailableError < LingoError
+  class BackendNotAvailableError < BackendError
 
     attr_reader :name, :file, :err
 
@@ -143,7 +147,7 @@ class Lingo
 
   end
 
-  class FileNotFoundError < LingoError
+  class FileError < LingoError
 
     attr_reader :name
 
@@ -151,6 +155,10 @@ class Lingo
       @name = name
     end
 
+  end
+
+  class FileNotFoundError < FileError
+
     def to_s
       "No such file `#{name}'."
     end
@@ -172,6 +180,14 @@ class Lingo
 
   end
 
+  class FileExistsError < FileError
+
+    def to_s
+      "File `#{name}' already exists."
+    end
+
+  end
+
   class NameNotFoundError < LingoError
 
     attr_reader :klass, :name

data/lib/lingo/filter.rb

@@ -28,7 +28,7 @@ class Lingo
 
   class Filter
 
-    def initialize(io, encoding = ENC)
+    def initialize(io, encoding = ENCODING)
       @io, @encoding = io, encoding
     end
 

data/lib/lingo/language.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -37,27 +37,27 @@ class Lingo
 
   module Language
 
-    CHAR_PUNCT = '.'
+    CHAR_PUNCT = '.'.freeze
 
-    TA_ABBREVIATION = 'ABRV'
-    TA_HELP         = 'HELP'
-    TA_HTML         = 'HTML'
-    TA_NUMBER       = 'NUMS'
-    TA_OTHER        = 'OTHR'
-    TA_PUNCTUATION  = 'PUNC'
-    TA_SKIP         = 'SKIP'
-    TA_SPACE        = 'SPAC'
-    TA_URL          = 'URLS'
-    TA_WIKI         = 'WIKI'
-    TA_WORD         = 'WORD'
+    TA_ABBREVIATION = 'ABRV'.freeze
+    TA_HELP         = 'HELP'.freeze
+    TA_HTML         = 'HTML'.freeze
+    TA_NUMBER       = 'NUMS'.freeze
+    TA_OTHER        = 'OTHR'.freeze
+    TA_PUNCTUATION  = 'PUNC'.freeze
+    TA_SKIP         = 'SKIP'.freeze
+    TA_SPACE        = 'SPAC'.freeze
+    TA_URL          = 'URLS'.freeze
+    TA_WIKI         = 'WIKI'.freeze
+    TA_WORD         = 'WORD'.freeze
 
-    WA_UNSET      = '-'
-    WA_IDENTIFIED = 'IDF'
-    WA_UNKNOWN    = '?'
-    WA_COMPOUND   = 'COM'
-    WA_MULTIWORD  = 'MUL'
-    WA_SEQUENCE   = 'SEQ'
-    WA_UNKMULPART = 'MU?'
+    WA_UNSET      = '-'.freeze
+    WA_IDENTIFIED = 'IDF'.freeze
+    WA_UNKNOWN    = '?'.freeze
+    WA_COMPOUND   = 'COM'.freeze
+    WA_MULTIWORD  = 'MUL'.freeze
+    WA_SEQUENCE   = 'SEQ'.freeze
+    WA_UNKMULPART = 'MU?'.freeze
 
     LA_SORTORDER = [
       LA_SEQUENCE   = 'q',
@@ -73,7 +73,7 @@ class Lingo
       LA_SYNONYM    = 'y',
       LA_STEM       = 'z',
       LA_UNKNOWN    = '?'
-    ].each_with_index.inject({}) { |h, (i, j)| h[i] = j; h }
+    ].each_with_index.inject({}) { |h, (i, j)| h[i.freeze] = j; h }
 
   end
 

data/lib/lingo/language/grammar.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -37,6 +37,8 @@ class Lingo
 
     class Grammar
 
+      include ArrayUtils
+
       HYPHEN_RE = %r{\A(.+)-([^-]+)\z}
 
       DEFAULTS = {
@@ -170,7 +172,7 @@ class Lingo
 
         flex.concat(blex).delete_if { |lex| lex.attr == LA_COMPOUND }
 
-        [forms.shift.product(*forms).map { |front, back|
+        [combinations(*forms).map { |front, back|
           Lexical.new(front + infix + back, LA_COMPOUND)
         }.concat(flex), sta, seq.join]
       end

data/lib/lingo/language/lexical_hash.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -36,8 +36,6 @@ class Lingo
 
     class LexicalHash
 
-      KEY_REF_RE = %r{\A#{Regexp.escape(Database::KEY_REF)}(\d+)\z}o
-
       def self.open(*args)
         yield lexical_hash = new(*args)
       ensure
@@ -53,17 +51,7 @@ class Lingo
       end
 
       def [](key)
-        rec = @src[Unicode.downcase(key)] or return
-
-        res = rec.map { |str|
-          str =~ KEY_REF_RE ? $1.to_i : begin
-            k, *w = str.split('#')
-            Lexical.new(k.strip, w)
-          end
-        }
-
-        res.uniq!
-        res
+        Database::Source.lexicals(@src[Unicode.downcase(key)])
       end
 
     end

data/lib/lingo/language/word.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -41,10 +41,6 @@ class Lingo
           new(form, attr) << lex
         end
 
-        def new_lexical(form, attr, lex_attr)
-          new_lexicals(form, attr, Lexical.new(form, lex_attr))
-        end
-
         def new_compound_head(lex, attr = WA_UNSET)
           form, head_lex = nil, []