lingo 1.8.6 → 1.8.7

data/lib/lingo/database/crypter.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -31,14 +31,16 @@ class Lingo
 
   class Database
 
-    class Crypter
+    module Crypter
 
-      def self.digest(key)
+      extend self
+
+      def digest(key)
         Digest::SHA1.hexdigest(key)
       end
 
       def encode(key, val)
-        [self.class.digest(key), crypt(:encrypt, key, val)]
+        [digest(key), crypt(:encrypt, key, val)]
       end
 
       def decode(key, val)
@@ -49,7 +51,7 @@ class Lingo
 
       def crypt(method, key, val)
         cipher = OpenSSL::Cipher.new('aes-128-cbc').send(method)
-        cipher.iv = cipher.key = self.class.digest(key)
+        cipher.iv = cipher.key = digest(key)
         cipher.update(val) + cipher.final
       end
 

data/lib/lingo/error.rb

@@ -68,14 +68,15 @@ class Lingo
 
   class BackendNotAvailableError < LingoError
 
-    attr_reader :mod, :file
+    attr_reader :name, :file, :err
 
-    def initialize(mod, file)
-      @mod, @file = mod, file
+    def initialize(name, file, err)
+      @name, @file, @err = name, file, err
     end
 
     def to_s
-      "Backend not available `#{mod}' for `#{file}'."
+      msg = "Backend not available `#{name}'"
+      error(file ? msg << " for `#{file}'" : msg)
     end
 
   end

data/lib/lingo/language.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -39,14 +39,22 @@ class Lingo
 
     CHAR_PUNCT = '.'
 
-    TA_WORD        = 'WORD'
-    TA_PUNCTUATION = 'PUNC'
-    TA_OTHER       = 'OTHR'
+    TA_ABBREVIATION = 'ABRV'
+    TA_HELP         = 'HELP'
+    TA_HTML         = 'HTML'
+    TA_NUMBER       = 'NUMS'
+    TA_OTHER        = 'OTHR'
+    TA_PUNCTUATION  = 'PUNC'
+    TA_SKIP         = 'SKIP'
+    TA_SPACE        = 'SPAC'
+    TA_URL          = 'URLS'
+    TA_WIKI         = 'WIKI'
+    TA_WORD         = 'WORD'
 
     WA_UNSET      = '-'
     WA_IDENTIFIED = 'IDF'
     WA_UNKNOWN    = '?'
-    WA_COMPOUND   = 'KOM'
+    WA_COMPOUND   = 'COM'
     WA_MULTIWORD  = 'MUL'
     WA_SEQUENCE   = 'SEQ'
     WA_UNKMULPART = 'MU?'

data/lib/lingo/language/grammar.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -39,6 +39,10 @@ class Lingo
 
       HYPHEN_RE = %r{\A(.+)-([^-]+)\z}
 
+      DEFAULTS = {
+        min_word_size: 8, min_avg_part_size: 4, min_part_size: 1, max_parts: 4
+      }
+
       def self.open(*args)
         yield grammar = new(*args)
       ensure
@@ -55,11 +59,8 @@ class Lingo
         cfg = lingo.dictionary_config['compound'] ||
               lingo.dictionary_config['compositum']  # DEPRECATE compositum
 
-        {
-          min_word_size: 8, min_avg_part_size: 4, min_part_size: 1, max_parts: 4
-        }.each { |k, v|
-          instance_variable_set("@#{k}", cfg.fetch(k.to_s.tr('_', '-'), v).to_i)
-        }
+        DEFAULTS.each { |k, v| instance_variable_set(
+          "@#{k}", cfg.fetch(k.to_s.tr('_', '-'), v).to_i) }
 
         #--
         # Die Wortklasse eines Kompositum-Wortteils kann separat gekennzeichnet
@@ -87,6 +88,11 @@ class Lingo
           permute_compound([[], [], ''],              str, level, tail)
       end
 
+      def find_compound_head(str)
+        compound = find_compound(str)
+        compound.head || compound if compound && !compound.unknown?
+      end
+
       private
 
       def permute_compound(ret, str, level, tail)
@@ -113,7 +119,7 @@ class Lingo
 
           level > 1 ? ret = res : ret.identify(lex.each { |l|
             l.attr += @append_wc unless l.attr == LA_COMPOUND
-          }, WA_COMPOUND) if !lex.empty? &&
+          }, WA_COMPOUND, seq) if !lex.empty? &&
             sta.size              <= @max_parts         &&
             sta.min               >= @min_part_size     &&
             str.length / sta.size >= @min_avg_part_size &&

data/lib/lingo/language/token.rb

@@ -56,6 +56,12 @@ class Lingo
         attr == TA_WORD
       end
 
+      alias_method :word_token?, :word?
+
+      def number?
+        attr == TA_NUMBER
+      end
+
       def position_and_offset
         "#{position}#{POSITION_SEP}#{offset}"
       end

data/lib/lingo/language/word.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -81,65 +81,52 @@ class Lingo
         super
       end
 
-      attr_reader :token
-
-      attr_writer :lexicals
-
-      def lexicals(compound_parts = true)
-        if !compound_parts && attr == WA_COMPOUND
-          @lexicals.select { |lex| lex.attr == LA_COMPOUND }
-        else
-          @lexicals
-        end
-      end
+      attr_accessor :lexicals, :pattern
 
       def add_lexicals(lex)
-        @lexicals.concat(lex - @lexicals)
+        lexicals.concat(lex - lexicals)
       end
 
       def attr?(*attr)
         !(attrs & attr).empty?
       end
 
-      def attrs(compound_parts = true)
-        lexicals(compound_parts).map { |i| i.attr }
+      def attrs
+        lexicals.map(&:attr)
+      end
+
+      def compound_attrs
+        attr == WA_COMPOUND ? attrs.grep(LA_COMPOUND) : attrs
       end
 
-      def genders(compound_parts = true)
-        lexicals(compound_parts).map { |i| i.gender }
+      def genders
+        lexicals.map(&:gender)
       end
 
-      def identify(lex, wc = nil)
+      def identify(lex, wc = nil, seq = nil)
         return self if lex.empty?
 
-        self.lexicals = lex
+        self.lexicals, self.pattern = lex, seq
         self.attr = wc ||= attr?(LA_COMPOUND) ? WA_COMPOUND : WA_IDENTIFIED
         self.head = self.class.new_compound_head(lex) if wc == WA_COMPOUND
 
         self
       end
 
-      def get_class(wc_re)
-        wc_re = Regexp.new(wc_re) unless wc_re.is_a?(Regexp)
-
-        lexicals.empty? ? attr =~ wc_re ? [self] : [] :
-          lexicals.select { |lex| lex.attr =~ wc_re }
-      end
+      def each_lex(wc_re = //)
+        return enum_for(:each_lex, wc_re) unless block_given?
 
-      def norm
-        identified? ? lexicals.first.form : form
-      end
+        wc_re = Regexp.new(wc_re) unless wc_re.is_a?(Regexp)
 
-      def compo_form
-        get_class(LA_COMPOUND).first if attr == WA_COMPOUND
-      end
+        lexicals.empty? ? attr =~ wc_re ? yield(self) : nil :
+          lexicals.each { |lex| yield lex if lex.attr =~ wc_re }
 
-      def full_compound?
-        attr == WA_COMPOUND && get_class('x+').empty?
+        nil
       end
 
-      def multiword_size(wc_re = LA_MULTIWORD)
-        lex = get_class(wc_re).first and lex.form.count(' ') + 1
+      def lex_form(wc_re = //)
+        each_lex(wc_re) { |lex|
+          break block_given? ? yield(lex.form) : lex.form }
       end
 
       def position_and_offset
@@ -148,7 +135,7 @@ class Lingo
 
       def <<(*lex)
         lex.flatten!
-        @lexicals.concat(lex)
+        lexicals.concat(lex)
         self
       end
 

data/lib/lingo/language/word_form.rb

@@ -43,7 +43,7 @@ class Lingo
         @form, @attr, @src = form || '', attr || '', src
       end
 
-      attr_accessor :form, :attr, :gender, :src, :head
+      attr_accessor :form, :attr, :gender, :src, :token, :head
 
       def unknown?
         [WA_UNKNOWN, WA_UNKMULPART].include?(attr)
@@ -53,6 +53,10 @@ class Lingo
         attr == WA_IDENTIFIED
       end
 
+      def word_token?
+        false
+      end
+
       def <=>(other)
         other.nil? ? 1 : to_a <=> other.to_a
       end

data/lib/lingo/srv.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -58,7 +58,7 @@ class Lingo
       r = LINGO.talk(q) unless q.empty?
       return r unless r && SRC_SEP
 
-      s = Hash.nest { [] }
+      s = Hash.array
 
       r.each { |t|
         a, b = t.split(SRC_SEP, 2)

data/lib/lingo/text_utils.rb

@@ -0,0 +1,96 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'nuggets/file/ext'
+
+class Lingo
+
+  module TextUtils
+
+    STDIN_EXT = %w[STDIN -].freeze
+
+    STDOUT_EXT = %w[STDOUT -].freeze
+
+    GZIP_RE = %r{\.gz(?:ip)?\z}i
+
+    private
+
+    def stdin?(path)
+      STDIN_EXT.include?(path)
+    end
+
+    def stdout?(path)
+      STDOUT_EXT.include?(path)
+    end
+
+    def open_stdin
+      stdin = set_encoding(lingo.config.stdin)
+      @progress ? StringIO.new(stdin.read) : stdin
+    end
+
+    def open_stdout
+      set_encoding(lingo.config.stdout)
+    end
+
+    def open_path(path, mode = 'rb')
+      path =~ GZIP_RE ? open_gzip(path, mode) : open_file(path, mode)
+    end
+
+    def open_file(path, mode)
+      File.open(path, mode, encoding: bom_encoding(mode))
+    end
+
+    def open_gzip(path, mode)
+      require_lib('zlib')
+
+      case mode
+        when 'r', 'rb'
+          @progress = false
+          Zlib::GzipReader.open(path, encoding: @encoding)
+        when 'w', 'wb'
+          Zlib::GzipWriter.open(path, encoding: @encoding)
+        else
+          raise ArgumentError, 'invalid access mode %s' % mode
+      end
+    end
+
+    def set_ext(path, ext)
+      File.set_ext(path.sub(GZIP_RE, ''), ".#{ext}")
+    end
+
+    def set_encoding(io, encoding = @encoding)
+      io.set_encoding(encoding)
+      io
+    end
+
+    def bom_encoding(mode = 'r', encoding = @encoding)
+      (mode.include?('r') || mode.include?('+')) &&
+        encoding.name.start_with?('UTF-') ? "BOM|#{encoding}" : encoding
+    end
+
+  end
+
+end

data/lib/lingo/version.rb

@@ -4,7 +4,7 @@ class Lingo
 
     MAJOR = 1
     MINOR = 8
-    TINY  = 6
+    TINY  = 7
 
     class << self
 

data/lib/lingo/web/views/index.erb

@@ -98,7 +98,7 @@
       <tr><th>m (=MUL)</th><td><%= t 'Phrase',
                                      'Mehrwortbegriff',
                                      'составная лексема' %></td></tr>
-      <tr><th>k (=KOM)</th><td><%= t 'Compound',
+      <tr><th>k (=COM)</th><td><%= t 'Compound',
                                      'Kompositum',
                                      'сложное слово' %></td></tr>
       <tr><th>+       </th><td><%= t 'Part of a compound',

data/test/attendee/ts_decomposer.rb

@@ -12,11 +12,29 @@ class TestAttendeeDecomposer < AttendeeTestCase
       wd('Kompositumzerlegung|?'),
       wd('Kompositumzerlegung|?')
     ], [
-      wd('Kleinseite|KOM', 'kleinseite|k', 'klein|a+', 'seite|s+'),
-      wd('Arrafat-Nachfolger|KOM', 'arrafat-nachfolger|k', 'arrafat|x+', 'nachfolger|s+'),
-      wd('Afganistan-Reisen|KOM', 'afganistan-reisen|k', 'afganistan|x+', 'reisen|v+', 'reise|s+'),
-      wd('Kompositumzerlegung|KOM', 'kompositumzerlegung|k', 'kompositum|s+', 'zerlegung|s+'),
-      wd('Kompositumzerlegung|KOM', 'kompositumzerlegung|k', 'kompositum|s+', 'zerlegung|s+')
+      wd('Kleinseite|COM', 'kleinseite|k', 'klein|a+', 'seite|s+'),
+      wd('Arrafat-Nachfolger|COM', 'arrafat-nachfolger|k', 'arrafat|x+', 'nachfolger|s+'),
+      wd('Afganistan-Reisen|COM', 'afganistan-reisen|k', 'afganistan|x+', 'reisen|v+', 'reise|s+'),
+      wd('Kompositumzerlegung|COM', 'kompositumzerlegung|k', 'kompositum|s+', 'zerlegung|s+'),
+      wd('Kompositumzerlegung|COM', 'kompositumzerlegung|k', 'kompositum|s+', 'zerlegung|s+')
+    ])
+  end
+
+  def test_nums
+    meet({ 'source' => 'sys-dic' }, [
+      wd('123-Reisen|?'),
+      wd('abc123-Reisen|?'),
+      wd('Reisen-24|?'),
+      wd('Reisen-123|?'),
+      wd('Reisen-24-Seite|?'),
+      wd('Reisen-123-Seite|?')
+    ], [
+      wd('123-Reisen|COM', '123-reisen|k', '123|x+', 'reisen|v+', 'reise|s+'),
+      wd('abc123-Reisen|COM', 'abc123-reisen|k', 'abc123|x+', 'reisen|v+', 'reise|s+'),
+      wd('Reisen-24|?'),
+      wd('Reisen-123|COM', 'reisen-123|k', 'reisen|v+', 'reise|s+', '123|x+'),
+      wd('Reisen-24-Seite|COM', 'reisen-24-seite|k', 'reisen-24|x+', 'seite|s+'),
+      wd('Reisen-123-Seite|COM', 'reisen-123-seite|k', 'reisen|v+', 'reise|s+', '123|x+', 'seite|s+')
     ])
   end