lingo 1.8.7 → 1.9.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1130ec52467314ba95af17e635888f60046c5b42
-  data.tar.gz: 6a882ea4f88b1fbcf1a66b1d5fafe8fa05458b89
+  metadata.gz: d863ede7d1dda418b36230165f9f635a8977d73a
+  data.tar.gz: 9d7caed0a6d27898605b97429b48965617c96960
 SHA512:
-  metadata.gz: 3e0b384a822c28961c99d411bbdd399d9a49b29fdc40688d3534a9897cef984a7447cdd21b05fda45dda9a0fad25d99f60d524064803edb218ff95bbf9fc4fe6
-  data.tar.gz: 5cfa5c7f235113238d0e8568e9948f27a7ec864af63ac696a5efdfceed6eb678724f3ca3df24d3ac296068aceb1bf20d1369f439edf7f670265550ace1ce1cd0
+  metadata.gz: 62044e35ee507ff911b0aa7b656d6247f7f34a168d23e0da2d7f64dd8010e73b0258cbdda3ea1d61a010c3ca1204c442a02e04866bac8afb92a07d9309ed5c9c
+  data.tar.gz: eb4df8fa604f9f36cbb8094041519ca9198d5a59b624ef70afb937fc9120b8afa2b1d48f95732a2ffb0a253bf2aba5dc16bdb61543f9217926b94b5f80f3e1f8

data/ChangeLog

@@ -2,6 +2,17 @@
 
 = Revision history for Lingo
 
+== 1.9.0 [unreleased]
+
+* Removed support for deprecated options and attendee names (+old+ → +new+):
+  * Lingo::Language::Grammar : +compositum+ → +compound+
+  * Lingo::Attendee::TextReader : +lir-record-pattern+ → +records+
+  * Lingo::Config : +multiworder+ → +multi_worder+, +objectfilter+ →
+    +object_filter+, +textreader+ → +text_reader+, +textwriter+ →
+    +text_writer+, +vectorfilter+ → +vector_filter+, +wordsearcher+ →
+    +word_searcher+
+* Fixed errors with XML input (issue #15 by Thomas Berger).
+
 == 1.8.7 [2015-08-07]
 
 * Added Lingo::Attendee::LsiFilter to correlate semantically related terms

data/README

@@ -34,7 +34,7 @@
 
 == VERSION
 
-This documentation refers to Lingo version 1.8.7
+This documentation refers to Lingo version 1.9.0
 
 
 == DESCRIPTION
@@ -555,7 +555,7 @@ Lingo is based on a collective development by Klaus Lepsky and John Vorhauer.
 == LICENSE AND COPYRIGHT
 
 Copyright (C) 2005-2007 John Vorhauer
-Copyright (C) 2007-2015 John Vorhauer, Jens Wille
+Copyright (C) 2007-2016 John Vorhauer, Jens Wille
 
 Lingo is free software: you can redistribute it and/or modify it under the
 terms of the GNU Affero General Public License as published by the Free

data/Rakefile

@@ -36,16 +36,18 @@ The main functions of Lingo are:
       ].to_a,
 
       dependencies: {
-        'cyclops'       => '~> 0.1',
-        'nuggets'       => '~> 1.3',
+        'cyclops'       => '~> 0.2',
+        'nuggets'       => '~> 1.4',
         'rubyzip'       => '~> 1.1',
-        'sinatra-bells' => '~> 0.0',
+        'sinatra-bells' => '~> 0.3',
         'unicode'       => '~> 0.4'
       },
 
       development_dependencies: {
-        'diff-lcs' => '~> 1.2',
-        'open4'    => '~> 1.3'
+        'diff-lcs'   => '~> 1.2',
+        'nokogiri'   => '~> 1.6',
+        'open4'      => '~> 1.3',
+        'pdf-reader' => '~> 1.3'
       },
 
       required_ruby_version: '>= 1.9.3'

data/lib/lingo.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -318,6 +318,7 @@ require_relative 'lingo/call'
 require_relative 'lingo/error'
 require_relative 'lingo/debug'
 require_relative 'lingo/config'
+require_relative 'lingo/filter'
 require_relative 'lingo/progress'
 require_relative 'lingo/database'
 require_relative 'lingo/language'

data/lib/lingo/attendee/multi_worder.rb

@@ -236,10 +236,6 @@ class Lingo
 
     end
 
-    # For backwards compatibility.
-    Multiworder  = MultiWorder
-    Multi_worder = MultiWorder
-
   end
 
 end

data/lib/lingo/attendee/object_filter.rb

@@ -85,10 +85,6 @@ class Lingo
 
     end
 
-    # For backwards compatibility.
-    Objectfilter  = ObjectFilter
-    Object_filter = ObjectFilter
-
   end
 
 end

data/lib/lingo/attendee/text_reader.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -26,15 +26,12 @@
 
 require 'find'
 
-%w[filemagic mime/types nokogiri nuggets/file/which pdf-reader].each { |lib|
-  begin
-    require lib
-  rescue LoadError
-  end
-}
-
 class Lingo
 
+  require_optional 'filemagic'
+  require_optional 'mime/types'
+  require_optional 'nuggets/file/which'
+
   class Attendee
 
     #--
@@ -118,11 +115,7 @@ class Lingo
         @filter   = get_key('filter', false)
         @progress = get_key('progress', false)
 
-        if has_key?('lir-record-pattern')
-          lingo.config.deprecate('lir-record-pattern', :records, self)
-        end
-
-        @lir  = get_re('records', get_key('lir-record-pattern', nil), %r{^\[(\d+)\.\]})  # DEPRECATE lir-record-pattern
+        @lir  = get_re('records', nil, %r{^\[(\d+)\.\]})
         @cut  = get_re('fields', !!@lir, %r{^.+?:\s*})
         @skip = get_re('skip', nil)
       end
@@ -165,8 +158,8 @@ class Lingo
       def filter(io, path, progress)
         case @filter == true ? file_type(io, path) : @filter.to_s
           when 'pdftotext' then filter_pdftotext(io, path, progress)
-          when /html/i     then filter_html(io)
-          when /xml/i      then filter_html(io, true)
+          when /html/i     then filter_xml(io, :HTML)
+          when /xml/i      then filter_xml(io)
           when /pdf/i      then filter_pdf(io)
           else io
         end
@@ -185,13 +178,13 @@ class Lingo
       end
 
       def filter_pdf(io)
-        Object.const_defined?(:PDF) && PDF.const_defined?(:Reader) ? text_enum(
-          PDF::Reader.new(io).pages) : cancel_filter(:PDF, 'pdf-reader')
+        cancel_filter(:PDF, 'pdf-reader') unless Object.const_defined?(:PDF)
+        Filter::PDF.new(io, @encoding)
       end
 
-      def filter_html(io, xml = false, type = xml ? :XML : :HTML)
-        Object.const_defined?(:Nokogiri) ? text_enum(Nokogiri.send(type,
-          io, nil, @encoding).children) : cancel_filter(type, :nokogiri)
+      def filter_xml(io, type = :XML)
+        cancel_filter(type, :nokogiri) unless Object.const_defined?(:Nokogiri)
+        Filter.const_get(type).new(io, @encoding)
       end
 
       def file_type(io, path)
@@ -239,10 +232,6 @@ class Lingo
         tempfiles.each(&:unlink)
       end
 
-      def text_enum(collection)
-        Enumerator.new { |y| collection.each { |x| y << x.text } }
-      end
-
       def get_files
         args = [get_key('glob', '*.txt'), get_key('recursive', false)]
 
@@ -263,10 +252,6 @@ class Lingo
 
     end
 
-    # For backwards compatibility.
-    Textreader  = TextReader
-    Text_reader = TextReader
-
   end
 
 end

data/lib/lingo/attendee/text_writer.rb

@@ -145,10 +145,6 @@ class Lingo
 
     end
 
-    # For backwards compatibility.
-    Textwriter  = TextWriter
-    Text_writer = TextWriter
-
   end
 
 end

data/lib/lingo/attendee/vector_filter.rb

@@ -240,10 +240,6 @@ class Lingo
 
     end
 
-    # For backwards compatibility.
-    Vectorfilter  = VectorFilter
-    Vector_filter = VectorFilter
-
   end
 
 end

data/lib/lingo/attendee/word_searcher.rb

@@ -83,10 +83,6 @@ class Lingo
 
     end
 
-    # For backwards compatibility.
-    Wordsearcher  = WordSearcher
-    Word_searcher = WordSearcher
-
   end
 
 end

data/lib/lingo/config.rb

@@ -42,12 +42,7 @@ class Lingo
       load_config('language', :lang)
       load_config('config')
 
-      if Array(self['meeting/attendees']).flat_map(&:keys).include?('textreader')
-        deprecate(:textreader, :text_reader)
-      end
-
-      if r = get('meeting/attendees', 'text_reader') ||
-             get('meeting/attendees', 'textreader')  # DEPRECATE textreader
+      if r = get('meeting/attendees', 'text_reader')
         f = @cli.files
 
         if i = r['files']
@@ -110,11 +105,11 @@ class Lingo
       @cli.send(:quit, *args)
     end
 
-    def deprecate(old, new, obj = self, what = :option)
+    def deprecate(old, new, obj = self, what = :option, ver = Version.next_minor)
       unless @deprecated[[source = obj.class.name.sub(/\ALingo::/, ''), old]]
         warn(
           "DEPRECATION WARNING: #{source} #{what} `#{old}' is deprecated " <<
-          "and will be removed in Lingo 1.9. Please use `#{new}' instead."
+          "and will be removed in Lingo #{ver}. Please use `#{new}' instead."
         )
       end
     end

data/lib/lingo/filter.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  class Filter
+
+    def initialize(io, encoding = ENC)
+      @io, @encoding = io, encoding
+    end
+
+    def each
+      raise NotImplementedError, 'must be implemented by subclass'
+    end
+
+    def close
+      @io.close
+    end
+
+  end
+
+end
+
+require_relative 'filter/pdf'
+require_relative 'filter/xml'

data/lib/lingo/filter/pdf.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  require_optional 'pdf-reader'
+
+  class Filter
+
+    class PDF < self
+
+      def initialize(*args)
+        super
+        @obj = ::PDF::Reader.new(@io)
+      end
+
+      def each(&block)
+        @obj.pages.each { |x| x.text.each_line(&block) }
+      end
+
+    end
+
+  end
+
+end

data/lib/lingo/filter/xml.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  require_optional 'nokogiri'
+
+  class Filter
+
+    class XML < self
+
+      TYPE = :XML
+
+      def initialize(*args)
+        super
+        @obj = Nokogiri.send(self.class::TYPE, @io, nil, @encoding.to_s)
+      end
+
+      def each(&block)
+        @obj.root.element_children.each { |n| n.content.each_line(&block) }
+      end
+
+    end
+
+    class HTML < XML
+
+      TYPE = :HTML
+
+    end
+
+  end
+
+end

data/lib/lingo/language/grammar.rb

@@ -52,12 +52,7 @@ class Lingo
       def initialize(config, lingo)
         @dic, @suggestions = Dictionary.new(config, lingo), []
 
-        if lingo.dictionary_config.key?('compositum')
-          lingo.config.deprecate(:compositum, :compound, self)
-        end
-
-        cfg = lingo.dictionary_config['compound'] ||
-              lingo.dictionary_config['compositum']  # DEPRECATE compositum
+        cfg = lingo.dictionary_config['compound']
 
         DEFAULTS.each { |k, v| instance_variable_set(
           "@#{k}", cfg.fetch(k.to_s.tr('_', '-'), v).to_i) }

data/lib/lingo/language/word.rb

@@ -114,7 +114,7 @@ class Lingo
       end
 
       def each_lex(wc_re = //)
-        return enum_for(:each_lex, wc_re) unless block_given?
+        return enum_for(__method__, wc_re) unless block_given?
 
         wc_re = Regexp.new(wc_re) unless wc_re.is_a?(Regexp)