lingo 1.8.6 → 1.8.7

data/lib/lingo/attendee/text_writer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -79,7 +79,11 @@ class Lingo
 
     class TextWriter < self
 
+      include TextUtils
+
       def init
+        @encoding = get_enc
+
         @ext = get_key('ext', 'txt2')
         @lir = get_key('lir-format', false)
 
@@ -97,11 +101,8 @@ class Lingo
           when :FILE
             @no_sep = true
 
-            if stdout?(@ext)
-              @filename, @file = @ext, lingo.config.stdout
-            else
-              @file = File.open(@filename = File.set_ext(param, ".#{@ext}"), 'w')
-            end
+            @io = stdout?(@ext) ? (@path = @ext; open_stdout) :
+              open_path(@path = set_ext(param, @ext), 'w')
 
             @lir_rec_no, @lir_rec_buf = '', []
           when :RECORD
@@ -113,16 +114,10 @@ class Lingo
             end
           when :EOL
             @no_sep = true
-
-            unless @lir
-              @file.puts unless @no_puts
-            end
+            @io.puts unless @lir || @no_puts
           when :EOF
             flush_lir_buffer if @lir
-
-            unless stdout?(@filename)
-              @file.close
-            end
+            @io.close unless stdout?(@path)
         end
       end
 
@@ -130,8 +125,8 @@ class Lingo
         obj = obj.form if obj.is_a?(WordForm)
 
         @lir ? @lir_rec_buf << obj : begin
-          @no_sep ? @no_sep = false : @file.print(@sep)
-          @file.print(obj)
+          @no_sep ? @no_sep = false : @io.print(@sep)
+          @io.print(obj)
         end
       end
 
@@ -141,17 +136,13 @@ class Lingo
         unless @lir_rec_no.empty? || @lir_rec_buf.empty?
           buf = [@lir_rec_no, @lir_rec_buf.join(@sep), "\n"]
           @sep =~ /\n/ ? buf.insert(1, "\n").unshift('*') : buf.insert(1, '*')
-          @file.print(*buf)
+          @io.print(*buf)
         end
 
         @lir_rec_no = ''
         @lir_rec_buf.clear
       end
 
-      def stdout?(filename)
-        %w[STDOUT -].include?(filename)
-      end
-
     end
 
     # For backwards compatibility.

data/lib/lingo/attendee/tokenizer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -84,29 +84,40 @@ class Lingo
 
       CHAR, DIGIT = Char::CHAR, Char::DIGIT
 
+      ALNUM = "(?:#{CHAR}|#{DIGIT})"
+
       PROTO = '(?:news|https?|ftps?)://'
 
       RULES = [
-        ['SPAC', /^\s+/],
-        ['WIKI', /^=+.+=+|^__[A-Z]+__/],
-        ['NUMS', /^[+-]?(?:\d{4,}|\d{1,3}(?:\.\d{3,3})*)(?:\.|(?:,\d+)?%?)/],
-        ['URLS', /^(?:www\.|mailto:|#{PROTO}|\S+?[._]\S+?@\S+?\.)[^\s<>]+/],
-        ['ABRV', /^(?:(?:(?:#{CHAR})+\.)+)(?:#{CHAR})+/],
-        ['WORD', /^(?:#{CHAR}|#{DIGIT}|-)+/],
-        ['PUNC', /^[!,.:;?¡¿]+/]
+        [TA_SPACE,
+          /^\s+/],
+        [TA_WIKI,
+          /^=+.+=+|^__[A-Z]+__/],
+        [TA_NUMBER,
+          /^[+-]?(?:\d{4,}|\d{1,3}(?:\.\d{3,3})*)(?:\.|(?:,\d+)?%?)/],
+        [TA_URL,
+          /^(?:www\.|mailto:|#{PROTO}|\S+?[._]\S+?@\S+?\.)[^\s<>]+/],
+        [TA_ABBREVIATION,
+          /^(?:(?:(?:#{CHAR})+\.)+)(?:#{CHAR})+/],
+        [TA_WORD,
+          /^#{ALNUM}(?:-*#{ALNUM})*/],
+        [TA_PUNCTUATION,
+          /^[!,.:;?¡¿]+/]
       ]
 
       OTHER = [
-        ['OTHR', /^["$#%&'()*+\/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷„“–]/],
-        ['HELP', /^\S+/]
+        [TA_OTHER,
+          /^[-"$#%&'()*+\/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷„“–]/],
+        [TA_HELP,
+          /^\S+/]
       ]
 
       NESTS = {
-        'HTML'          => ['<',   '>'],
-        'WIKI:VARIABLE' => ['{{{', '}}}'],
-        'WIKI:TEMPLATE' => ['{{',  '}}'],
-        'WIKI:LINK_INT' => ['[[',  ']]'],
-        'WIKI:LINK_EXT' => [/^\[\s*#{PROTO}/, ']']
+        TA_HTML               => ['<',   '>'],
+        TA_WIKI + ':VARIABLE' => ['{{{', '}}}'],
+        TA_WIKI + ':TEMPLATE' => ['{{',  '}}'],
+        TA_WIKI + ':LINK_INT' => ['[[',  ']]'],
+        TA_WIKI + ':LINK_EXT' => [/^\[\s*#{PROTO}/, ']']
       }
 
       class << self
@@ -162,12 +173,12 @@ class Lingo
         @tags  = get_key('tags',  false)
         @wiki  = get_key('wiki',  false)
 
-        @skip_tags = get_array('skip-tags', '', :downcase)
+        @skip_tags = get_ary('skip-tags', '', :downcase)
         @tags = true unless @skip_tags.empty?
 
         skip = []
-        skip << 'HTML' unless @tags
-        skip << 'WIKI' unless @wiki
+        skip << TA_HTML unless @tags
+        skip << TA_WIKI unless @wiki
 
         [@rules = RULES.dup, @nests = NESTS.dup].each { |hash|
           hash.delete_if { |name, _| skip.include?(Token.clean(name)) }
@@ -230,7 +241,7 @@ class Lingo
           next unless line =~ expr
 
           rest = $'
-          forward_token($&, name, rest) if name != 'SPAC' || @space
+          forward_token($&, name, rest) if name != TA_SPACE || @space
 
           yield rest
         }
@@ -289,12 +300,12 @@ class Lingo
       end
 
       def forward_token(form, attr, rest = '')
-        forward(Token.new(form, @override.empty? ? attr : 'SKIP',
+        forward(Token.new(form, @override.empty? ? attr : TA_SKIP,
           @position += 1, @offset - form.bytesize - rest.bytesize))
       end
 
       def overriding?(nest)
-        nest == 'HTML' && !@skip_tags.empty?
+        nest == TA_HTML && !@skip_tags.empty?
       end
 
     end

data/lib/lingo/attendee/variator.rb

@@ -39,7 +39,7 @@ class Lingo
     #
     # === Mögliche Verlinkung
     # Erwartet:: Daten vom Typ *Word* (andere werden einfach durchgereicht) z.B. von Wordsearcher
-    # Erzeugt:: Daten vom Typ *Word* zur Weiterleitung z.B. an Synonymer, Decomposer, Multiworder, Sequencer, Noneword_filter oder Vector_filter
+    # Erzeugt:: Daten vom Typ *Word* zur Weiterleitung z.B. an Synonymer, Decomposer, Multiworder, Sequencer oder Vector_filter
     #
     # === Parameter
     # Kursiv dargestellte Parameter sind optional (ggf. mit Angabe der Voreinstellung).
@@ -76,14 +76,14 @@ class Lingo
 
       def init
         @marker = get_key('marker', '*')
-        @max    = get_key('max-var', max = 10000).to_i
+        @max    = get_int('max-var', max = 10000)
         @max    = max unless @max > 0
         @var    = get_key('variations')
 
         raise MissingConfigError.new(:variations) if @var.empty?
 
         @check = Hash.new(false)
-        get_array('check', WA_UNKNOWN).each { |s| @check[s.upcase] = true }
+        get_ary('check', WA_UNKNOWN).each { |s| @check[s.upcase] = true }
 
         set_dic
         set_gra

data/lib/lingo/attendee/vector_filter.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -90,15 +90,13 @@ class Lingo
 
       DEFAULT_GENDER_SEPARATOR = Database::Source::WordClass::GENDER_SEPARATOR
 
-      TERMINALS = [:FILE, :RECORD, :EOF]
-
       def init
         @lex  = get_re('lexicals', '[sy]')
-        @skip = get_array('skip', DEFAULT_SKIP, :upcase)
+        @skip = get_ary('skip', DEFAULT_SKIP, :upcase)
 
         @src = @pos = @sort_fmt = @sort_rel = @docnum = nil
 
-        @tokens, @vectors, @word_count = [], Hash.nest(1) { [] }, Hash.new(0)
+        @tokens, @vectors, @word_count = [], Hash.array(1), Hash.new(0)
 
         if @dict = get_key('dict', false)
           @norm = get_key('norm', false)
@@ -110,7 +108,7 @@ class Lingo
           @pos = get_key('pos', false)
           @pos = DEFAULT_POS_SEPARATOR if @pos == true
 
-          @tokens = get_array('tokens', '', :upcase)
+          @tokens = get_ary('tokens', '', :upcase)
           @tokens.concat(Tokenizer.rules) if @tokens.delete('ALL')
         end
 
@@ -157,7 +155,7 @@ class Lingo
           pos = obj.position_and_offset if @pos
 
           obj.is_a?(Token) ? forward_vector(obj, pos) :
-            obj.get_class(@lex).each { |lex| forward_vector(lex, pos, lex.src) }
+            obj.each_lex(@lex) { |lex| forward_vector(lex, pos, lex.src) }
         end
       end
 
@@ -172,14 +170,14 @@ class Lingo
       end
 
       def forward_dict(obj, sep = DEFAULT_GENDER_SEPARATOR)
-        vectors = obj.get_class(@lex).map { |lex|
+        vectors = obj.each_lex(@lex).map { |lex|
           "#{lex.form} ##{lex.attr}".tap { |str|
             str << sep << lex.gender if lex.gender
           }
         }
 
         unless vectors.empty?
-          vec = @norm ? obj.lexicals.first.form : obj.form
+          vec = @norm ? obj.lex_form : obj.form
           forward_vector("#{vec}#{@dict}#{vectors.join(' ')}")
         end
       end

data/lib/lingo/attendee/word_searcher.rb

@@ -36,7 +36,7 @@ class Lingo
     #
     # === Mögliche Verlinkung
     # Erwartet:: Daten vom Typ *Token* (andere werden einfach durchgereicht) z.B. von Tokenizer, Abbreviator
-    # Erzeugt:: Daten vom Typ *Word* für erkannte Wörter z.B. für Synonymer, Decomposer, Ocr_variator, Multiworder, Sequencer, Noneword_filter, Vector_filter
+    # Erzeugt:: Daten vom Typ *Word* für erkannte Wörter z.B. für Synonymer, Decomposer, Ocr_variator, Multiworder, Sequencer, Vector_filter
     #
     # === Parameter
     # Kursiv dargestellte Parameter sind optional (ggf. mit Angabe der Voreinstellung).
@@ -77,8 +77,8 @@ class Lingo
       end
 
       def process(obj)
-        forward(obj.is_a?(Token) && obj.word? ?
-          @dic.find_word(obj.form, obj) : obj)
+        forward(obj)
+        forward(@dic.find_word(obj.form, obj)) if obj.word_token?
       end
 
     end

data/lib/lingo/buffered_attendee.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -34,53 +34,20 @@ class Lingo
     end
 
     def process(obj)
-      @buffer << obj
+      obj.word_token? ? forward(obj) : @buffer << obj
       process_buffer if process_buffer?
     end
 
     private
 
-    def form_at(index, klass = WordForm)
-      obj = @buffer[index]
-      obj.form if obj.is_a?(klass)
-    end
-
-    def forward_number_of_token(len = default = @buffer.size, punct = !default)
-      begin
-        unless @buffer.empty?
-          forward(item = @buffer.delete_at(0))
-          len -= 1 unless punct && item.form == CHAR_PUNCT
-        end
-      end while len > 0
-    end
-
-    def valid_tokens_in_buffer
-      @buffer.count { |item| item.form != CHAR_PUNCT }
-    end
-
     def process_buffer?
-      !instance_variable_defined?(:@expected_tokens_in_buffer) ||
-      valid_tokens_in_buffer >= @expected_tokens_in_buffer
+      raise NotImplementedError
     end
 
     def process_buffer
       raise NotImplementedError
     end
 
-    def control_multi(cmd)
-      if [:RECORD, :EOF].include?(cmd)
-        @eof_handling = true
-
-        while valid_tokens_in_buffer > 1
-          process_buffer
-        end
-
-        forward_number_of_token
-
-        @eof_handling = false
-      end
-    end
-
   end
 
 end

data/lib/lingo/config.rb

@@ -61,7 +61,7 @@ class Lingo
     attr_reader :language_file, :config_file
 
     def to_h
-      @opts
+      { 'version' => VERSION }.merge(@opts)
     end
 
     def [](key)

data/lib/lingo/ctl.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -25,9 +25,6 @@
 #++
 
 require 'optparse'
-require 'zip'
-
-Zip.unicode_names = true
 
 class Lingo
 
@@ -35,7 +32,7 @@ class Lingo
 
     extend self
 
-    PROG, VERSION, OPTWIDTH = $0, '0.0.2', 18
+    PROG, VERSION, OPTWIDTH = $0, '0.0.3', 21
     PROGNAME, OPTIONS = File.basename(PROG), {}
 
     COMMANDS, ALIASES = {}, Hash.nest { |k|
@@ -67,151 +64,8 @@ Usage: #{PROG} <command> [arguments] [options]
       COMMANDS[name], ALIASES[short] = desc, name
     end
 
-    { config: %w[c configuration],
-      lang:   %w[l language],
-      dict:   %w[d dictionary dictionaries],
-      store:  %w[s store],
-      sample: %w[e sample\ text\ file]
-    }.each { |n, (s, q, r)|
-      t = n == :store
-
-      cmd([:list,  :l, n], s, "List available #{r || "#{q}s"}", '[name...]') if !t
-      cmd([:find,  :f, n], s, "Find #{q} in Lingo search path",      'name')
-      cmd([:copy,  :c, n], s, "Copy #{q} to local Lingo directory",  'name') if !t
-      cmd([:clear, :c, n], s, 'Remove store files to force rebuild', 'name') if  t
-    }
-
-    { demo:    [:d, 'Initialize demo directory', '[path]', 'current directory'],
-      archive: [:a, 'Create archive of directory', '[path]', 'current directory'],
-      rackup:  [:r, 'Print path to rackup file', 'name'],
-      path:    [:p, 'Print search path for dictionaries and configurations'],
-      help:    [:h, 'Print help for available commands'],
-      version: [:v, 'Print Lingo version number']
-    }.each { |n, (s, *a)| cmd(n.to_s, s.to_s, *a) }
-
     private
 
-    def list(what, doit = true)
-      names = Regexp.union(*ARGV.empty? ? '' : ARGV)
-
-      Lingo.list(what, path: path_for_scope).select { |file|
-        File.basename(file) =~ names ? doit ? puts(file) : true : false
-      }
-    end
-
-    def find(what, doit = true)
-      name = ARGV.shift or missing_arg(:name)
-      no_args
-
-      file = Lingo.find(what, name, path: path_for_scope) { usage }
-      doit ? puts(file) : file
-    end
-
-    def copy(what)
-      usage('Source and target are the same.') if OPTIONS[:scope] == :local
-
-      source = find(what, false)
-      target = File.join(path_for_scope(:local), Lingo.basepath(what, source))
-
-      usage('Source and target are the same.') if source == target
-
-      return unless overwrite?(target)
-
-      FileUtils.mkdir_p(File.dirname(target))
-      FileUtils.cp(source, target, verbose: true)
-    end
-
-    def do_archive
-      OPTIONS.update(path: ARGV.shift, scope: :local)
-      no_args
-
-      source = File.expand_path(path_for_scope.first)
-      target = "#{source}.zip"
-
-      abort "No such directory: #{source}" unless Dir.exist?(source)
-
-      return unless overwrite?(target, true)
-
-      base, name = File.split(source)
-
-      Dir.chdir(base) {
-        Zip::File.open(target, Zip::File::CREATE) { |zipfile|
-          Dir[File.join(name, '**', '*')].each { |file|
-            zipfile.add(file, file)
-          }
-        }
-      }
-
-      puts "Directory successfully archived at `#{target}'."
-    end
-
-    def do_clearstore
-      store = Dir["#{find(:store, false)}.*"]
-      FileUtils.rm(store, verbose: true) unless store.empty?
-    end
-
-    def do_demo
-      OPTIONS.update(path: ARGV.shift, scope: :system)
-      no_args
-
-      path = path_for_scope(:local).first
-
-      copy_list(:config) { |i| !File.basename(i).start_with?('test') }
-      copy_list(:lang)
-      copy_list(:dict)   { |i|  File.basename(i).start_with?('user') }
-      copy_list(:sample)
-
-      puts "Demo directory successfully initialized at `#{path}'."
-    end
-
-    def do_rackup(doit = true)
-      name = ARGV.shift or missing_arg(:name)
-      no_args
-
-      require 'lingo/app'
-
-      if file = Lingo::App.rackup(name)
-        doit ? puts(file) : file
-      else
-        usage("Invalid app name `#{name.inspect}'.")
-      end
-    end
-
-    def do_path
-      no_args
-      puts path_for_scope || PATH
-    end
-
-    def do_help(opts = nil)
-      no_args
-
-      msg = opts ? [opts, 'Commands:'] : []
-
-      aliases = Hash.nest { [] }
-      ALIASES.each { |k, v| aliases[v] << k }
-
-      COMMANDS.each { |c, (d, *e)|
-        a = aliases[c]
-        c = "#{c} (#{a.join(', ')})" unless a.empty?
-
-        if opts
-          msg << "    %-#{OPTWIDTH}s %s" % [c, d]
-        else
-          msg << "#{c}" << "  - #{d}"
-          e.each { |i| msg <<  "  + #{i}" }
-        end
-      }
-
-      abort msg.join("\n")
-    end
-
-    def do_version(doit = true)
-      no_args
-
-      msg = "Lingo v#{Lingo::VERSION}"
-      doit ? puts(msg) : msg
-    end
-
     def parse_options
       OptionParser.new(USAGE, OPTWIDTH) { |opts|
         opts.separator ''
@@ -266,12 +120,6 @@ Usage: #{PROG} <command> [arguments] [options]
       usage('Too many arguments.') unless ARGV.empty?
     end
 
-    def copy_list(what)
-      files = list(what, false)
-      files.select! { |i| yield i } if block_given?
-      files.each { |file| ARGV.replace([file]); copy(what) }
-    end
-
     def overwrite?(target, unlink = false)
       !File.exist?(target) || if agree?("#{target} already exists. Overwrite?")
         File.unlink(target) if unlink
@@ -280,7 +128,7 @@ Usage: #{PROG} <command> [arguments] [options]
     end
 
     def agree?(msg)
-      print "#{msg} (y/n) [n] "
+      print "#{msg} (y/n) [n]: "
 
       case answer = $stdin.gets.chomp
         when /\Ano?\z/i, ''  then nil
@@ -301,3 +149,7 @@ Usage: #{PROG} <command> [arguments] [options]
   end
 
 end
+
+require_relative 'ctl/files'
+require_relative 'ctl/analysis'
+require_relative 'ctl/other'