lingo 1.8.4.2 → 1.8.5

data/lib/lingo/attendee/tokenizer.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -82,33 +82,47 @@ class Lingo
 
       CHAR, DIGIT = Char::CHAR, Char::DIGIT
 
+      PROTO = '(?:news|https?|ftps?)://'
+
       RULES = [
-        ['WIKI', /^=+.+=+$/],
         ['SPAC', /^\s+/],
-        ['HTML', /^<[^>]+>/],
-        ['WIKI', /^\[\[.+?\]\]/],
+        ['WIKI', /^=+.+=+|^__[A-Z]+__/],
         ['NUMS', /^[+-]?(?:\d{4,}|\d{1,3}(?:\.\d{3,3})*)(?:\.|(?:,\d+)?%?)/],
-        ['URLS', /^(?:(?:mailto:|(?:news|https?|ftps?):\/\/)\S+|^(?:www(?:\.\S+)+)|[^\s.]+(?:[\._]\S+)+@\S+(?:\.\S+)+)/],
+        ['URLS', /^(?:www\.|mailto:|#{PROTO}|\S+?[._]\S+?@\S+?\.)\S+/],
         ['ABRV', /^(?:(?:(?:#{CHAR})+\.)+)(?:#{CHAR})+/],
         ['WORD', /^(?:#{CHAR}|#{DIGIT}|-)+/],
-        ['PUNC', /^[!,.:;?¡¿]/],
-        ['OTHR', /^["$#%&'()*+\-\/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷]/],
-        ['HELP', /^[^ ]*/]
+        ['PUNC', /^[!,.:;?¡¿]+/]
+      ]
+
+      OTHER = [
+        ['OTHR', /^["$#%&'()*+\/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷]/],
+        ['HELP', /^\S+/]
       ]
 
+      NESTS = {
+        'HTML'          => ['<',   '>'],
+        'WIKI:VARIABLE' => ['{{{', '}}}'],
+        'WIKI:TEMPLATE' => ['{{',  '}}'],
+        'WIKI:LINK_INT' => ['[[',  ']]'],
+        'WIKI:LINK_EXT' => [/^\[\s*#{PROTO}/, ']']
+      }
+
       class << self
 
         def rule(name)
           RULES.assoc(name)
         end
 
+        def rules(name)
+          RULES.select { |rule,| rule == name }
+        end
+
         def delete(*names)
-          names.each { |name| RULES.delete(rule(name)) }
+          names.map { |name| rules(name).each { |rule| RULES.delete(rule) } }
         end
 
-        def replace(name, expr)
-          rule = rule(name) or return
-          rule[1] = block_given? ? yield(rule[1]) : expr
+        def replace(name, expr = nil)
+          rules(name).each { |rule| rule[1] = expr || yield(*rule) }
         end
 
         def insert(*rules)
@@ -152,22 +166,37 @@ class Lingo
         skip << 'HTML' unless @tags
         skip << 'WIKI' unless @wiki
 
-        @rules = RULES.reject { |name, _| skip.include?(name) }
+        [@rules = RULES.dup, @nests = NESTS.dup].each { |hash|
+          hash.delete_if { |name, _| skip.include?(Token.clean(name)) }
+        }
+
+        @nest, nest_re = [], []
 
-        @filename = @cont = nil
+        @nests.each { |name, re|
+          re.map!.with_index { |r, i| r.is_a?(Regexp) ?
+            r : /^#{'.*?' if i > 0}#{Regexp.escape(r)}/ }
+
+          nest_re << "(?<#{name}>#{Regexp.new(
+            re[0].source.sub(/^\^/, ''), re[0].options)})"
+        }
+
+        @nest_re = /^(?<_>.*?)(?:#{nest_re.join('|')})/
+
+        @filename = @linenum = nil
       end
 
       def control(cmd, param)
         case cmd
-          when STR_CMD_FILE then @filename = param
-          when STR_CMD_LIR  then @filename = nil
-          when STR_CMD_EOF  then @cont     = nil
+          when STR_CMD_FILE then @filename, @linenum = param, 1
+          when STR_CMD_LIR  then @filename, @linenum = nil, nil
+          when STR_CMD_EOL  then @linenum += 1 if @linenum
+          when STR_CMD_EOF  then @nest.clear
         end
       end
 
       def process(obj)
         if obj.is_a?(String)
-          tokenize(obj) { |*i| forward(Token.new(*i)) }
+          tokenize(obj)
           forward(STR_CMD_EOL, @filename) if @filename
         else
           forward(obj)
@@ -178,44 +207,66 @@ class Lingo
 
       # tokenize("Eine Zeile.")  ->  [:Eine/WORD:, :Zeile/WORD:, :./PUNC:]
       def tokenize(line)
-        case @cont
-          when 'HTML'
-            if line =~ /^[^<>]*>/
-              yield $&, @cont
-              line, @cont = $', nil
-            else
-              yield line, @cont
-              return
-            end
-          when 'WIKI'
-            if line =~ /^[^\[\]]*\]\]/
-              yield $&, @cont
-              line, @cont = $', nil
-            else
-              yield line, @cont
-              return
-            end
-          when nil
-            if @tags && line =~ /<[^<>]*$/
-              yield $&, @cont = 'HTML'
-              line = $`
-            end
-
-            if @wiki && line =~ /\[\[[^\[\]]*$/
-              yield $&, @cont = 'WIKI'
-              line = $`
-            end
+        @nest.empty? ? tokenize_line(line) : tokenize_nest(line)
+      rescue => err
+        raise err if err.is_a?(TokenizeError)
+        raise TokenizeError.new(line, @filename, @linenum, err)
+      end
+
+      def tokenize_line(line)
+        while (length = line.length) > 0 && tokenize_rule(line) { |rest|
+          length == rest.length ? break : line = rest
+        }
         end
 
-        while (l = line.length) > 0 && @rules.find { |name, expr|
-          if line =~ expr
-            yield $&, name if name != 'SPAC' || @space
-            l == $'.length ? break : line = $'
-          end
+        tokenize_open(line) unless line.empty?
+      end
+
+      def tokenize_rule(line, rules = @rules)
+        rules.find { |name, expr|
+          next unless line =~ expr
+          forward_token($&, name) if name != 'SPAC' || @space
+          yield $'
         }
+      end
+
+      def tokenize_nest(line)
+        mdo = @nest_re.match(line)
+        mdc = @nests[@nest.last].last.match(line)
+
+        if mdo && (!mdc || mdo[0].length < mdc[0].length)
+          forward_token(mdo[:_], @nest.last) unless mdo[:_].empty?
+
+          nest = @nests.keys.find { |name| mdo[name] }
+          forward_nest(mdo[nest], mdo.post_match, nest)
+        elsif mdc
+          forward_token(mdc[0], @nest.pop)
+          tokenize(mdc.post_match)
+        else
+          forward_token(line, @nest.last)
         end
       end
 
+      def tokenize_open(line)
+        @nests.each { |nest, (open_re, _)|
+          next unless line =~ open_re
+          return forward_nest($&, $', nest)
+        }
+
+        tokenize_rule(line, OTHER) { |rest| line = rest }
+        tokenize(line)
+      end
+
+      def forward_nest(match, rest, nest)
+        forward_token(match, nest)
+        @nest << nest
+        tokenize(rest)
+      end
+
+      def forward_token(*args)
+        forward(Token.new(*args))
+      end
+
     end
 
   end

data/lib/lingo/attendee/variator.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -90,6 +90,7 @@ class Lingo
       end
 
       def control(cmd, param)
+        # can control
       end
 
       def process(obj)

data/lib/lingo/attendee/vector_filter.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -86,10 +86,15 @@ class Lingo
       def init
         if @debug = get_key('debug', false)
           @prompt = get_key('prompt', 'lex:) ')
+          @preamble = get_key('preamble', true)
         else
           @lex  = get_re('lexicals', '[sy]')
           @skip = get_array('skip', DEFAULT_SKIP, :upcase)
 
+          @dict = get_key('dict', false)
+          @norm = get_key('norm', false) if @dict
+          @dict = Database::Source::WordClass::DEFAULT_SEPARATOR if @dict == true
+
           @src = get_key('src', false)
           @src = DEFAULT_SRC_SEP if @src == true
 
@@ -112,15 +117,32 @@ class Lingo
 
       def process(obj)
         if @debug
+          forward((@preamble = nil; @lingo.config.to_h.to_yaml)) if @preamble
           forward("#{@prompt} #{obj.inspect}") if eval(@debug)
         elsif obj.is_a?(Word) && !@skip.include?(obj.attr)
           @word_count += 1
 
-          obj.get_class(@lex).each { |lex|
-            vec = Unicode.downcase(lex.form)
-            vec << @src << lex.src if @src && lex.src
-            @sort_format ? @vectors << vec : forward(vec)
-          }
+          if @dict
+            vec, sep = [], Database::Source::WordClass::GENDER_SEPARATOR
+
+            obj.get_class(@lex).each { |lex|
+              str = "#{lex.form} ##{lex.attr}"
+              str << sep << lex.gender if lex.gender
+              vec << str
+            }
+
+            unless vec.empty?
+              wrd = @norm ? obj.lexicals.first.form : obj.form
+              vec = Unicode.downcase("#{wrd}#{@dict}#{vec.join(' ')}")
+              @sort_format ? @vectors << vec : forward(vec)
+            end
+          else
+            obj.get_class(@lex).each { |lex|
+              vec = Unicode.downcase(lex.form)
+              vec << @src << lex.src if @src && lex.src
+              @sort_format ? @vectors << vec : forward(vec)
+            }
+          end
         end
       end
 

data/lib/lingo/attendee/word_searcher.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -72,6 +72,7 @@ class Lingo
       end
 
       def control(cmd, param)
+        # can control
       end
 
       def process(obj)

data/lib/lingo/attendee.rb

@@ -83,6 +83,8 @@ class Lingo
       # Make sure config exists
       lingo.dictionary_config
 
+      @dic = @gra = nil
+
       init if self.class.method_defined?(:init)
 
       @can_control = self.class.method_defined?(:control)
@@ -91,6 +93,8 @@ class Lingo
       @skip_command = false
     end
 
+    attr_reader :lingo
+
     def add_subscriber(subscriber)
       @subscriber.concat(subscriber)
     end
@@ -133,7 +137,7 @@ class Lingo
     end
 
     def has_key?(key)
-      @config && @config.has_key?(key)
+      @config && @config.key?(key)
     end
 
     def get_key(key, default = nodefault = true)
@@ -154,11 +158,11 @@ class Lingo
     end
 
     def dictionary(src, mod)
-      Language::Dictionary.new({ 'source' => src, 'mode' => mod }, @lingo)
+      Language::Dictionary.new({ 'source' => src, 'mode' => mod }, lingo)
     end
 
     def grammar(src, mod)
-      Language::Grammar.new({ 'source' => src, 'mode' => mod }, @lingo)
+      Language::Grammar.new({ 'source' => src, 'mode' => mod }, lingo)
     end
 
     def set_dic
@@ -170,7 +174,7 @@ class Lingo
     end
 
     def warn(*msg)
-      @lingo.warn(*msg)
+      lingo.warn(*msg)
     end
 
     def require_lib(lib)

data/lib/lingo/call.rb

@@ -48,7 +48,7 @@ class Lingo
       end
     end
 
-    def talk(str)
+    def talk(str, raw = false)
       config.stdin.reopen(str)
 
       start
@@ -57,7 +57,7 @@ class Lingo
         io = config.send(key)
         io.rewind
 
-        lines = io.readlines.each { |i| i.chomp! }
+        lines = io.readlines
 
         io.truncate(0)
         io.rewind
@@ -65,8 +65,12 @@ class Lingo
         lines
       }
 
+      return res.join if raw
+
+      res.each { |i| i.chomp! }
+
       block_given? ? res.map! { |i| yield i } : begin
-        Lingo.sort!(res)
+        res.sort! unless ENV['LINGO_NO_SORT']
         res.uniq!
         res
       end

data/lib/lingo/cli.rb

@@ -24,11 +24,11 @@
 ###############################################################################
 #++
 
-require 'nuggets/cli'
+require 'cyclops'
 
 class Lingo
 
-  class CLI < ::Nuggets::CLI
+  class CLI < Cyclops
 
     class << self
 
@@ -55,26 +55,18 @@ class Lingo
     end
 
     def opts(opts)
-      opts.on('-c', '--config YAML', "Config file [Default: #{defaults[:config]}#{' (currently not present)' unless File.readable?(defaults[:config])}]") { |config|
-        options[:config] = config
-      }
-
-      opts.separator ''
-
-      opts.on('-l', '--language LANG', "Language for processing [Default: #{defaults[:language]}]") { |language|
-        options[:language] = language
-      }
+      opts.option(:language__LANG, "Language for processing [Default: #{defaults[:language]}]")
 
-      opts.separator ''
+      opts.separator
 
-      opts.on('-L', '--log FILE', 'Log file to print debug information to') { |log|
+      opts.option(:log__FILE, :L, 'Log file to print debug information to') { |log|
         options[:log] = stderr.reopen(log == '-' ? stdout : File.open(log, 'a+', encoding: ENC))
       }
 
-      opts.separator ''
+      opts.separator
 
-      opts.on('-P', '--profile PATH', 'Print profiling results') { |profile|
-        options[:profile] = profile == '-' ? stdout : profile
+      opts.option(:profile__PATH, :P, 'Print profiling results') { |profile|
+        options[:profile] = stdout if profile == '-'
       }
     end
 

data/lib/lingo/config.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -24,11 +24,9 @@
 ###############################################################################
 #++
 
-require 'yaml'
+require 'safe_yaml/load'
 require_relative 'cli'
 
-YAML::ENGINE.yamler = 'psych'
-
 class Lingo
 
   class Config
@@ -58,8 +56,12 @@ class Lingo
       end
     end
 
+    def to_h
+      @opts
+    end
+
     def [](key)
-      key_to_nodes(key).inject(@opts) { |hash, node| hash[node] }
+      key_to_nodes(key).inject(to_h) { |hash, node| hash[node] }
     end
 
     def []=(key, val)
@@ -125,7 +127,10 @@ class Lingo
 
     def load_config(key, type = key.to_sym)
       file = Lingo.find(type, @opts[key]) { quit }
-      @opts.update(File.open(file, encoding: ENC) { |f| YAML.load(f) })
+      File.open(file, encoding: ENC) { |f| @opts.update(SafeYAML.load(f)) }
+    rescue Psych::SyntaxError => err
+      err.message << " (in #{file})"
+      raise
     end
 
   end

data/lib/lingo/ctl.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -25,6 +25,9 @@
 #++
 
 require 'optparse'
+require 'zip'
+
+Zip.unicode_names = true
 
 class Lingo
 
@@ -36,7 +39,7 @@ class Lingo
     PROGNAME, OPTIONS = File.basename(PROG), {}
 
     COMMANDS, ALIASES = {}, Hash.new { |h, k|
-      h[k] = COMMANDS.has_key?(k) ? k : 'usage'
+      h[k] = COMMANDS.key?(k) ? k : 'usage'
     }
 
     USAGE = <<-EOT
@@ -79,6 +82,7 @@ Usage: #{PROG} <command> [arguments] [options]
     }
 
     { demo:    [:d, 'Initialize demo directory', '[path]', 'current directory'],
+      archive: [:a, 'Create archive of directory', '[path]', 'current directory'],
       rackup:  [:r, 'Print path to rackup file', 'name'],
       path:    [:p, 'Print search path for dictionaries and configurations'],
       help:    [:h, 'Print help for available commands'],
@@ -111,19 +115,47 @@ Usage: #{PROG} <command> [arguments] [options]
 
       usage('Source and target are the same.') if source == target
 
+      return unless overwrite?(target)
+
       FileUtils.mkdir_p(File.dirname(target))
       FileUtils.cp(source, target, verbose: true)
     end
 
+    def do_archive
+      OPTIONS.update(path: ARGV.shift, scope: :local)
+      no_args
+
+      source = File.expand_path(path_for_scope.first)
+      target = "#{source}.zip"
+
+      abort "No such directory: #{source}" unless Dir.exist?(source)
+
+      return unless overwrite?(target, true)
+
+      base, name = File.split(source)
+
+      Dir.chdir(base) {
+        Zip::File.open(target, Zip::File::CREATE) { |zipfile|
+          Dir[File.join(name, '**', '*')].each { |file|
+            zipfile.add(file, file)
+          }
+        }
+      }
+
+      puts "Directory successfully archived at `#{target}'."
+    end
+
     def do_clearstore
       store = Dir["#{find(:store, false)}.*"]
       FileUtils.rm(store, verbose: true) unless store.empty?
     end
 
     def do_demo
-      OPTIONS.update(path: path = ARGV.shift, scope: :system)
+      OPTIONS.update(path: ARGV.shift, scope: :system)
       no_args
 
+      path = path_for_scope(:local).first
+
       copy_list(:config) { |i| !File.basename(i).start_with?('test') }
       copy_list(:lang)
       copy_list(:dict)   { |i|  File.basename(i).start_with?('user') }
@@ -240,6 +272,25 @@ Usage: #{PROG} <command> [arguments] [options]
       files.each { |file| ARGV.replace([file]); copy(what) }
     end
 
+    def overwrite?(target, unlink = false)
+      !File.exist?(target) || if agree?("#{target} already exists. Overwrite?")
+        File.unlink(target) if unlink
+        true
+      end
+    end
+
+    def agree?(msg)
+      print "#{msg} (y/n) [n] "
+
+      case answer = $stdin.gets.chomp
+        when /\Ano?\z/i, ''  then nil
+        when /\Ay(?:es)?\z/i then true
+        else puts 'Please enter "yes" or "no".'; agree?(msg)
+      end
+    rescue Interrupt
+      abort ''
+    end
+
   end
 
   def self.ctl

data/lib/lingo/database/crypter.rb

@@ -24,6 +24,7 @@
 ###############################################################################
 #++
 
+require 'openssl'
 require 'digest/sha1'
 
 class Lingo
@@ -34,31 +35,24 @@ class Lingo
 
     class Crypter
 
-      HEX_CHARS = '0123456789abcdef'.freeze
-
-      def digest(key)
+      def self.digest(key)
         Digest::SHA1.hexdigest(key)
       end
 
       def encode(key, val)
-        [digest(key), crypt(key, val).each_byte.with_object('') { |b, s|
-          b.divmod(16).each { |i| s << HEX_CHARS[i] }
-        }]
+        [self.class.digest(key), crypt(:encrypt, key, val)]
       end
 
       def decode(key, val)
-        crypt(key, val.each_byte.each_slice(2).with_object('') { |b, s|
-          q, r = b.map { |i| HEX_CHARS.index(i.chr(ENC)) }
-          s << q * 16 + r
-        })
+        crypt(:decrypt, key, val).force_encoding(ENC)
       end
 
       private
 
-      def crypt(k, v)
-        c, y = '', k.codepoints.reverse_each.cycle
-        v.each_codepoint { |x| c << (x ^ y.next).chr(ENC) }
-        c
+      def crypt(method, key, val)
+        cipher = OpenSSL::Cipher.new('aes-128-cbc').send(method)
+        cipher.iv = cipher.key = self.class.digest(key)
+        cipher.update(val) + cipher.final
       end
 
     end

data/lib/lingo/database/hash_store.rb

@@ -63,7 +63,7 @@ class Lingo
       # Dup key, because we're reusing everything.
       def _each
        @db.each { |key, val| yield key.dup, val }
-     end
+      end
 
     end
 

data/lib/lingo/database/{show_progress.rb → progress.rb}

@@ -28,18 +28,17 @@ class Lingo
 
   class Database
 
-    class ShowProgress < ShowProgress
+    class Progress < Progress
 
       def initialize(obj, src, doit = true)
-        name = obj.instance_variable_get(:@config)['name']
-        super(obj, src.size, name, doit, 'convert', false)
+        super(obj, src.size, obj.config['name'], doit, 'convert', false)
 
-        if defined?(@cnt)
-          cnt, rej = src.rejected
+        if defined?(@count)
+          count, rejected = src.rejected
 
-          if cnt > 0
-            print ' (', cnt, ' rejected'
-            print ': ', rej if rej
+          if count > 0
+            print ' (', count, ' rejected'
+            print ': ', rejected if rejected
             print ')'
           end