lingo 1.9.0.pre1 → 1.9.0

data/lib/lingo/cli.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -64,7 +64,8 @@ class Lingo
       opts.separator
 
       opts.option(:log__FILE, :L, 'Log file to print debug information to') { |log|
-        options[:log] = stderr.reopen(log == '-' ? stdout : File.open(log, 'a+', encoding: ENC))
+        options[:log] = stderr.reopen(
+          log == '-' ? stdout : File.open(log, 'a+', encoding: ENCODING))
       }
 
       opts.separator
@@ -76,4 +77,21 @@ class Lingo
 
   end
 
+  def self.CLI(args, extra = nil, &block)
+    opt, req = args.partition { |arg| arg.sub!(/\?\z/, '') }
+
+    unless (n = ARGV.size - req.size) >= 0 && n <= opt.size
+      msg = "Usage: #{$0}#{args.map { |arg| [' ', arg].zip(
+        opt.include?(arg) ? %w[[ ]] : %w[< >]).join }.join}"
+
+      abort Array(extra).unshift(msg).join("\n\n")
+    end
+
+    Object.new.extend(TextUtils).instance_eval(&block)
+  rescue LingoError => err
+    abort err.to_s
+  ensure
+    ObjectSpace.each_object(Zlib::GzipWriter, &:close)
+  end
+
 end

data/lib/lingo/config.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -53,7 +53,7 @@ class Lingo
       end
     end
 
-    attr_reader :language_file, :config_file
+    attr_reader :lang_file, :config_file
 
     def to_h
       { 'version' => VERSION }.merge(@opts)
@@ -128,7 +128,8 @@ class Lingo
       file = Lingo.find(type, @opts[key]) { quit }
       instance_variable_set("@#{type}_file", file)
 
-      File.open(file, encoding: ENC) { |f| @opts.update(SafeYAML.load(f)) }
+      File.open(file, encoding: ENCODING) { |f|
+        @opts.update(SafeYAML.load(f)) }
     rescue Psych::SyntaxError => err
       err.message << " (in #{file})"
       raise

data/lib/lingo/ctl.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -31,6 +31,7 @@ class Lingo
   module Ctl
 
     extend self
+    extend TextUtils
 
     PROG, VERSION, OPTWIDTH = $0, '0.0.3', 21
     PROGNAME, OPTIONS = File.basename(PROG), {}
@@ -120,25 +121,6 @@ Usage: #{PROG} <command> [arguments] [options]
       usage('Too many arguments.') unless ARGV.empty?
     end
 
-    def overwrite?(target, unlink = false)
-      !File.exist?(target) || if agree?("#{target} already exists. Overwrite?")
-        File.unlink(target) if unlink
-        true
-      end
-    end
-
-    def agree?(msg)
-      print "#{msg} (y/n) [n]: "
-
-      case answer = $stdin.gets.chomp
-        when /\Ano?\z/i, ''  then nil
-        when /\Ay(?:es)?\z/i then true
-        else puts 'Please enter "yes" or "no".'; agree?(msg)
-      end
-    rescue Interrupt
-      abort ''
-    end
-
   end
 
   def self.ctl

data/lib/lingo/ctl/analysis.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -24,8 +24,6 @@
 ###############################################################################
 #++
 
-require 'csv'
-
 class Lingo
 
   module Ctl
@@ -97,11 +95,11 @@ class Lingo
         File.basename(path.chomp(File.extname(path))) }.uniq.join('-'))
 
       lambda { |key, &block| overwrite?(file = "#{name}.#{key}.csv") &&
-        puts("#{file}: #{Array(CSV.open(file, 'wb', &block)).join(' / ')}") }
+        puts("#{file}: #{Array(open_csv(file, 'wb', &block)).join(' / ')}") }
     end
 
     def csv_foreach(paths)
-      paths.each { |path| CSV.foreach(path, headers: true) { |row|
+      paths.each { |path| foreach_csv(path, headers: true) { |row|
         yield path, *row.values_at(*%w[string token word pattern]) } }
     end
 

data/lib/lingo/ctl/files.rb

@@ -76,9 +76,9 @@ class Lingo
       FileUtils.cp(source, target, verbose: true)
     end
 
-    def do_clearstore
-      store = Dir["#{find(:store, false)}.*"]
-      FileUtils.rm(store, verbose: true) unless store.empty?
+    def clear(what)
+      target = Dir["#{find(what, false)}.*"]
+      FileUtils.rm(target, verbose: true) unless target.empty?
     end
 
   end

data/lib/lingo/database.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -45,6 +45,8 @@ class Lingo
     KEY_REF = '*'
     SYS_KEY = '~'
 
+    KEY_REF_RE = %r{\A#{Regexp.escape(KEY_REF)}(\d+)\z}o
+
     BACKENDS       = []
     BACKEND_BY_EXT = {}
 
@@ -58,6 +60,25 @@ class Lingo
         klass.class_eval('def store_ext; EXT; end', __FILE__, __LINE__) if meth
       end
 
+      def backend_by_ext(file, ext = File.extname(file))
+        get_backend(BACKEND_BY_EXT[ext], file) or
+          raise BackendNotFoundError.new(file)
+      end
+
+      def find_backend(env = 'LINGO_BACKEND')
+        env && get_backend(ENV[env]) || BACKENDS.find { |name|
+          backend = get_backend(name, nil, true) and return backend }
+      end
+
+      def get_backend(name, file = nil, relax = false)
+        return unless name
+
+        Object.const_get(name)
+        const_get("#{name}Store")
+      rescue TypeError, NameError => err
+        raise BackendNotAvailableError.new(name, file, err) unless relax
+      end
+
       def open(*args, &block)
         new(*args).open(&block)
       end
@@ -75,23 +96,12 @@ class Lingo
         FileUtils.mkdir_p(File.dirname(@stofile))
       rescue SourceFileNotFoundError => err
         @stofile = skip_ext = err.id
-
-        unless err.name
-          if name = BACKEND_BY_EXT[File.extname(@stofile)]
-            backend = get_backend(name, @stofile)
-          else
-            raise BackendNotFoundError.new(@stofile)
-          end
-        end
+        backend = self.class.backend_by_ext(@stofile) unless err.name
       rescue NoWritableStoreError
         backend = HashStore
       end
 
-      unless backend ||= get_backend(ENV['LINGO_BACKEND'])
-        BACKENDS.find { |name| backend = get_backend(name, nil, true) }
-      end
-
-      extend(@backend = backend || HashStore)
+      extend(@backend = backend || self.class.find_backend || HashStore)
 
       @stofile << store_ext unless skip_ext || !respond_to?(:store_ext)
 
@@ -151,15 +161,6 @@ class Lingo
 
     private
 
-    def get_backend(name, file = nil, relax = false)
-      return unless name
-
-      Object.const_get(name)
-      self.class.const_get("#{name}Store")
-    rescue TypeError, NameError => err
-      raise BackendNotAvailableError.new(name, file, err) unless relax
-    end
-
     def config_hash
       hashes = [config]
 
@@ -228,11 +229,11 @@ class Lingo
     end
 
     def _encode!(str)
-      str.force_encoding(ENC)
+      str.force_encoding(ENCODING)
     end
 
     def convert(verbose = lingo.config.stderr.tty?)
-      src = Source.get(config.fetch('txt-format', 'key_value'), @id, lingo)
+      src = Source.from_config(config, @id)
 
       sep, hyphenate, key_map, val_map = prepare_lex
 

data/lib/lingo/database/crypter.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -35,23 +35,27 @@ class Lingo
 
       extend self
 
+      KEYLEN = 16
+
+      CIPHER = 'AES-128-CBC'.freeze
+
       def digest(key)
         Digest::SHA1.hexdigest(key)
       end
 
       def encode(key, val)
-        [digest(key), crypt(:encrypt, key, val)]
+        [digest = digest(key), crypt(:encrypt, key, val, digest)]
       end
 
       def decode(key, val)
-        crypt(:decrypt, key, val).force_encoding(ENC)
+        crypt(:decrypt, key, val, digest(key)).force_encoding(ENCODING)
       end
 
       private
 
-      def crypt(method, key, val)
-        cipher = OpenSSL::Cipher.new('aes-128-cbc').send(method)
-        cipher.iv = cipher.key = digest(key)
+      def crypt(method, key, val, digest)
+        cipher = OpenSSL::Cipher.new(CIPHER).send(method)
+        cipher.iv = cipher.key = digest[0, KEYLEN]
         cipher.update(val) + cipher.final
       end
 

data/lib/lingo/database/source.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -49,30 +49,57 @@ class Lingo
 
     class Source
 
-      def self.get(name, *args)
-        Lingo.get_const(name, self).new(*args)
+      LEXICAL_SEPARATOR = '#'.freeze
+
+      DEFAULT_SEPARATOR = nil
+
+      DEFAULT_DEF_WC = nil
+
+      MAX_LENGTH = 4096
+
+      class << self
+
+        def from_id(id, lingo)
+          from_config(lingo.database_config(id), id)
+        end
+
+        def from_config(config, id = nil)
+          format = config.fetch('txt-format', 'key_value')
+          Lingo.get_const(format, self).new(config['name'], config, id)
+        end
+
+        def lexicals(val, sep = LEXICAL_SEPARATOR, ref = KEY_REF_RE)
+          val.map { |str|
+            str =~ ref ? $1.to_i : begin
+              k, *w = str.split(sep)
+              Language::Lexical.new(k.strip, w)
+            end
+          }.uniq if val
+        end
+
       end
 
       attr_reader :pos
 
-      def initialize(id, lingo, def_wc_default = nil)
-        @config = lingo.database_config(id)
+      def initialize(name = nil, config = {}, id = nil)
+        @config = config
 
-        source_file = Lingo.find(:dict, name = @config['name'], relax: true)
+        src_file = Lingo.find(:dict, name, relax: true) if name
 
-        reject_file = begin
-          Lingo.find(:store, source_file) << '.rev'
+        rej_file = begin
+          Lingo.find(:store, src_file) << '.rev'
         rescue NoWritableStoreError, SourceFileNotFoundError
-        end
+        end if id && src_file
 
-        @src = Pathname.new(source_file)
-        @rej = Pathname.new(reject_file) if reject_file
+        @src = Pathname.new(src_file) if src_file
+        @rej = Pathname.new(rej_file) if rej_file
 
-        raise SourceFileNotFoundError.new(name, id) unless @src.exist?
+        raise id ? SourceFileNotFoundError.new(name, id) :
+          FileNotFoundError.new(name) if name && !@src.exist?
 
-        @def = @config.fetch('def-wc', def_wc_default)
+        @sep = config.fetch('separator', self.class::DEFAULT_SEPARATOR)
+        @def = config.fetch('def-wc', self.class::DEFAULT_DEF_WC)
         @def = @def.downcase if @def
-        @sep = @config['separator']
 
         @wrd = "(?:#{Language::Char::ANY})+"
         @pat = /^#{@wrd}$/
@@ -85,33 +112,47 @@ class Lingo
       end
 
       def each
-        reject_file = @rej.open('w', encoding: ENC) if @rej
+        return enum_for(__method__) unless block_given?
+        each_line { |line, key, val| yield parse_line(line, key, val) }
+      end
+
+      def each_line
+        return enum_for(__method__) unless block_given?
+
+        rej_file = @rej.open('w', encoding: ENCODING) if @rej
 
-        @src.each_line($/, encoding: ENC) { |line|
+        @src.each_line($/, encoding: ENCODING) { |line|
           @pos += length = line.bytesize
 
           line.strip!
           next if line.empty? || line.start_with?('#')
 
-          line.chomp!
-          line.replace(Unicode.downcase(line))
-
-          if length < 4096 && line =~ @pat
-            yield convert_line(line, $1, $2)
+          if length < MAX_LENGTH && line.replace(Unicode.downcase(line)) =~ @pat
+            yield line, $1, $2
           else
             @rej_cnt += 1
-            reject_file.puts(line) if reject_file
+            rej_file.puts(line) if rej_file
           end
         }
 
         self
       ensure
-        if reject_file
-          reject_file.close
-          @rej.delete if @rej.size == 0
+        if rej_file
+          rej_file.close
+          @rej.delete if @rej.size.zero?
         end
       end
 
+      def each_lexical
+        return enum_for(__method__) unless block_given?
+        each { |key, val| yield key, self.class.lexicals(val) }
+      end
+
+      def each_dump(*args)
+        return enum_for(__method__, *args) unless block_given?
+        each_lexical { |key, val| yield dump_line(key, val, *args) }
+      end
+
       def set(db, key, val)
         db[key] = val
       end
@@ -120,6 +161,12 @@ class Lingo
         [@rej_cnt, @rej]
       end
 
+      private
+
+      def lexical(*args)
+        args.join(LEXICAL_SEPARATOR)
+      end
+
     end
 
   end

data/lib/lingo/database/source/key_value.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2016 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -39,17 +39,21 @@ class Lingo
 
       class KeyValue < self
 
-        DEFAULT_SEPARATOR = '*'
+        DEFAULT_SEPARATOR = '*'.freeze
 
-        def initialize(id, lingo)
-          super(id, lingo, Language::LA_UNKNOWN)
-          @pat = /^(#{@wrd})#{Regexp.escape(@sep ||= DEFAULT_SEPARATOR)}(#{@wrd})$/
+        DEFAULT_DEF_WC = Language::LA_UNKNOWN
+
+        def initialize(*)
+          super
+          @pat = /^(#{@wrd})#{Regexp.escape(@sep)}(#{@wrd})$/
         end
 
-        private
+        def parse_line(line, key, val)
+          [key.strip, [lexical(val.strip, @def)]]
+        end
 
-        def convert_line(line, key, val)
-          [key.strip, %W[#{val.strip}##{@def}]]
+        def dump_line(key, val, sep = @sep, *)
+          val.map(&:form).unshift(key).join(sep)
         end
 
       end