lingo 1.8.6 → 1.8.7

data/lib/lingo/ctl/analysis.rb

@@ -0,0 +1,136 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'csv'
+
+class Lingo
+
+  module Ctl
+
+    { stats: [:s, 'Extract statistics from analysis file(s)', 'path...'],
+      trans: [:t, 'Transpose columns and rows of analysis file(s)', 'path...']
+    }.each { |n, (s, *a)| cmd("analysis#{n}", "a#{s}", *a) }
+
+    private
+
+    def do_analysisstats
+      require 'nuggets/array/histogram'
+
+      paths, write = paths_write
+      stats, patts = Hash.array(1), Hash.array(1)
+
+      csv_foreach(paths) { |path, _, token, word, pattern|
+        token ? stats[:tokens][path] << token : word ? begin
+          stats[:words][path] << word
+          patts[word][path] << pattern if pattern
+        end : nil
+      }
+
+      stats.each { |k, h| write.(k) { |csv|
+        csv << ['file', *c = columns(g = histograms(h))]
+        histograms_to_csv(csv, c, g)
+        h.values.map(&:size)
+      } }
+
+      write.(:patterns) { |csv|
+        csv << ['file', 'word', *c = columns(patts, :values)]
+        patts.sort.each { |k, h| histograms_to_csv(csv, c, histograms(h), k) }
+        c.size - 1
+      }
+    end
+
+    def do_analysistrans
+      paths, write = paths_write
+
+      rows, comm, more, less = Hash.array(1), {}, Hash.array(1), Hash.array(1)
+
+      csv_foreach(paths) { |path, string, token, word, _|
+        rows[token || word][path] << string }
+
+      c = rows.keys.sort.each { |k|
+        a = (h = rows[k]).first.last
+
+        paths.size == 1 ? comm[k] = a : begin
+          comm[k] = a & (o = h.drop(1)).flat_map(&:last)
+          o.each { |path, b| more[path][k] = b - a; less[path][k] = a - b }
+        end
+      }
+
+      rows.clear
+
+      write.(:transpose) { |csv| transpose_csv(csv, c, comm) }
+
+      { transmore: more, transless: less }.each { |k, v| v.each { |path, h|
+        csv_writer(path, *paths).(k) { |csv| transpose_csv(csv, c, h) } } }
+    end
+
+    def paths_write
+      ARGV.empty? ? missing_arg(:path) : [a = ARGV.each { |x|
+        abort "No such file: #{x}" unless File.exist?(x) }, csv_writer(*a)]
+    end
+
+    def csv_writer(*paths)
+      name = File.join(File.dirname(paths.first), paths.map { |path|
+        File.basename(path.chomp(File.extname(path))) }.uniq.join('-'))
+
+      lambda { |key, &block| overwrite?(file = "#{name}.#{key}.csv") &&
+        puts("#{file}: #{Array(CSV.open(file, 'wb', &block)).join(' / ')}") }
+    end
+
+    def csv_foreach(paths)
+      paths.each { |path| CSV.foreach(path, headers: true) { |row|
+        yield path, *row.values_at(*%w[string token word pattern]) } }
+    end
+
+    def columns(hash, map = :keys)
+      hash.values.map(&map).flatten.uniq.sort
+    end
+
+    def histograms(hash)
+      hash.each_with_object({}) { |(k, v), h|
+        h[k] = v.histogram.tap { |x| x.default = nil } }
+    end
+
+    def histograms_to_csv(csv, columns, histograms, *args)
+      histograms.each { |key, histogram|
+        others = histograms.values_at(*histograms.keys - [key])
+        others = [{}] if others.empty?
+
+        csv << args.dup.unshift(key).concat(columns.map { |header|
+          value = histogram[header] or next
+          value if others.any? { |other| other[header] != value }
+        })
+      }
+    end
+
+    def transpose_csv(csv, columns, rows)
+      csv << columns; values = rows.values_at(*columns); rows.clear
+      values.map(&:size).max.times { |i| csv << values.map { |v| v[i] } }
+    end
+
+  end
+
+end

data/lib/lingo/ctl/files.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Lingo
+
+  module Ctl
+
+    { config: %w[c configuration],
+      lang:   %w[l language],
+      dict:   %w[d dictionary dictionaries],
+      store:  %w[s store],
+      sample: %w[e sample\ text\ file]
+    }.each { |n, (s, q, r)|
+      t = n == :store
+
+      cmd([:list,  :l, n], s, "List available #{r || "#{q}s"}", '[name...]') if !t
+      cmd([:find,  :f, n], s, "Find #{q} in Lingo search path",      'name')
+      cmd([:copy,  :c, n], s, "Copy #{q} to local Lingo directory",  'name') if !t
+      cmd([:clear, :c, n], s, 'Remove store files to force rebuild', 'name') if  t
+    }
+
+    private
+
+    def list(what, doit = true)
+      names = Regexp.union(*ARGV.empty? ? '' : ARGV)
+
+      Lingo.list(what, path: path_for_scope).select { |file|
+        File.basename(file) =~ names ? doit ? puts(file) : true : false
+      }
+    end
+
+    def find(what, doit = true, path = path_for_scope)
+      name = ARGV.shift or missing_arg(:name)
+      no_args
+
+      file = Lingo.find(what, name, path: path) { |err| usage(err) }
+      doit ? puts(file) : file
+    end
+
+    def copy(what)
+      usage('Source and target are the same.') if OPTIONS[:scope] == :local
+
+      local_path = path_for_scope(:local)
+
+      source = find(what, false, path_for_scope || Lingo::PATH - local_path)
+      target = File.expand_path(Lingo.basepath(what, source), local_path[0])
+
+      usage('Source and target are the same.') if source == target
+
+      return unless overwrite?(target)
+
+      FileUtils.mkdir_p(File.dirname(target))
+      FileUtils.cp(source, target, verbose: true)
+    end
+
+    def do_clearstore
+      store = Dir["#{find(:store, false)}.*"]
+      FileUtils.rm(store, verbose: true) unless store.empty?
+    end
+
+  end
+
+end

data/lib/lingo/ctl/other.rb

@@ -0,0 +1,140 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'zip'
+Zip.unicode_names = true
+
+class Lingo
+
+  module Ctl
+
+    { demo:    [:d, 'Initialize demo directory', '[path]', 'current directory'],
+      archive: [:a, 'Create archive of directory', '[path]', 'current directory'],
+      rackup:  [:r, 'Print path to rackup file', 'name'],
+      path:    [:p, 'Print search path for dictionaries and configurations'],
+      help:    [:h, 'Print help for available commands', '[command...]'],
+      version: [:v, 'Print Lingo version number']
+    }.each { |n, (s, *a)| cmd(n.to_s, s.to_s, *a) }
+
+    private
+
+    def do_archive
+      OPTIONS.update(path: ARGV.shift, scope: :local)
+      no_args
+
+      source = File.expand_path(path_for_scope.first)
+      target = "#{source}.zip"
+
+      abort "No such directory: #{source}" unless Dir.exist?(source)
+
+      return unless overwrite?(target, true)
+
+      base, name = File.split(source)
+
+      Dir.chdir(base) {
+        Zip::File.open(target, Zip::File::CREATE) { |zipfile|
+          Dir[File.join(name, '**', '*')].each { |file|
+            zipfile.add(file, file)
+          }
+        }
+      }
+
+      puts "Directory successfully archived at `#{target}'."
+    end
+
+    def do_demo
+      OPTIONS.update(path: ARGV.shift, scope: :system)
+      no_args
+
+      path = path_for_scope(:local).first
+
+      copy_list(:config) { |i| !File.basename(i).start_with?('test') }
+      copy_list(:lang)
+      copy_list(:dict)   { |i|  File.basename(i).start_with?('user') }
+      copy_list(:sample)
+
+      puts "Demo directory successfully initialized at `#{path}'."
+    end
+
+    def do_rackup(doit = true)
+      name = ARGV.shift or missing_arg(:name)
+      no_args
+
+      require 'lingo/app'
+
+      if file = Lingo::App.rackup(name)
+        doit ? puts(file) : file
+      else
+        usage("Invalid app name `#{name.inspect}'.")
+      end
+    end
+
+    def do_path
+      no_args
+      puts path_for_scope || PATH
+    end
+
+    def do_help(opts = nil)
+      msg = opts ? [opts, 'Commands:'] : []
+
+      args = ARGV unless opts || ARGV.empty?
+
+      aliases = Hash.array
+      ALIASES.each { |k, v| aliases[v] << k }
+
+      COMMANDS.each { |c, (d, *e)|
+        a = aliases[c]
+        next if args && ([c, *a] & args).empty?
+
+        c = "#{c} (#{a.join(', ')})" unless a.empty?
+
+        if opts
+          msg << "    %-#{OPTWIDTH}s %s" % [c, d]
+        else
+          msg << "#{c}" << "  - #{d}"
+          e.each { |i| msg <<  "  + #{i}" }
+        end
+      }
+
+      msg.empty? ? abort : abort(msg.join("\n"))
+    end
+
+    def do_version(doit = true)
+      no_args
+
+      msg = "Lingo v#{Lingo::VERSION}"
+      doit ? puts(msg) : msg
+    end
+
+    def copy_list(what)
+      files = list(what, false)
+      files.select! { |i| yield i } if block_given?
+      files.each { |file| ARGV.replace([file]); copy(what) }
+    end
+
+  end
+
+end

data/lib/lingo/database.rb

@@ -6,7 +6,7 @@
 # Lingo -- A full-featured automatic indexing system                          #
 #                                                                             #
 # Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2014 John Vorhauer, Jens Wille                           #
+# Copyright (C) 2007-2015 John Vorhauer, Jens Wille                           #
 #                                                                             #
 # Lingo is free software; you can redistribute it and/or modify it under the  #
 # terms of the GNU Affero General Public License as published by the Free     #
@@ -67,22 +67,33 @@ class Lingo
     def initialize(id, lingo)
       @id, @lingo, @config, @db = id, lingo, lingo.database_config(id), nil
 
-      @srcfile = Lingo.find(:dict, config['name'], relax: true)
-      @crypter = config.key?('crypt') && Crypter.new
-
-      @val = Hash.nest { [] }
+      @val, @crypt, @srcfile = Hash.array, config.key?('crypt'),
+        Lingo.find(:dict, config['name'], relax: true)
 
       begin
         @stofile = Lingo.find(:store, @srcfile)
         FileUtils.mkdir_p(File.dirname(@stofile))
       rescue SourceFileNotFoundError => err
         @stofile = skip_ext = err.id
-        backend = backend_from_file(@stofile) unless err.name
+
+        unless err.name
+          if name = BACKEND_BY_EXT[File.extname(@stofile)]
+            backend = get_backend(name, @stofile)
+          else
+            raise BackendNotFoundError.new(@stofile)
+          end
+        end
       rescue NoWritableStoreError
         backend = HashStore
       end
 
-      use_backend(backend, skip_ext)
+      unless backend ||= get_backend(ENV['LINGO_BACKEND'])
+        BACKENDS.find { |name| backend = get_backend(name, nil, true) }
+      end
+
+      extend(@backend = backend || HashStore)
+
+      @stofile << store_ext unless skip_ext || !respond_to?(:store_ext)
 
       convert unless uptodate?
     end
@@ -131,7 +142,7 @@ class Lingo
       val.uniq!
 
       val = val.join(FLD_SEP)
-      @crypter ? _set(*@crypter.encode(key, val)) : _set(key, val)
+      @crypt ? _set(*Crypter.encode(key, val)) : _set(key, val)
     end
 
     def warn(*msg)
@@ -140,26 +151,13 @@ class Lingo
 
     private
 
-    def use_backend(backend = nil, skip_ext = false)
-      [ENV['LINGO_BACKEND'], *BACKENDS].each { |mod|
-        backend = get_backend(mod) and break if mod
-      } unless backend
-
-      extend(@backend = backend || HashStore)
-
-      @stofile << store_ext if !skip_ext && respond_to?(:store_ext)
-    end
-
-    def get_backend(mod)
-      self.class.const_get("#{mod}Store") if Object.const_defined?(mod)
-    rescue TypeError, NameError
-    end
-
-    def backend_from_file(file)
-      ext = File.extname(file)
+    def get_backend(name, file = nil, relax = false)
+      return unless name
 
-      mod = BACKEND_BY_EXT[ext] or raise BackendNotFoundError.new(file)
-      get_backend(mod) or raise BackendNotAvailableError.new(mod, file)
+      Object.const_get(name)
+      self.class.const_get("#{name}Store")
+    rescue TypeError, NameError => err
+      raise BackendNotAvailableError.new(name, file, err) unless relax
     end
 
     def config_hash
@@ -223,9 +221,9 @@ class Lingo
     end
 
     def _val(key)
-      if val = _get(@crypter ? Crypter.digest(key) : key)
+      if val = _get(@crypt ? Crypter.digest(key) : key)
         _encode!(val)
-        @crypter ? @crypter.decode(key, val) : val
+        @crypt ? Crypter.decode(key, val) : val
       end
     end
 
@@ -236,32 +234,41 @@ class Lingo
     def convert(verbose = lingo.config.stderr.tty?)
       src = Source.get(config.fetch('txt-format', 'key_value'), @id, lingo)
 
-      sep, key_map, val_map = prepare_lex
+      sep, hyphenate, key_map, val_map = prepare_lex
 
       Progress.new(self, src, verbose) { |progress| create {
         src.each { |key, val|
           progress << src.pos
 
-          if key
-            key.chomp!('.')
-
-            if sep && key.include?(sep)
-              key = key.split(sep).map!(&key_map).join(sep)
-              val = val.map { |v| val_map[v.split(sep)].join(sep) } if val_map
+          set_key(src, key, val, sep) { |keys, cnt|
+            key = keys.map!(&key_map).join(sep)
+            val = val.map { |v| val_map[v.split(sep)].join(sep) } if val_map
 
-              if (cnt = key.count(sep)) > 2
-                self[key.split(sep)[0, 3].join(sep)] = ["#{KEY_REF}#{cnt + 1}"]
-              end
-            end
-          end
+            hyphenate.repeated_permutation(cnt - 1) { |h| set_key(src, keys.
+              zip(h).join, val, sep) unless h.uniq.size == 1 } if hyphenate
 
-          src.set(self, key, val)
+            [key, val]
+          }
         }
 
         uptodate!
       } }
     end
 
+    def set_key(src, key, val, sep, len = 3)
+      if key
+        key.chomp!('.')
+
+        if sep && key.include?(sep)
+          keys = key.split(sep); cnt = keys.size
+          key, val = yield keys, cnt if block_given?
+          self[keys[0, len].join(sep)] = ["#{KEY_REF}#{cnt}"] if cnt > len
+        end
+      end
+
+      src.set(self, key, val)
+    end
+
     def prepare_lex
       use_lex = config['use-lex'] or return
 
@@ -275,6 +282,9 @@ class Lingo
 
       args = nil
 
+      wac = Language::WA_COMPOUND
+      lac = Language::LA_COMPOUND
+
       if inflect = config['inflect']
         inflect, wc = inflect == true ? %w[s e] : inflect.split(SEP_RE), 'a'
 
@@ -286,19 +296,16 @@ class Lingo
         end
       end
 
-      [' ', lambda { |form|
+      [sep = ' ', config['hyphenate'] && [sep, '-'], lambda { |form|
         word = dic.find_word(form)
 
         if word.unknown?
-          compo = gra.find_compound(form)
+          comp = gra.find_compound(form)
 
-          if compo_form = compo.compo_form
-            compo_form.form
-          else
-            compo.norm
-          end
+          comp.attr == wac && comp.lex_form(lac) ||
+            (comp.identified? ? comp.lex_form : comp.form)
         else
-          word.norm
+          word.identified? ? word.lex_form : word.form
         end
       }, inflect && lambda { |forms|
         inflectables = []
@@ -306,24 +313,21 @@ class Lingo
         forms.each { |form|
           word = dic.find_word(word_form = form[re])
 
-          if word.identified? and lexical = word.get_class(wc).first
-            inflectables << form if form == lexical.form
+          if word.identified? && _form = word.lex_form(wc)
+            inflectables << form if form == _form
           else
             unless inflectables.empty?
-              comp = gra.find_compound(word_form) if word.unknown?
-              word = comp.head || comp if comp && !comp.unknown?
+              word = gra.find_compound_head(word_form) || word if word.unknown?
 
-              if word.attr?(*inflect)
-                suffix = suffixes[word.genders.compact.first]
-                inflectables.each { |lex_form| lex_form << suffix } if suffix
+              if word.attr?(*inflect) && suffix =
+                suffixes[word.genders.compact.first]
+                inflectables.each { |lex_form| lex_form << suffix }
               end
             end
 
-            break
+            break forms
           end
         }
-
-        forms
       }]
     end