lingo 1.8.2 → 1.8.3

data/lib/lingo/{reportable.rb → srv.rb}

@@ -1,61 +1,59 @@
-# encoding: utf-8
-
-#--
-###############################################################################
-#                                                                             #
-# Lingo -- A full-featured automatic indexing system                          #
-#                                                                             #
-# Copyright (C) 2005-2007 John Vorhauer                                       #
-# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
-#                                                                             #
-# Lingo is free software; you can redistribute it and/or modify it under the  #
-# terms of the GNU Affero General Public License as published by the Free     #
-# Software Foundation; either version 3 of the License, or (at your option)   #
-# any later version.                                                          #
-#                                                                             #
-# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
-# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
-# more details.                                                               #
-#                                                                             #
-# You should have received a copy of the GNU Affero General Public License    #
-# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
-#                                                                             #
-###############################################################################
-#++
-
-class Lingo
-
-  # Provides counters.
-
-  module Reportable
-
-    def init_reportable(prefix = nil)
-      @reportable_hash   = Hash.new(0)
-      @reportable_prefix = prefix ? "#{prefix}: " : ''
-    end
-
-    def inc(key)
-      @reportable_hash[key] += 1
-    end
-
-    def add(key, val)
-      @reportable_hash[key] += val
-    end
-
-    def set(key, val)
-      @reportable_hash[key] = val
-    end
-
-    def get(key)
-      @reportable_hash[key]
-    end
-
-    def report
-      q = @reportable_prefix
-      @reportable_hash.each_with_object({}) { |(k, v), r| r["#{q}#{k}"] = v }
-    end
-
-  end
-
-end
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'json'
+require_relative 'app'
+
+class Lingo
+
+  class Srv < App
+
+    init_app(__FILE__) { %W[-c #{File.join(root, 'lingosrv.cfg')}] }
+
+    LINGO = Call.new(ARGV).call
+    abort 'Something went wrong...' unless LINGO.is_a?(Call)
+
+    c = LINGO.config.get('meeting/attendees', 'vector_filter', 'src')
+    SRC_SEP = c == true ? Attendee::VectorFilter::DEFAULT_SRC_SEP : c
+
+    get('')   { doit }
+    get('/')  { doit }
+    post('/') { doit }
+
+    def doit
+      q = params[:q]
+      r = LINGO.talk(q) if q && !q.empty?
+
+      r = r.inject(Hash.new { |h, k| h[k] = [] }) { |h, s|
+        a, b = s.split(SRC_SEP, 2); h[b] << a; h
+      } if r && SRC_SEP
+
+      to_json(q, r)
+    end
+
+  end
+
+end

data/lib/lingo/version.rb

@@ -4,7 +4,7 @@ class Lingo
 
     MAJOR = 1
     MINOR = 8
-    TINY  = 2
+    TINY  = 3
 
     class << self
 

data/lib/lingo/web/config.ru

@@ -0,0 +1,4 @@
+require 'lingo'
+require 'lingo/web'
+
+run Lingo::Web

data/lib/lingo/web/lingoweb.cfg

@@ -0,0 +1,14 @@
+---
+meeting:
+  attendees:
+    - text_reader:   { files: STDIN }
+
+    - tokenizer:     { }
+    - word_searcher: { source: sys-dic, mode: first }
+    - decomposer:    { source: sys-dic }
+    - multi_worder:  { source: sys-mul }
+    - sequencer:     { stopper: PUNC,OTHR }
+    - synonymer:     { skip: '?,t', source: sys-syn }
+
+    - vector_filter: { debug: 'true', prompt: '' }
+    - text_writer:   { ext: STDOUT, sep: "\n" }

data/lib/lingo/web/public/lingoweb.css

@@ -0,0 +1,74 @@
+html, body {
+  margin-top:       0;
+}
+
+a img {
+  border:           none;
+}
+
+form {
+  white-space:      nowrap;
+}
+
+fieldset {
+  display:          inline;
+  width:            47%;
+}
+
+textarea {
+  width:            98.9%;
+  height:           30em;
+  background-color: white;
+}
+
+#welcome {
+  font-size:        70%;
+  color:            #333333;
+  margin-bottom:    0.5em;
+  text-align:       center;
+}
+
+#legend {
+  font-size:        75%;
+  color:            #333333;
+  margin-bottom:    0.5em;
+}
+
+#legend table {
+  margin-left:      2em;
+}
+
+#legend th {
+  font-size:        110%;
+  font-weight:      normal;
+  font-family:      monospace;
+  text-align:       left;
+}
+
+#legend td {
+  padding-left:     1em;
+}
+
+#footer {
+  border-style:     solid;
+  border-color:     black;
+  border-width:     1px 0;
+  padding:          2px 4px;
+}
+
+#footer a {
+  font-weight:      bold;
+}
+
+a:link, a:visited {
+  text-decoration:  none;
+  color:            #F35327;
+}
+
+fieldset, #footer {
+  background-color: #DFDFDF;
+}
+
+fieldset.error {
+  background-color: #FDB331;
+}

data/lib/lingo/web/views/index.erb

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
+    "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+  <meta http-equiv="content-type" content="application/xhtml+xml; charset=utf-8" />
+  <title>Lingo Web - <%= t 'Automatic indexing online', 'Automatische Indexierung Online' %></title>
+  <link rel="stylesheet" type="text/css" href="<%= url_for '/lingoweb.css' %>" />
+</head>
+<body>
+  <div id="header">
+    <a href="http://lex-lingo.de"><img src="<%= url_for '/lingo.png' %>" alt="Lingo" /></a>
+  </div>
+
+  <div id="welcome">
+    <strong><%= t 'Welcome to Lingo Web!', 'Willkommen bei Lingo Web!' %></strong>
+    <%= t %q{
+      Lingo Web provides the opportunity to test the functionality of
+      <a href="http://lex-lingo.de">Lingo</a>.<br />
+      Lingo is an open source indexing system for research and teaching.
+    }, %q{
+      Lingo Web bietet die Möglichkeit, die Funktionsweise von
+      <a href="http://lex-lingo.de">Lingo</a> zu testen.<br />
+      Lingo ist ein frei verfügbares System zur linguistisch und statistisch
+      basierten automatischen Indexierung des Deutschen und Englischen.
+    } %>
+  </div>
+
+  <div id="main">
+    <form action="<%= url_for '/' %>" method="post">
+      <div>
+        <fieldset><legend><strong><%= t 'Input', 'Eingabe' %></strong></legend>
+          <textarea name="q" rows="20" cols="50"><%= @q %></textarea>
+        </fieldset>
+
+        <fieldset><legend><strong><%= t 'Output', 'Ausgabe' %></strong></legend>
+          <textarea readonly="readonly" rows="20" cols="50"><%= @r %></textarea>
+        </fieldset>
+
+        <br />
+
+        <strong><%= t 'Language', 'Sprache' %></strong> = <select name="l">
+        <% for l in LANGS %>
+          <option value="<%= l %>"<%= ' selected="selected"' if l == @l %>><%= l %></option>
+        <% end %>
+        </select>
+
+        <br />
+        <br />
+
+        <input type="submit" value="<%= t 'Start processing...', 'Verarbeitung starten...' %>"></input> |
+        <input type="reset" value="<%= t 'Reset form', 'Formular zurücksetzen' %>"></input> |
+        <a href="<%= url_for '/' %>"><%= t 'New request', 'Neue Anfrage' %></a>
+      </div>
+    </form>
+
+    <br />
+  </div>
+
+  <div id="legend">
+    <strong><%= t 'Legend', 'Legende' %></strong>:
+    <table>
+      <tr><th>s       </th><td><%= t 'Noun',                                                    'Substantiv' %></td></tr>
+      <tr><th>a       </th><td><%= t 'Adjective',                                               'Adjektiv' %></td></tr>
+      <tr><th>v       </th><td><%= t 'Verb',                                                    'Verb' %></td></tr>
+      <tr><th>e       </th><td><%= t 'Proper name',                                             'Eigenname' %></td></tr>
+      <tr><th>w       </th><td><%= t 'Word class without suffixes',                             'Wortklasse ohne Suffixe' %></td></tr>
+      <tr><th>t       </th><td><%= t 'Word class without suffixes (e.g. high frequency terms)', 'Wortklasse ohne Suffixe (z.B. Hochfrequenzterme)' %></td></tr>
+      <tr><th>y       </th><td><%= t 'Synonym',                                                 'Synonym' %></td></tr>
+      <tr><th>q (=SEQ)</th><td><%= t 'Sequence (algorithmically identified phrase)',            'Sequenz (algorithmisch erkannter Mehrwortbegriff)' %></td></tr>
+      <tr><th>m (=MUL)</th><td><%= t 'Phrase',                                                  'Mehrwortbegriff' %></td></tr>
+      <tr><th>k (=KOM)</th><td><%= t 'Compound',                                                'Kompositum' %></td></tr>
+      <tr><th>+       </th><td><%= t 'Part of a compound',                                      'Kompositum-Bestandteil' %></td></tr>
+      <tr><th>x+      </th><td><%= t 'Unknown part of a hyphenated compound',                   'unbekannter Kompositum-Bestandteil einer Bindestrich-Konstruktion' %></td></tr>
+      <tr><th>?       </th><td><%= t 'Unknown word',                                            'unbekanntes Wort' %></td></tr>
+      <tr><th>MU?     </th><td><%= t 'Part of a phrase (unknown word)',                         'Mehrwortbestandteil (unbekanntes Wort)' %></td></tr>
+      <tr><th>HELP    </th><td><%= t 'e.g. Special characters',                                 'z.B. unbekanntes Sonderzeichen' %></td></tr>
+      <tr><th>ABRV    </th><td><%= t 'Possible abbreviation with a full stop in the middle',    'mögliche Abk. mit eingeschlossenem Punkt (z.B. "Ausst.Kat")' %></td></tr>
+      <tr><th>PUNC    </th><td><%= t 'Punctuation etc.',                                        'Satzzeichen etc.' %></td></tr>
+      <tr><th>OTHR    </th><td><%= t 'Other character',                                         'Sonstiges Zeichen' %></td></tr>
+      <tr><th>URLS    </th><td><%= t 'URL',                                                     'URL' %></td></tr>
+      <tr><th>NUMS    </th><td><%= t 'Number',                                                  'Zahl' %></td></tr>
+    </table>
+  </div>
+
+  <div id="footer">
+    <em>powered by</em> <a href="http://lex-lingo.de">Lingo</a>
+    <em>and</em> <a href="http://www.sinatrarb.com">Sinatra</a>
+    -- <strong>v<%= Lingo::VERSION %></strong>
+  </div>
+</body>
+</html>

data/lib/lingo/web.rb

@@ -0,0 +1,94 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Lingo -- A full-featured automatic indexing system                          #
+#                                                                             #
+# Copyright (C) 2005-2007 John Vorhauer                                       #
+# Copyright (C) 2007-2012 John Vorhauer, Jens Wille                           #
+#                                                                             #
+# Lingo is free software; you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Lingo is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Lingo. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'json'
+require 'nuggets/util/ruby'
+
+require_relative 'app'
+
+class Lingo
+
+  class Web < App
+
+    init_app(__FILE__)
+
+    UILANGS, LANGS = %w[en de], Lingo.list(:lang).map! { |lang|
+      lang[%r{.*/(\w+)\.}, 1]
+    }.uniq.sort!
+
+    auth, cfg = %w[auth cfg].map! { |ext|
+      File.join(root, "lingoweb.#{ext}")
+    }
+
+    if File.readable?(auth)
+      c = File.read(auth).chomp.split(':', 2)
+      use(Rack::Auth::Basic) { |*b| b == c } unless c.empty?
+    end
+
+    LINGO = Hash.new { |h, k| h[k] = Lingo.call(cfg, ['-l', k]) }
+
+    before do
+      @uilang = if hal = env['HTTP_ACCEPT_LANGUAGE']
+        hals = hal.split(',').map { |l| l.split('-').first.strip }
+        (hals & UILANGS).first
+      end || UILANGS.first
+
+      @q = params[:q]
+      @l = params[:l] || @uilang
+      @l = LANGS.first unless LANGS.include?(@l)
+    end
+
+    get('')   { redirect url_for('/') }
+    get('/')  { doit }
+    post('/') { doit }
+
+    helpers do
+      def url_for(path)
+        "#{request.script_name}#{path}"
+      end
+
+      def t(*t)
+        (i = UILANGS.index(@uilang)) && t[i] || t.first
+      end
+    end
+
+    def doit
+      @r = LINGO[@l].talk(@q) { |_| _ } if @q && !@q.empty?
+
+      case params[:f]
+        when 'json'
+          to_json(@q, @r)
+        when 'text'
+          @r &&  @r.join("\n")
+        else
+          @r &&= @r.join("\n")
+          erb :index
+      end
+    end
+
+  end
+
+end

data/lib/lingo.rb

@@ -24,13 +24,12 @@
 ###############################################################################
 #++
 
+require 'unicode'
 require 'stringio'
 require 'pathname'
 require 'fileutils'
-require 'benchmark'
 require 'nuggets/file/ext'
 require 'nuggets/env/user_home'
-require 'nuggets/numeric/duration'
 require 'nuggets/string/camelscore'
 
 class Lingo
@@ -60,7 +59,7 @@ class Lingo
   }
 
   # Default encoding
-  ENC = 'UTF-8'.freeze
+  Encoding.default_external = Encoding.default_internal = ENC = 'UTF-8'.freeze
 
   SEP_RE = %r{[; ,|]}
 
@@ -76,17 +75,18 @@ class Lingo
 
     def list(type, options = {})
       options = options_for(type, options)
-      path    = path_for(options)
 
-      glob = file_with_ext('*', options)
+      glob, list = file_with_ext('*', options), []
       glob = File.join('??', glob) if type == :dict
 
-      [].tap { |list| walk(path, options) { |dir|
+      walk(path = path_for(options), options) { |dir|
         Dir[File.join(dir, glob)].sort!.each { |file|
           pn = Pathname.new(file)
           list << realpath_for(pn, path) if pn.file?
         }
-      } }
+      }
+
+      list
     end
 
     def find(type, file, options = {})
@@ -130,7 +130,7 @@ class Lingo
     private
 
     def include_path(path, pre = false)
-      PATH.insert(pre ? 0 : -1, *path.map!(&:to_s))
+      PATH.insert(pre ? 0 : -1, *path.map! { |i| i.to_s })
     end
 
     def find_file(file, path, options)
@@ -223,13 +223,15 @@ class Lingo
 
   end
 
-  attr_reader :dictionaries, :report_status, :report_time
-
   def initialize(*args)
+    Debug.ps(:lingo_new)
+
     @config_args = args
     reset(false)
   end
 
+  attr_reader :dictionaries
+
   def config
     @config ||= Config.new(*@config_args)
   end
@@ -252,8 +254,12 @@ class Lingo
   end
 
   def talk
-    invite
-    start
+    Debug.profile(config['profile']) {
+      invite
+      start
+    }
+
+    Debug.ps(:lingo_talk)
   ensure
     reset
   end
@@ -288,38 +294,30 @@ class Lingo
     } }
   end
 
-  def start(report_status = config['status'], report_time = config['perfmon'])
-    @report_status, @report_time = report_status, report_time
-
-    time = Benchmark.realtime {
-      @attendees.first.listen(AgendaItem.new(Attendee::STR_CMD_TALK))
-    }
-
-    if report_status || report_time
-      warn "Require protocol...\n#{separator = '-' * 61}"
-      @attendees.first.listen(AgendaItem.new(Attendee::STR_CMD_STATUS))
-      warn "#{separator}\nThe duration of the meeting was #{time.to_hms(2)}"
-    end
+  def start
+    @attendees.first.listen(AgendaItem.new(Attendee::STR_CMD_TALK))
   end
 
   def reset(close = true)
-    dictionaries.each(&:close) if close
+    dictionaries.each { |i| i.close } if close
     @dictionaries, @attendees = [], []
     @lexical_hash = Hash.new { |h, k| h[k] = Language::LexicalHash.new(k, self) }
   end
 
   def warn(*msg)
-    config.stderr.puts(*msg)
+    config.warn(*msg)
+  end
+
+  def deprecate(old, new, obj = self)
+    config.deprecate(old, new, obj)
   end
 
 end
 
 require_relative 'lingo/call'
 require_relative 'lingo/error'
+require_relative 'lingo/debug'
 require_relative 'lingo/config'
-require_relative 'lingo/core_ext'
-require_relative 'lingo/cachable'
-require_relative 'lingo/reportable'
 require_relative 'lingo/agenda_item'
 require_relative 'lingo/show_progress'
 require_relative 'lingo/database'

data/lingo.cfg

@@ -30,7 +30,7 @@ meeting:
     # Schreibweisen variieren und erneut suchen
 #   - variator:        { source: sys-dic }
 
-    # Bindestrichergänzungen rekonstruieren
+    # Worttrennungen aufheben
 #   - dehyphenizer:    { source: sys-dic }
 
     # Wortstämme für nicht erkannte Wörter einfügen

data/lir.cfg

@@ -26,9 +26,21 @@ meeting:
     # Zeile in einzelnen Sinnbestandteile (Token) zerlegen
     - tokenizer:       { }
 
+    # Abkürzungen erkennen und auflösen
+#   - abbreviator:     { source: sys-abk }
+
     # Verbleibende Token im Wörterbuch suchen
     - word_searcher:   { source: sys-dic, mode: first }
 
+    # Schreibweisen variieren und erneut suchen
+#   - variator:        { source: sys-dic }
+
+    # Worttrennungen aufheben
+#   - dehyphenizer:    { source: sys-dic }
+
+    # Wortstämme für nicht erkannte Wörter einfügen
+#   - stemmer:         { }
+
     # Nicht erkannte Wörter auf Kompositum testen
     - decomposer:      { source: sys-dic }
 
@@ -60,10 +72,22 @@ meeting:
     - noneword_filter: { in: syn }
     - text_writer:     { ext: non, sep: '|' }
 
+    # Erstelle Datei mit Endung .ste für Wortstämme
+    - vector_filter:   { in: syn, lexicals: z }
+    - text_writer:     { ext: ste, sep: '|' }
+
     # Erstelle Datei mit Endung .vec für erkannte Indexterme
     - vector_filter:   { in: syn, lexicals: '^[ksavem]$' }
     - text_writer:     { ext: vec, sep: '|' }
 
+    # Erstelle Datei mit Endung .ven für erkannte Indexterme mit absoluter Häufigkeit
+    - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: term_abs }
+    - text_writer:     { ext: ven, sep: '|' }
+
+    # Erstelle Datei mit Endung .ver für erkannte Indexterme mit relativer Häufigkeit
+    - vector_filter:   { in: syn, lexicals: '^[ksavem]$', sort: term_rel }
+    - text_writer:     { ext: ver, sep: '|' }
+
     # Erstelle Datei mit Endung .mul für erkannte Mehrwortgruppen
     - vector_filter:   { in: syn, lexicals: m }
     - text_writer:     { ext: mul, sep: '|' }