lingo 1.8.2 → 1.8.3

data/lib/lingo/attendee/tokenizer.rb

@@ -44,28 +44,10 @@ class Lingo
     # === Konfiguration
     # Der Tokenizer benötigt zur Identifikation einzelner Token Regeln, nach denen er
     # arbeiten soll. Die benötigten Regeln werden aufgrund des Umfangs nicht als Parameter,
-    # sondern in der Sprachkonfiguration hinterlegt, die sich standardmäßig in der Datei
-    # <tt>de.lang</tt> befindet (YAML-Format).
-    #   language:
-    #     attendees:
-    #       tokenizer:
-    #         regulars:
-    #           - _CHR_: '\wÄÖÜÁÂÀÉÊÈÍÎÌÓÔÒÚÛÙÝäöüáâàéêèíîìóôòúûùý'
-    #           - NUMS:  '[+-]?(\d{4,}|\d{1,3}(\.\d{3,3})*)(\.|(,\d+)?%?)'
-    #           - URLS:  '((mailto:|(news|http|https|ftp|ftps)://)\S+|^(www(\.\S+)+)|\S+([\._]\S+)+@\S+(\.\S+)+)'
-    #           - ABRV:  '(([_CHR_]+\.)+)[_CHR_]+'
-    #           - ABRS:  '(([_CHR_]{1,1}\.)+)(?!\.\.)'
-    #           - WORD:  '[_CHR_\d]+'
-    #           - PUNC:  '[!,\.:;?]'
-    #           - OTHR:  '[—„!\"#$%&()*\+,\-\./:;<=>?@\[\\\]^_`{|}~´]'
-    #           - HELP:  '.*'
+    # sondern in einer Programmkonstanten hinterlegt.
     # Die Regeln werden in der angegebenen Reihenfolge abgearbeitet, solange bis ein Token
     # erkannt wurde. Sollte keine Regel zutreffen, so greift die letzt Regel +HELP+ in jedem
     # Fall.
-    # Regeln, deren Name in Unterstriche eingefasst sind, werden als Makro interpretiert.
-    # Makros werden genutzt, um lange oder sich wiederholende Bestandteile von Regeln
-    # einmalig zu definieren und in den Regeln über den Makronamen eine Auflösung zu forcieren.
-    # Makros werden selber nicht für die Erkennung von Token eingesetzt.
     #
     # === Generierte Kommandos
     # Damit der nachfolgende Datenstrom einwandfrei verarbeitet werden kann, generiert der Tokenizer
@@ -98,33 +80,79 @@ class Lingo
 
     class Tokenizer < self
 
+      CHAR, DIGIT = Char::CHAR, Char::DIGIT
+
+      RULES = [
+        ['WIKI', /^=+.+=+$/],
+        ['SPAC', /^\s+/],
+        ['HTML', /^<[^>]+>/],
+        ['WIKI', /^\[\[.+?\]\]/],
+        ['NUMS', /^[+-]?(?:\d{4,}|\d{1,3}(?:\.\d{3,3})*)(?:\.|(?:,\d+)?%?)/],
+        ['URLS', /^(?:(?:mailto:|(?:news|https?|ftps?):\/\/)\S+|^(?:www(?:\.\S+)+)|[^\s.]+(?:[\._]\S+)+@\S+(?:\.\S+)+)/],
+        ['ABRV', /^(?:(?:(?:#{CHAR})+\.)+)(?:#{CHAR})+/],
+        ['WORD', /^(?:#{CHAR}|#{DIGIT}|-)+/],
+        ['PUNC', /^[!,.:;?¡¿]/],
+        ['OTHR', /^["$#%&'()*+\-\/<=>@\[\\\]^_{|}~¢£¤¥¦§¨©«¬®¯°±²³´¶·¸¹»¼½¾×÷]/],
+        ['HELP', /^[^ ]*/]
+      ]
+
+      class << self
+
+        def rule(name)
+          RULES.assoc(name)
+        end
+
+        def delete(*names)
+          names.each { |name| RULES.delete(rule(name)) }
+        end
+
+        def replace(name, expr)
+          rule = rule(name) or return
+          rule[1] = block_given? ? yield(rule[1]) : expr
+        end
+
+        def insert(*rules)
+          _insert(0, rules)
+        end
+
+        def append(*rules)
+          _insert(-1, rules)
+        end
+
+        def insert_before(name, *rules)
+          _insert_name(name, rules, 0)
+        end
+
+        def insert_after(name, *rules)
+          _insert_name(name, rules, -1)
+        end
+
+        private
+
+        def _insert(index, rules)
+          rules.push(*rules.pop) if rules.last.is_a?(Hash)
+          RULES.insert(index, *rules)
+        end
+
+        def _insert_name(name, rules, offset)
+          index = RULES.index(rule(name))
+          _insert(index ? index - offset : offset, rules)
+        end
+
+      end
+
       protected
 
       def init
         @space = get_key('space', false)
-        @tags  = get_key('tags',  true)
-        @wiki  = get_key('wiki',  true)
-
-        # default rules
-        @rules = [['SPAC', /^\s+/]]
-        @rules << ['HTML', /^<[^>]+>/]       unless @tags
-        @rules << ['WIKI', /^\[\[.+?\]\]/]   unless @wiki
-        @rules.unshift(['WIKI', /^=+.+=+$/]) unless @wiki
-
-        get_key('regulars', []).each_with_object({}) { |rule, macros|
-          expr = rule.values.first.gsub(/_(\w+?)_/) {
-            macros[$&] || begin
-              Database::Source.const_get("UTF8_#{$1.upcase}")
-            rescue NameError
-            end
-          }
+        @tags  = get_key('tags',  false)
+        @wiki  = get_key('wiki',  false)
 
-          if (name = rule.keys.first) =~ /^_\w+_$/
-            macros[name] = expr
-          else
-            @rules << [name, /^#{expr}/]
-          end
-        }
+        skip = []
+        skip << 'HTML' unless @tags
+        skip << 'WIKI' unless @wiki
+
+        @rules = RULES.reject { |name, _| skip.include?(name) }
 
         @filename = @cont = nil
       end
@@ -139,15 +167,7 @@ class Lingo
 
       def process(obj)
         if obj.is_a?(String)
-          inc('Anzahl Zeilen')
-
-          tokenize(obj) { |form, attr|
-            inc("Anzahl Muster #{attr}")
-            inc('Anzahl Token')
-
-            forward(Token.new(form, attr))
-          }
-
+          tokenize(obj) { |*i| forward(Token.new(*i)) }
           forward(STR_CMD_EOL, @filename) if @filename
         else
           forward(obj)
@@ -157,44 +177,42 @@ class Lingo
       private
 
       # tokenize("Eine Zeile.")  ->  [:Eine/WORD:, :Zeile/WORD:, :./PUNC:]
-      def tokenize(textline)
+      def tokenize(line)
         case @cont
           when 'HTML'
-            if textline =~ /^[^<>]*>/
+            if line =~ /^[^<>]*>/
               yield $&, @cont
-              textline, @cont = $', nil
+              line, @cont = $', nil
             else
-              yield textline, @cont
+              yield line, @cont
               return
             end
           when 'WIKI'
-            if textline =~ /^[^\[\]]*\]\]/
+            if line =~ /^[^\[\]]*\]\]/
               yield $&, @cont
-              textline, @cont = $', nil
+              line, @cont = $', nil
             else
-              yield textline, @cont
+              yield line, @cont
               return
             end
           when nil
-            if !@tags && textline =~ /<[^<>]*$/
+            if @tags && line =~ /<[^<>]*$/
               yield $&, @cont = 'HTML'
-              textline = $`
+              line = $`
             end
 
-            if !@wiki && textline =~ /\[\[[^\[\]]*$/
+            if @wiki && line =~ /\[\[[^\[\]]*$/
               yield $&, @cont = 'WIKI'
-              textline = $`
+              line = $`
             end
         end
 
-        until textline.empty?
-          @rules.each { |name, expr|
-            if textline =~ expr
-              yield $&, name if name != 'SPAC' || @space
-              textline = $'
-              break
-            end
-          }
+        while (l = line.length) > 0 && @rules.find { |name, expr|
+          if line =~ expr
+            yield $&, name if name != 'SPAC' || @space
+            l == $'.length ? break : line = $'
+          end
+        }
         end
       end
 

data/lib/lingo/attendee/variator.rb

@@ -90,22 +90,24 @@ class Lingo
       end
 
       def control(cmd, param)
-        report_on(cmd, @dic, @gra)
       end
 
       def process(obj)
         if obj.is_a?(Word) && @check[obj.attr]
-          inc('Anzahl gesuchter Wörter')
+          vars, max = [obj.form], @max
 
-          @var.each_with_object([obj.form]) { |a, v| variate(v, *a) }.
-            tap { |v| v.slice!(@max..-1) }.each { |var|
+          @var.each { |args|
+            variate(vars, *args)
+            break unless vars.length < max
+          }
+
+          vars.each { |var|
             next if (word = find_word(var)).unknown? || (
               word.attr == WA_COMPOUND && word.lexicals.any? { |lex|
                 lex.attr.start_with?(LA_TAKEITASIS)
               }
             )
 
-            inc('Anzahl gefundener Wörter')
             return forward(word.tap { word.form = @marker + var })
           }
         end

data/lib/lingo/attendee/vector_filter.rb

@@ -79,15 +79,20 @@ class Lingo
 
     class VectorFilter < self
 
+      DEFAULT_SRC_SEP = '|'
+
       protected
 
       def init
         if @debug = get_key('debug', false)
           @prompt = get_key('prompt', 'lex:) ')
         else
-          @lex  = Regexp.new(get_key('lexicals', '[sy]').downcase)
+          @lex  = get_re('lexicals', '[sy]')
           @skip = get_array('skip', DEFAULT_SKIP, :upcase)
 
+          @src = get_key('src', false)
+          @src = DEFAULT_SRC_SEP if @src == true
+
           if sort = get_key('sort', 'normal')
             @sort_format, @sort_method = sort.downcase.split('_', 2)
           end
@@ -111,25 +116,20 @@ class Lingo
         elsif obj.is_a?(Word) && !@skip.include?(obj.attr)
           @word_count += 1
 
-          cnt = obj.get_class(@lex).each { |lex|
-            vec = lex.form.downcase
+          obj.get_class(@lex).each { |lex|
+            vec = Unicode.downcase(lex.form)
+            vec << @src << lex.src if @src && lex.src
             @sort_format ? @vectors << vec : forward(vec)
-          }.size
-
-          add('Anzahl von Vektor-Wörtern', cnt)
+          }
         end
       end
 
       private
 
       def send_vectors
-        add('Objekte gefiltert', @vectors.size)
-
         if @sort_format == 'normal'
-          @vectors.sort!
           @vectors.uniq!
-
-          @vectors.each(&method(:forward)).clear
+          flush(@vectors.sort!)
         else
           cnt, fmt = Hash.new(0), '%d'
 

data/lib/lingo/attendee/word_searcher.rb

@@ -72,18 +72,10 @@ class Lingo
       end
 
       def control(cmd, param)
-        report_on(cmd, @dic)
       end
 
       def process(obj)
-        if obj.is_a?(Token) && obj.attr == TA_WORD
-          inc('Anzahl gesuchter Wörter')
-
-          obj = @dic.find_word(obj.form)
-          inc('Anzahl gefundener Wörter') unless obj.unknown?
-        end
-
-        forward(obj)
+        forward(obj.is_a?(Token) && obj.word? ? @dic.find_word(obj.form) : obj)
       end
 
     end

data/lib/lingo/attendee.rb

@@ -56,7 +56,7 @@ class Lingo
   # was macht attendee
   # - verkettung der attendees anhand von konfigurationsinformationen
   # - bereitstellung von globalen und spezifischen konfigurationsinformationen
-  # - behandlung von bestimmten übergreifenden Kommandos, z.B. STR_CMD_TALK, STR_CMD_STATUS
+  # - behandlung von bestimmten übergreifenden Kommandos, z.B. STR_CMD_TALK
   # - separierung und routing von kommando bzw. datenobjekten
   #
   # was macht die abgeleitet klasse
@@ -67,39 +67,28 @@ class Lingo
   class Attendee
 
     include Language
-    include Reportable
 
     STR_CMD_TALK   = 'TALK'
-    STR_CMD_STATUS = 'STATUS'
     STR_CMD_LIR    = 'LIR-FORMAT'
     STR_CMD_FILE   = 'FILE'
     STR_CMD_EOL    = 'EOL'
     STR_CMD_RECORD = 'RECORD'
     STR_CMD_EOF    = 'EOF'
 
-    STA_NUM_COMMANDS = 'Received Commands'
-    STA_NUM_OBJECTS  = 'Received Objects '
-    STA_TIM_COMMANDS = 'Time to control  '
-    STA_TIM_OBJECTS  = 'Time to process  '
-
     DEFAULT_SKIP = [TA_PUNCTUATION, TA_OTHER].join(',')
 
     def initialize(config, lingo)
-      @lingo = lingo
-
-      init_reportable
+      @lingo, @config, @subscriber = lingo, config, []
 
       # Make sure config exists
       lingo.dictionary_config
 
-      @config, @subscriber = config, []
-
       init if self.class.method_defined?(:init)
 
       @can_control = self.class.method_defined?(:control)
       @can_process = self.class.method_defined?(:process)
 
-      @skip_command, @timer = false, nil
+      @skip_command = false
     end
 
     def add_subscriber(subscriber)
@@ -107,28 +96,17 @@ class Lingo
     end
 
     def listen(obj)
-      unless obj.is_a?(AgendaItem)
-        @can_process ? stat_timer(:objects) { process(obj) } : forward(obj)
-      else
+      if obj.is_a?(AgendaItem)
         args = obj.to_a
-        stat_timer(:commands) { control(*args) } if @can_control
-
-        case obj.cmd
-          when STR_CMD_TALK
-            nil
-          when STR_CMD_STATUS
-            report_time
-            report_status
-
-            forward(*args)
-          else
-            forward(*args) unless skip_command!
-        end
+        control(*args) if @can_control
+        forward(*args) unless obj.cmd == STR_CMD_TALK || skip_command!
+      else
+        @can_process ? process(obj) : forward(obj)
       end
     end
 
     def talk(obj)
-      charge_timer { @subscriber.each { |attendee| attendee.listen(obj) } }
+      @subscriber.each { |attendee| attendee.listen(obj) }
     end
 
     private
@@ -138,77 +116,6 @@ class Lingo
       g && (block_given? ? !yield(w) : w.unknown?) ? g.find_compound(f) : w
     end
 
-    def report_on(cmd, *rep)
-      rep.each { |r| r.report.each { |q| set(*q) } } if cmd == STR_CMD_STATUS
-    end
-
-    def sta_for(key)
-      %w[NUM TIM].map { |i| self.class.const_get("STA_#{i}_#{key.upcase}") }
-    end
-
-    def stat_timer(key)
-      n, t = sta_for(key)
-      inc(n)
-
-      return yield unless @lingo.report_time
-
-      @timer = Time.now.to_i
-      res = yield
-      add(t, Time.now.to_i - @timer)
-      res
-    end
-
-    def charge_timer
-      return yield unless @lingo.report_time
-
-      res = nil
-      @timer += Benchmark.realtime { res = yield }
-      res
-    end
-
-    def report_time
-      return unless @lingo.report_time
-
-      msg = 'Perf: %-15s ' <<
-            '=> %7d commands in %s (%s/cmd)' <<
-            ',  %8d objects in %s (%s/obj)'
-
-      arg = [@config['name']]
-
-      %w[commands objects].each { |k|
-        n, t = sta_for(k).map(&method(:get))
-        arg << n
-
-        arg.concat([1, n].map { |m|
-          s = m.zero? ? 0.0 : t / m.to_f
-
-          '%9.3f %-2s' %
-            if s < 0.001
-              [s * 1000.0 ** 2, 'µs']
-            elsif s < 1.0
-              [s * 1000.0,      'ms']
-            elsif s < 60.0
-              [s,               's']
-            elsif s < 60.0 ** 2
-              [s / 60.0,        'm']
-            else
-              [s / 60.0 ** 2,   'h']
-            end
-        })
-      }
-
-      warn msg % arg
-    end
-
-    def report_status
-      return unless @lingo.report_status
-
-      msg = "Attendee <%s> was connected from '%s' to '%s' reporting..."
-
-      warn msg % @config.values_at(*%w[name in out]), nil,
-        report.sort.map! { |k, v| " #{k} = #{v}" }, nil
-    end
-
     def skip_command
       @skip_command = true
     end
@@ -221,17 +128,29 @@ class Lingo
       talk(param ? AgendaItem.new(obj, param) : obj)
     end
 
+    def flush(buffer)
+      buffer.each { |i| forward(i) }.clear
+    end
+
     def has_key?(key)
       @config && @config.has_key?(key)
     end
 
-    def get_key(key, default = nodefault = Object.new)
+    def get_key(key, default = nodefault = true)
       raise MissingConfigError.new(key) if nodefault && !has_key?(key)
       @config.fetch(key, default)
     end
 
-    def get_array(key, default = nil, m = nil)
-      get_key(key, default).split(SEP_RE).tap { |ary| ary.map!(&m) if m }
+    def get_array(key, default = nil, method = nil)
+      ary = get_key(key, default).split(SEP_RE)
+      ary.map!(&method) if method
+      ary
+    end
+
+    def get_re(key, default = nil, standard = nil)
+      if value = get_key(key, default)
+        value == true ? standard : Regexp.new(value)
+      end
     end
 
     def dictionary(src, mod)

data/lib/lingo/buffered_attendee.rb

@@ -29,7 +29,7 @@ class Lingo
   class BufferedAttendee < Attendee
 
     def initialize(config, lingo)
-      @buffer, @inserts = [], []
+      @buffer = []
       super
     end
 
@@ -47,11 +47,6 @@ class Lingo
       obj.form if obj.is_a?(klass)
     end
 
-    def forward_buffer
-      @inserts.sort_by!(&:first).each { |i| @buffer.insert(*i) }.clear
-      @buffer.each(&method(:forward)).clear
-    end
-
     def forward_number_of_token(len = default = @buffer.size, punct = !default)
       begin
         unless @buffer.empty?
@@ -74,9 +69,7 @@ class Lingo
       raise NotImplementedError
     end
 
-    def control_multi(cmd, dic = @dic)
-      report_on(cmd, dic)
-
+    def control_multi(cmd)
       if [STR_CMD_RECORD, STR_CMD_EOF].include?(cmd)
         @eof_handling = true
 

data/lib/lingo/call.rb

@@ -28,6 +28,8 @@ class Lingo
 
   class Call < self
 
+    CHANNELS = %w[stdout stderr].freeze
+
     def initialize(args = [])
       super(args, StringIO.new, StringIO.new, StringIO.new)
     end
@@ -51,20 +53,23 @@ class Lingo
 
       start
 
-      %w[stdout stderr].flat_map { |key|
-        io = config.send(key).tap(&:rewind)
-        io.readlines.each(&:chomp!).tap {
-          io.truncate(0)
-          io.rewind
-        }
-      }.tap { |res|
-        if block_given?
-          res.map!(&Proc.new)
-        else
-          res.sort!
-          res.uniq!
-        end
+      res = CHANNELS.flat_map { |key|
+        io = config.send(key)
+        io.rewind
+
+        lines = io.readlines.each { |i| i.chomp! }
+
+        io.truncate(0)
+        io.rewind
+
+        lines
       }
+
+      block_given? ? res.map! { |i| yield i } : begin
+        res.sort!
+        res.uniq!
+        res
+      end
     end
 
   end

data/lib/lingo/cli.rb

@@ -36,8 +36,7 @@ class Lingo
         super.merge(
           config:   'lingo.cfg',
           language: 'de',
-          status:   false,
-          perfmon:  false
+          profile:  false
         )
       end
 
@@ -68,18 +67,14 @@ class Lingo
 
       opts.separator ''
 
-      opts.on('-s', '--status', 'Print status information after processing') {
-        options[:status] = true
-      }
-
-      opts.on('-p', '--perfmon', 'Print performance details after processing') {
-        options[:perfmon] = true
+      opts.on('-L', '--log FILE', 'Log file to print debug information to') { |log|
+        options[:log] = stderr.reopen(log == '-' ? stdout : File.open(log, 'a+', encoding: ENC))
       }
 
       opts.separator ''
 
-      opts.on('-L', '--log FILE', 'Log file to print debug and status information to') { |log|
-        options[:log] = @stderr.reopen(File.open(log, 'a+', encoding: ENC))
+      opts.on('-P', '--profile PATH', 'Print profiling results') { |profile|
+        options[:profile] = profile == '-' ? stdout : profile
       }
     end