norikra 1.2.1-java → 1.2.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e2938132b83a49bb328d060517750c5cb9d7e95
-  data.tar.gz: 7c20e09c3795b209946f08f49d3503ccccec3caf
+  metadata.gz: fa80888abaaa606aa96b77fc3e1e33b17f8bb098
+  data.tar.gz: 423c64919bae627115f30f37ac6750883a534676
 SHA512:
-  metadata.gz: 6a684145944825a6c813ec8695a7eee3ee6411de5654af120a90a77f7ee4e0fae9becef6535c476b5afce4754a7923d9188a8feca8c912a0c55393f2a6989794
-  data.tar.gz: 4140d210916fcb7d575527fce40f9b3bda7d7fc3a9f7af9d88b2ef1c6202678d63b28c7369f71d227df399548d708486ccfea5aca2b7bfeb6f771acfda88e773
+  metadata.gz: 9981e141e6d99f2cb9a793629a6f0c46db0df1f8df8661e2fe6bfbddabc49fb946da8b839cd0b91352c09a6816340a1078b99d7211f29f90e706517b3576c31e
+  data.tar.gz: 755251d7e8fe172c7f72e8cebb37ecfdea10937de3f39c36300651647efa8ad1423ff7fd953e65392e794a0daf227e1312a8f3cbcb50cfb4d96b37e8db155191

data/Changes.md

@@ -3,6 +3,8 @@
 Changes of norikra.
 
 ## v1
+* v1.2.2
+  * Change API for custom listner plugins (incompatible with 1.2.0, 1.2.1)
 * v1.2.1
   * Fix bug to fail to rewrite nullable fields
 * v1.2.0

data/lib/norikra/engine.rb

@@ -45,7 +45,7 @@ module Norikra
 
       @waiting_queries = []
 
-      @listeners = []
+      @listeners = {} # Listener.label => Listener
       @running_listeners = {} # query_name => listener
     end
 
@@ -373,7 +373,7 @@ module Norikra
 
     def register_query(query)
 
-      if lo_target_name = Norikra::Listener::Loopback.check(query.group)
+      if lo_target_name = Norikra::Listener::Loopback.target(query.group)
         raise "Invalid loopback target name should be checked before. THIS IS BUG." unless Norikra::Target.valid?(lo_target_name)
 
         target = Norikra::Target.new(lo_target_name)
@@ -388,7 +388,7 @@ module Norikra
         if reason = query.invalid?
           raise Norikra::ClientError, "invalid query '#{query.name}': #{reason}"
         end
-        if lo_target_name = Norikra::Listener::Loopback.check(query.group)
+        if lo_target_name = Norikra::Listener::Loopback.target(query.group)
           raise Norikra::ClientError, "loopback target '#{lo_target_name}'" unless Norikra::Target.valid?(lo_target_name)
         end
 
@@ -520,8 +520,29 @@ module Norikra
       end
     end
 
-    def load_listener(listener_klass)
-      @listeners.push(listener_klass)
+    def load_listener(klass)
+      @listeners[klass.label] = klass
+      klass
+    end
+
+    def create_listener(query)
+      opts = Norikra::Listener.parse(query.group)
+      klass = if opts && @listeners.has_key?(opts[:name])
+                @listeners[opts[:name]]
+              else
+                Norikra::Listener::MemoryPool
+              end
+      argument = opts ? opts[:argument] : nil
+      trace("selecting listeners"){ { group: query.group, listener: klass, argument: argument } }
+
+      inst = klass.new(argument, query.name, query.group)
+      inst.events_statistics = @statistics[:events]
+
+      inst.engine = self if inst.respond_to?(:engine=)
+      inst.output_pool = @output_pool if inst.respond_to?(:output_pool=)
+
+      inst.start
+      inst
     end
 
     # this method should be protected with @mutex lock
@@ -537,19 +558,7 @@ module Norikra
       statement_model = administrator.compileEPL(query.expression)
       Norikra::Query.rewrite_query(statement_model, event_type_name_map)
 
-      listener = nil
-      @listeners.each do |klass|
-        trace("checking listeners"){ {target: klass, result: klass.check(query.group)} }
-        if klass.check(query.group)
-          listener = klass.new(query.name, query.group, @statistics[:events])
-          break
-        end
-      end
-      raise "BUG: no listener is selected" unless listener
-      listener.engine = self if listener.respond_to?(:engine=)
-      listener.output_pool = @output_pool if listener.respond_to?(:output_pool=)
-      listener.start
-      @running_listeners[query.name] = listener
+      @running_listeners[query.name] = listener = create_listener(query)
 
       epl = administrator.create(statement_model)
       epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], listener

data/lib/norikra/listener.rb

@@ -16,6 +16,14 @@ require 'json'
 
 module Norikra
   module Listener
+    def self.parse(group_name)
+      if group_name && group_name =~ /^([_A-Z]+)\((.*)\)$/
+        {name: $1, argument: $2}
+      else
+        nil
+      end
+    end
+
     def self.listup
       return unless defined? Gem
 
@@ -41,7 +49,7 @@ module Norikra
       end
 
       known_consts = [:Base, :MemoryPool, :Loopback, :Stdout]
-      listeners = [Norikra::Listener::Stdout, Norikra::Listener::Loopback]
+      listeners = []
       self.constants.each do |c|
         next if known_consts.include?(c)
 
@@ -50,7 +58,9 @@ module Norikra
           listeners.push(klass)
         end
       end
-      listeners.push(Norikra::Listener::MemoryPool)
+      [Norikra::Listener::Stdout, Norikra::Listener::Loopback].each do |listener|
+        listeners.push(listener)
+      end
       listeners
     end
 
@@ -59,33 +69,37 @@ module Norikra
 
       DEFAULT_ASYNC_INTERVAL = 0.1
 
-      def self.check(group_name)
+      attr_writer :events_statistics
+      # attr_writer :engine
+      # attr_writer :output_pool
+
+      def self.label
         raise NotImplementedError
       end
 
-      def initialize(query_name, query_group, events_statistics)
+      def initialize(argument, query_name, query_group)
+        @argument = argument
         @query_name = query_name
         @query_group = query_group
-        @events_statistics = events_statistics
 
         @async_interval = DEFAULT_ASYNC_INTERVAL
 
+        @mode = if self.respond_to?(:process_sync)
+                  :sync
+                elsif self.respond_to?(:process_async)
+                  :async
+                else
+                  raise "BUG: Invalid custom listener '#{self.class.to_s}'"
+                end
+
         @thread = nil
         @events = []
         @mutex = Mutex.new
         @running = true
       end
 
-      # def engine=(engine)
-      #   @engine = engine
-      # end
-
-      # def output_pool=(output_pool)
-      #   @output_pool = output_pool
-      # end
-
       def start
-        if self.respond_to?(:process_async)
+        if @mode == :async
           trace "starting thread to process events in background", query: @query_name
           @thread = Thread.new(&method(:background))
         end
@@ -117,9 +131,6 @@ module Norikra
         end
       end
 
-      # def process_async
-      # end
-
       def shutdown
         trace "stopping listener", query: @query_name
         @running = false
@@ -149,74 +160,88 @@ module Norikra
 
       def update(new_events, old_events)
         t = Time.now.to_i
-        events = new_events.map{|e| [t, type_convert(e)]}
-        trace("updated event"){ { query: @query_name, group: @query_group, event: events } }
-        push(events)
-        @events_statistics[:output] += events.size
+        if @mode == :sync
+          news = new_events.map{|e| type_convert(e) }
+          olds = if old_events.respond_to?(:map)
+                   old_events.map{|e| type_convert(e) }
+                 else
+                   type_convert(old_events)
+                 end
+          trace("produced events"){ { listener: self.class, query: @query_name, group: @query_group, news: news, olds: olds } }
+          process_sync(news, olds)
+          @events_statistics[:output] += news.size
+        else # async
+          events = new_events.map{|e| [t, type_convert(e)]}
+          trace("pushed events"){ { listener: self.class, query: @query_name, group: @query_group, event: events } }
+          push(events)
+          @events_statistics[:output] += events.size
+        end
       end
     end
 
     class MemoryPool < Base
-      def self.check(group_name)
-        true
-      end
+      attr_writer :output_pool
 
-      def output_pool=(output_pool)
-        @output_pool = output_pool
+      def self.label
+        nil # Memory pool listener is built-in and implicit listener
       end
 
-      def update(new_events, old_events)
+      def process_sync(news, olds)
         t = Time.now.to_i
-        events = new_events.map{|e| [t, type_convert(e)]}
-        trace("updated event"){ { query: @query_name, group: @query_group, event: events } }
+        events = news.map{|e| [t, e]}
         @output_pool.push(@query_name, @query_group, events)
-        @events_statistics[:output] += events.size
       end
     end
 
     class Loopback < Base
-      def self.check(group_name)
-        group_name && group_name =~ /^LOOPBACK\((.+)\)$/ && $1
+      attr_writer :engine
+
+      def self.label
+        "LOOPBACK"
       end
 
-      def initialize(query_name, query_group, events_statistics)
-        super
-        @loopback_target = Loopback.check(query_group)
-        raise "BUG: query group is not 'LOOPBACK(...)'" unless @loopback_target
+      def self.target(query_group)
+        if query_group
+          opts = Norikra::Listener.parse(query_group)
+          if opts && opts[:name] == label()
+            opts[:argument]
+          else
+            nil
+          end
+        else
+          nil
+        end
       end
 
-      def engine=(engine)
-        @engine = engine
+      def initialize(argument, query_name, query_group)
+        super
+        if @argument.nil? || @argument.empty?
+          raise Norikra::ClientError, "LOOPBACK target name not specified"
+        end
       end
 
-      def update(new_events, old_events)
-        event_list = new_events.map{|e| type_convert(e) }
-        trace("loopback event"){ { query: @query_name, group: @query_group, event: event_list } }
-        @events_statistics[:output] += event_list.size
-        #
+      def process_sync(news, olds)
         # We does NOT convert 'container.$0' into container['field'].
         # Use escaped names like 'container__0'. That is NOT so confused.
-        @engine.send(@loopback_target, event_list)
+        trace("loopback event"){ { query: @query_name, group: @query_group, event: news } }
+        @engine.send(@argument, news)
       end
     end
 
     class Stdout < Base
-      def self.check(group_name)
-        group_name && group_name == "STDOUT()"
+      attr_accessor :stdout
+
+      def self.label
+        "STDOUT"
       end
 
-      def initialize(query_name, query_group, events_statistics)
+      def initialize(argument, query_name, query_group)
         super
-        raise "BUG: query group is not 'STDOUT()'" unless Stdout.check(query_group)
         @stdout = STDOUT
       end
 
-      def update(new_events, old_events)
-        event_list = new_events.map{|e| type_convert(e) }
-        trace("stdout event"){ { query: @query_name, event: event_list } }
-        @events_statistics[:output] += event_list.size
-
-        event_list.each do |e|
+      def process_sync(news, olds)
+        news.each do |e|
           @stdout.puts @query_name + "\t" + JSON.dump(e)
         end
       end

data/lib/norikra/listener_spec_helper.rb

@@ -0,0 +1,35 @@
+module Norikra
+  module ListenerSpecHelper
+
+    ### TODO: more util methods?
+    # utilities w/ #start, #shutdown ?
+    # utilities to produce dummy output events ?
+
+    class DummyEngine
+      attr_reader :events
+
+      def initialize
+        @events = {}
+      end
+
+      def send(target, events)
+        @events[target] ||= []
+        @events[target].push(*events)
+      end
+    end
+
+    class DummyOutputPool
+      attr_reader :pool
+
+      def initialize
+        @pool = {}
+      end
+
+      def push(query_name, query_group, events)
+        @pool[query_group] ||= {}
+        @pool[query_group][query_name] ||= []
+        @pool[query_group][query_name].push(*events)
+      end
+    end
+  end
+end

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "1.2.1"
+  VERSION = "1.2.2"
 end

data/spec/listener_spec.rb

@@ -1,45 +1,26 @@
 # -*- coding: utf-8 -*-
 require_relative './spec_helper'
-
 require 'json'
 require 'norikra/listener'
+require 'norikra/listener_spec_helper'
 
-class DummyEngine
-  attr_reader :events
-
-  def initialize
-    @events = {}
-  end
-
-  def send(target, events)
-    @events[target] ||= []
-    @events[target].push(*events)
+class DummySyncListener < Norikra::Listener::Base
+  def process_sync(news, olds)
+    # drop
   end
 end
 
-class DummyOutputPool
-  attr_reader :pool
-
-  def initialize
-    @pool = {}
-  end
-
-  def push(query_name, query_group, events)
-    @pool[query_group] ||= {}
-    @pool[query_group][query_name] ||= []
-    @pool[query_group][query_name].push(*events)
+class DummyAsyncListener < Norikra::Listener::Base
+  def process_async(news, olds)
+    # drop
   end
 end
 
 describe Norikra::Listener::Base do
-  it 'should be initialized' do
-    statistics = {output: 0}
-    expect { Norikra::Listener::Base.new('name', 'group', statistics) }.not_to raise_error
-  end
-
   describe '#type_convert' do
     statistics = {output: 0}
-    listener = Norikra::Listener::Base.new('name', 'group', statistics)
+    listener = DummySyncListener.new('', 'name', 'group')
+    listener.events_statistics = statistics
 
     it 'returns value itself for number, boolean and nil' do
       val = 10001
@@ -115,103 +96,77 @@ describe Norikra::Listener::Base do
     end
   end
 
-  ### TODO: add specs of #start, #process_async, #push and default #update
-  ### TODO: add norikra/listener_spec_helper.rb
+  ### TODO: add specs of .parse, #start, #shutdown, #push and #update
 end
 
 describe Norikra::Listener::MemoryPool do
-  describe '.check' do
-    it 'always returns true' do
-      expect(Norikra::Listener::MemoryPool.check(nil)).to be_truthy
-      expect(Norikra::Listener::MemoryPool.check('')).to be_truthy
-      expect(Norikra::Listener::MemoryPool.check('FOO')).to be_truthy
-      expect(Norikra::Listener::MemoryPool.check('FOO()')).to be_truthy
+  describe '.label' do
+    it 'returns nil' do
+      expect(Norikra::Listener::MemoryPool.label).to be_nil
     end
   end
 
-  describe '#update' do
-    statistics = {output: 0}
-    listener = Norikra::Listener::MemoryPool.new('name', 'group', statistics)
-    listener.output_pool = dummy_pool = DummyOutputPool.new
+  describe '#process_sync' do
+    listener = Norikra::Listener::MemoryPool.new(nil, 'name', 'group')
+    listener.output_pool = dummy_pool = Norikra::ListenerSpecHelper::DummyOutputPool.new
 
     it 'pushs events into pool, with current time' do
-      listener.update([{"n1" => 100, "s" => "string one"}, {"n1" => 101, "s" => "string two"}], [])
-      expect(statistics[:output]).to eql(2)
+      listener.process_sync([{"n1" => 100, "s" => "string one"}, {"n1" => 101, "s" => "string two"}], [])
       expect(dummy_pool.pool['group']['name'].size).to eql(2)
       expect(dummy_pool.pool['group']['name'][0][0]).to be_a(Fixnum)
       expect(dummy_pool.pool['group']['name'][0][1]).to eql({"n1" => 100, "s" => "string one"})
       expect(dummy_pool.pool['group']['name'][1][1]).to eql({"n1" => 101, "s" => "string two"})
 
-      listener.update([{"n1" => 102, "s" => "string three"}], [])
-      expect(statistics[:output]).to eql(3)
+      listener.process_sync([{"n1" => 102, "s" => "string three"}], [])
       expect(dummy_pool.pool['group']['name'].size).to eql(3)
     end
   end
 end
 
 describe Norikra::Listener::Loopback do
-  describe '.check' do
-    it 'returns nil for nil group' do
-      expect(Norikra::Listener::Loopback.check(nil)).to be_nil
-    end
-
-    it 'returns nil for string group name without prefix' do
-      expect(Norikra::Listener::Loopback.check('a')).to be_nil
-      expect(Norikra::Listener::Loopback.check('group1')).to be_nil
-      expect(Norikra::Listener::Loopback.check('LOOPBACK')).to be_nil
-    end
-
-    it 'returns specified string as loopback target by parentheses' do
-      expect(Norikra::Listener::Loopback.check('LOOPBACK()')).to be_nil
-      expect(Norikra::Listener::Loopback.check('LOOPBACK(a)')).to eql('a')
-      expect(Norikra::Listener::Loopback.check('LOOPBACK(loopback_target)')).to eql('loopback_target')
-      expect(Norikra::Listener::Loopback.check('LOOPBACK(target name)')).to eql('target name') # should be invalid on 'open'
+  describe '.label' do
+    it 'returns "LOOPBACK"' do
+      expect(Norikra::Listener::Loopback.label).to eql("LOOPBACK")
     end
   end
 
   it 'should be initialized' do
     statistics = {output: 0}
-    expect { Norikra::Listener::Loopback.new('name', 'LOOPBACK(target1)', statistics) }.not_to raise_error
+    inst = Norikra::Listener::Loopback.new('target1', 'name', 'LOOPBACK(target1)')
+    inst.events_statistics = statistics
   end
 
-  describe '#update' do
-    statistics = {output: 0}
-    listener = Norikra::Listener::Loopback.new('name', 'LOOPBACK(target1)', statistics)
-    listener.engine = dummy_engine = DummyEngine.new
+  describe '#process_sync' do
+    listener = Norikra::Listener::Loopback.new('target1', 'name', 'LOOPBACK(target1)')
+    listener.engine = dummy_engine = Norikra::ListenerSpecHelper::DummyEngine.new
 
     it 'sends events into engine with target name' do
-      listener.update([{"n1" => 100, "s" => "string one"}, {"n1" => 101, "s" => "string two"}], [])
-      expect(statistics[:output]).to eql(2)
+      listener.process_sync([{"n1" => 100, "s" => "string one"}, {"n1" => 101, "s" => "string two"}], [])
       expect(dummy_engine.events['target1'].size).to eql(2)
       expect(dummy_engine.events['target1'][0]).to eql({"n1" => 100, "s" => "string one"})
       expect(dummy_engine.events['target1'][1]).to eql({"n1" => 101, "s" => "string two"})
 
-      listener.update([{"n1" => 102, "s" => "string three"}], [])
-      expect(statistics[:output]).to eql(3)
+      listener.process_sync([{"n1" => 102, "s" => "string three"}], [])
       expect(dummy_engine.events['target1'].size).to eql(3)
     end
   end
 end
 
 describe Norikra::Listener::Stdout do
-  describe '.check' do
-    it 'returns true if group name is "STDOUT()"' do
-      expect(Norikra::Listener::Stdout.check(nil)).to be_falsy
-      expect(Norikra::Listener::Stdout.check("")).to be_falsy
-      expect(Norikra::Listener::Stdout.check("foo")).to be_falsy
-      expect(Norikra::Listener::Stdout.check("STDOUT")).to be_falsy
-      expect(Norikra::Listener::Stdout.check("STDOUT()")).to be_truthy
+  describe '.label' do
+    it 'returns "STDOUT"' do
+      expect(Norikra::Listener::Stdout.label).to eql("STDOUT")
     end
   end
 
   it 'should be initialized' do
     statistics = {output: 0}
-    expect { Norikra::Listener::Stdout.new('name', 'STDOUT()', statistics) }.not_to raise_error
+    inst = Norikra::Listener::Stdout.new('', 'name', 'STDOUT()')
+    inst.events_statistics = statistics
   end
 
-  describe '#update' do
-    statistics = {output: 0}
-    listener = Norikra::Listener::Stdout.new('name', 'STDOUT()', statistics)
+  describe '#process_sync' do
+    listener = Norikra::Listener::Stdout.new('', 'name', 'STDOUT()')
     dummyio = StringIO.new
     listener.instance_eval{ @stdout = dummyio }
 
@@ -219,12 +174,10 @@ describe Norikra::Listener::Stdout do
       dummyio.truncate(0)
 
       events1 = [{"n1" => 100, "s" => "string one"}, {"n1" => 101, "s" => "string two"}]
-      listener.update(events1, [])
-      expect(statistics[:output]).to eql(2)
+      listener.process_sync(events1, [])
 
       events2 = [{"n1" => 102, "s" => "string three"}]
-      listener.update(events2, [])
-      expect(statistics[:output]).to eql(3)
+      listener.process_sync(events2, [])
 
       results = []
       dummyio.string.split("\n").each do |line|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: norikra
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: java
 authors:
 - TAGOMORI Satoshi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-16 00:00:00.000000000 Z
+date: 2015-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mizuno
@@ -218,6 +218,7 @@ files:
 - lib/norikra/field.rb
 - lib/norikra/fieldset.rb
 - lib/norikra/listener.rb
+- lib/norikra/listener_spec_helper.rb
 - lib/norikra/logger.rb
 - lib/norikra/logger_mizuno_patch.rb
 - lib/norikra/output_pool.rb