norikra 1.1.2-java → 1.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90c6be383b83590b832fb6271a3642a42295ae6f
-  data.tar.gz: 23555f2d09692459ef6fa17b85a47af61129e769
+  metadata.gz: e7e969ece3deb5788c7e7e76c9d6cc4961b4307c
+  data.tar.gz: bbfdc5a4056935c355082877e6ef347b7ded54ec
 SHA512:
-  metadata.gz: b93442c3b3e202a880fa75f90065770b1b44b210789684b16889b24b7c10f91144488e6f2564b25e78ae0850c2a8f24c711e83c0c8b8d8a90608ee5c2e0b249e
-  data.tar.gz: b34e828cd14c7c4fb00a2d83c17131590be520e15559c9caef5e429d46a92b311e027a12a283721de81000efbb12d79b752df3e1559a4890bef4cabf93788b2d
+  metadata.gz: 4ea0f4f04ba875e2432db51a179d75e3ae18e3a05fc03e945b14fdce187b61da969cd334e11d67aaf8a49b0a032b389f2f57a6c385c98f284b4ecd5bd3156156
+  data.tar.gz: 0add6a6f2539d188f2de82856f1d8cc222f9e6dd60b31b96acbc10167f047d9395c71273dde65a97d23b9b0838d30e0c7661b899d226077def93301e272515da

data/.ruby-version

@@ -1 +1 @@
-jruby-1.7.16.1
+jruby-1.7.18

data/Changes.md

@@ -3,6 +3,10 @@
 Changes of norikra.
 
 ## v1
+* v1.2.0
+  * Add `NULLABLE(...)` field to query NULL explicitly
+  * Pluggable listener and user defined listener gems
+  * Fix bug to ignore fields in Esper built-in function
 * v1.1.2
   * Enable `-javaagent` option
   * Fix default/pre-set threads

data/Gemfile

@@ -2,3 +2,7 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in norikra.gemspec
 gemspec
+
+# to check plugin behaviors
+gem "norikra-udf-mock"
+gem "norikra-listener-mock"

data/README.md

@@ -38,12 +38,6 @@ Fix code and tests:
 1. fix code in `lib`
 1. run `bundle exec rake`
 
-Run tests faster than 2 or more times:
-
-1. execute `spork`
-1. execute `script/spec_server_pry` in another terminal
-1. run `rspec` in pry console (executed fastly after second times)
-
 Execute norikra server with target/query continuation:
 
 1. `bundle exec rake devserver`
@@ -70,6 +64,9 @@ See: http://norikra.github.io/
 
 ## Changes
 
+* v1.2
+  * `NULLABLE()` fields
+  * Pluggable listeners
 * v1.1
   * Suspend/Resume queries
 * v1.0

data/lib/norikra/cli.rb

@@ -11,70 +11,70 @@ module Norikra
     def self.register_common_start_options(klass)
       klass.module_exec {
         ### Server options
-        option :host, :type => :string, :default => nil, :aliases => "-H", :desc => 'host address that server listen [0.0.0.0]'
-        option :port, :type => :numeric, :default => nil, :aliases => "-P", :desc => 'port that server uses [26571]'
-        option :'ui-port', :type => :numeric, :default => nil, :desc => 'port that Web UI server uses [26578]'
+        option :host, type: :string, default: nil, aliases: "-H", desc: 'host address that server listen [0.0.0.0]'
+        option :port, type: :numeric, default: nil, aliases: "-P", desc: 'port that server uses [26571]'
+        option :'ui-port', type: :numeric, default: nil, desc: 'port that Web UI server uses [26578]'
 
-        option :stats, :type => :string, :default => nil, :aliases => "-s", \
-                       :desc => 'status file path to load/dump targets and queries [none]'
-        option :'stats-secondary', :type => :string, :default => nil, \
-                       :desc => 'status file secondary path, to dump stats with date/time, like "stats.%Y%m%d.json" [none]'
-        option :'suppress-dump-stat', :type => :boolean, :default => false, \
-                       :desc => 'specify not to update stat file with updated targets/queries on runtime [false]'
-        option :'dump-stat-interval', :type => :numeric, :default => nil, \
-                       :desc => 'interval(seconds) of status file dumps on runtime [none (on shutdown only)]'
+        option :stats, type: :string, default: nil, aliases: "-s", \
+                       desc: 'status file path to load/dump targets and queries [none]'
+        option :'stats-secondary', type: :string, default: nil, \
+                       desc: 'status file secondary path, to dump stats with date/time, like "stats.%Y%m%d.json" [none]'
+        option :'suppress-dump-stat', type: :boolean, default: false, \
+                       desc: 'specify not to update stat file with updated targets/queries on runtime [false]'
+        option :'dump-stat-interval', type: :numeric, default: nil, \
+                       desc: 'interval(seconds) of status file dumps on runtime [none (on shutdown only)]'
 
         ### Daemonize options
-        option :daemonize, :type => :boolean, :default => false, :aliases => "-d", \
-                           :desc => 'daemonize Norikra server [false (foreground)]'
-        option :pidfile, :type => :string, :default => DEFAULT_PID_PATH, :aliases => "-p", \
-                         :desc => "pidfile path when daemonized [#{DEFAULT_PID_PATH}]"
-        option :outfile, :type => :string, :default => nil, \
-                         :desc => "stdout redirect file when daemonized [${logdir}/norikra.out]"
+        option :daemonize, type: :boolean, default: false, aliases: "-d", \
+                           desc: 'daemonize Norikra server [false (foreground)]'
+        option :pidfile, type: :string, default: DEFAULT_PID_PATH, aliases: "-p", \
+                         desc: "pidfile path when daemonized [#{DEFAULT_PID_PATH}]"
+        option :outfile, type: :string, default: nil, \
+                         desc: "stdout redirect file when daemonized [${logdir}/norikra.out]"
 
         ### JVM options
-        option :'bare-jvm', :type => :boolean, :default => false, :desc => "use JVM without any recommended options"
-        option :'gc-log', :type => :string, :default => nil, :desc => "output gc logs on specified file path"
+        option :'bare-jvm', type: :boolean, default: false, desc: "use JVM without any recommended options"
+        option :'gc-log', type: :string, default: nil, desc: "output gc logs on specified file path"
 
         ### Performance options
         # performance predefined configuration sets
-        option :micro, :type => :boolean, :default => false, \
-               :desc => 'development or testing (inbound:0, outbound:0, route:0, timer:0, rpc:2)'
-        option :small, :type => :boolean, :default => false, \
-               :desc => 'virtual or small scale servers (inbound:1, outbount:1, route:1, timer:1, rpc:2)'
-        option :middle, :type => :boolean, :default => false, \
-               :desc => 'rackmount servers (inbound:4, outbound:2, route:2, timer:2, rpc:4)'
-        option :large, :type => :boolean, :default => false, \
-               :desc => 'high performance servers (inbound: 6, outbound: 6, route:4, timer:4, rpc: 8)'
+        option :micro, type: :boolean, default: false, \
+               desc: 'development or testing (inbound:0, outbound:0, route:0, timer:0, rpc:2)'
+        option :small, type: :boolean, default: false, \
+               desc: 'virtual or small scale servers (inbound:1, outbount:1, route:1, timer:1, rpc:2)'
+        option :middle, type: :boolean, default: false, \
+               desc: 'rackmount servers (inbound:4, outbound:2, route:2, timer:2, rpc:4)'
+        option :large, type: :boolean, default: false, \
+               desc: 'high performance servers (inbound: 6, outbound: 6, route:4, timer:4, rpc: 8)'
         # Esper
-        option :'inbound-threads',  :type => :numeric, :default => nil, :desc => 'number of threads for inbound data'
-        option :'outbound-threads', :type => :numeric, :default => nil, :desc => 'number of threads for outbound data'
-        option :'route-threads',    :type => :numeric, :default => nil, :desc => 'number of threads for events routing for query execution'
-        option :'timer-threads',    :type => :numeric, :default => nil, :desc => 'number of threads for internal timers for query execution'
+        option :'inbound-threads',  type: :numeric, default: nil, desc: 'number of threads for inbound data'
+        option :'outbound-threads', type: :numeric, default: nil, desc: 'number of threads for outbound data'
+        option :'route-threads',    type: :numeric, default: nil, desc: 'number of threads for events routing for query execution'
+        option :'timer-threads',    type: :numeric, default: nil, desc: 'number of threads for internal timers for query execution'
         ### about capacity options of esper's capacity-bound queue processing, see Esper's thread reference.
         # http://esper.codehaus.org/esper-4.10.0/doc/reference/en-US/html/configuration.html#config-engine-threading-advanced
         # default nil: unbound queueing
-        option :'inbound-thread-capacity',  :type => :numeric, :default => nil
-        option :'outbound-thread-capacity', :type => :numeric, :default => nil
-        option :'route-thread-capacity',    :type => :numeric, :default => nil
-        option :'timer-thread-capacity',    :type => :numeric, :default => nil
+        option :'inbound-thread-capacity',  type: :numeric, default: nil
+        option :'outbound-thread-capacity', type: :numeric, default: nil
+        option :'route-thread-capacity',    type: :numeric, default: nil
+        option :'timer-thread-capacity',    type: :numeric, default: nil
         # Jetty
-        option :'rpc-threads', :type => :numeric, :default => nil, :desc => 'number of threads for rpc handlers'
-        option :'web-threads', :type => :numeric, :default => nil, :desc => 'number of threads for WebUI handlers'
+        option :'rpc-threads', type: :numeric, default: nil, desc: 'number of threads for rpc handlers'
+        option :'web-threads', type: :numeric, default: nil, desc: 'number of threads for WebUI handlers'
 
         ### Logging options
-        option :logdir, :type => :string, :default => nil, :aliases => "-l", \
-               :desc => "directory path of logfiles when daemonized [nil (console for foreground)]"
-        option :'log-filesize', :type => :string, :default => nil, :desc => 'log rotation size [10MB]'
-        option :'log-backups' , :type => :numeric, :default => nil, :desc => 'log rotation backups [10]'
-        option :'log-buffer-lines', :type => :numeric, :default => nil, :desc => 'log lines to fetch from API [1000]'
-        option :'log4j-properties-path', :type => :string, :default => nil, :desc => 'path to log4j.properties. ignore other log* options when this option is present'
+        option :logdir, type: :string, default: nil, aliases: "-l", \
+               desc: "directory path of logfiles when daemonized [nil (console for foreground)]"
+        option :'log-filesize', type: :string, default: nil, desc: 'log rotation size [10MB]'
+        option :'log-backups' , type: :numeric, default: nil, desc: 'log rotation backups [10]'
+        option :'log-buffer-lines', type: :numeric, default: nil, desc: 'log lines to fetch from API [1000]'
+        option :'log4j-properties-path', type: :string, default: nil, desc: 'path to log4j.properties. ignore other log* options when this option is present'
 
         ### Loglevel options
-        option :'more-quiet',   :type => :boolean, :default => false,                   :desc => 'set loglevel as ERROR'
-        option :quiet,          :type => :boolean, :default => false, :aliases => "-q", :desc => 'set loglevel as WARN'
-        option :verbose,        :type => :boolean, :default => false, :aliases => "-v", :desc => 'set loglevel as DEBUG'
-        option :'more-verbose', :type => :boolean, :default => false,                   :desc => 'set loglevel as TRACE'
+        option :'more-quiet',   type: :boolean, default: false,                desc: 'set loglevel as ERROR'
+        option :quiet,          type: :boolean, default: false, aliases: "-q", desc: 'set loglevel as WARN'
+        option :verbose,        type: :boolean, default: false, aliases: "-v", desc: 'set loglevel as DEBUG'
+        option :'more-verbose', type: :boolean, default: false,                desc: 'set loglevel as TRACE'
       }
     end
   end
@@ -117,7 +117,7 @@ module Norikra
     ### 'start' and 'serverprocess' have almost same option set (for parse/help)
     ### DIFF: jvm options (-X)
     Norikra::CLIUtil.register_common_start_options(self)
-    option :help, :type => :boolean, :default => false, :aliases => "-h", :desc => "show this message"
+    option :help, type: :boolean, default: false, aliases: "-h", desc: "show this message"
     desc "start [-Xxxx] [other options]", "Start Norikra server process"
     def start(*optargs)
       if options[:help]
@@ -168,7 +168,7 @@ module Norikra
         File.open(outfile, 'w'){|file| file.write 'write test on parent process'}
 
         pidfile = File.open(options[:pidfile], 'w')
-        pid = spawn(jruby_path, *args, :pgroup => 0)
+        pid = spawn(jruby_path, *args, pgroup: 0)
         pidfile.write(pid.to_s)
         pidfile.close
         waiting_child = true
@@ -183,12 +183,12 @@ module Norikra
     end
 
     Norikra::CLIUtil.register_common_start_options(self)
-    desc "serverproc", "execute server process actually (don't execute this subcommand directly)", :hide => true
+    desc "serverproc", "execute server process actually (don't execute this subcommand directly)", hide: true
     def serverproc
       conf = {}
 
       if options[:daemonize]
-        conf[:daemonize] = {:outfile => options[:outfile]}
+        conf[:daemonize] = {outfile: options[:outfile]}
       end
 
       ### stat file
@@ -247,9 +247,9 @@ module Norikra
       server.shutdown
     end
 
-    option :pidfile, :type => :string, :default => DEFAULT_PID_PATH, :aliases => "-p", \
-                     :desc => "pidfile path when daemonized [#{DEFAULT_PID_PATH}]"
-    option :timeout, :type => :numeric, :default => 5, :desc => "timeout seconds to wait process exit [5]"
+    option :pidfile, type: :string, default: DEFAULT_PID_PATH, aliases: "-p", \
+                     desc: "pidfile path when daemonized [#{DEFAULT_PID_PATH}]"
+    option :timeout, type: :numeric, default: 5, desc: "timeout seconds to wait process exit [5]"
     desc "stop [options]", "stop daemonized Norikra server"
     def stop
       unless test(?r,options[:pidfile])

data/lib/norikra/engine.rb

@@ -44,6 +44,9 @@ module Norikra
       @suspended_queries = []
 
       @waiting_queries = []
+
+      @listeners = []
+      @running_listeners = {} # query_name => listener
     end
 
     def statistics
@@ -109,7 +112,7 @@ module Norikra
 
           threads = t[sym][:threads].to_i
           capacity = t[sym][:capacity].to_i
-          info "Engine #{sym} thread pool enabling", :threads => threads, :capacity => (capacity == 0 ? 'default' : capacity)
+          info "Engine #{sym} thread pool enabling", threads: threads, capacity: (capacity == 0 ? 'default' : capacity)
 
           cam = camelize(sym)
           threading.send("setThreadPool#{cam}".to_sym, true)
@@ -138,7 +141,7 @@ module Norikra
     def open(target_name, fields=nil, auto_field=true)
       # fields nil || [] => lazy
       # fields {'fieldname' => 'type'} : type 'string', 'boolean', 'int', 'long', 'float', 'double'
-      info "opening target", :target => target_name, :fields => fields, :auto_field => auto_field
+      info "opening target", target: target_name, fields: fields, auto_field: auto_field
       raise Norikra::ArgumentError, "invalid target name" unless Norikra::Target.valid?(target_name)
       target = Norikra::Target.new(target_name, fields, auto_field)
       return false if @targets.include?(target)
@@ -146,7 +149,7 @@ module Norikra
     end
 
     def close(target_name)
-      info "closing target", :target => target_name
+      info "closing target", target: target_name
       targets = @targets.select{|t| t.name == target_name}
       return false if targets.size != 1
       target = targets.first
@@ -157,7 +160,7 @@ module Norikra
     end
 
     def modify(target_name, auto_field)
-      info "modify target", :target => target_name, :auto_field => auto_field
+      info "modify target", target: target_name, auto_field: auto_field
       targets = @targets.select{|t| t.name == target_name}
       if targets.size != 1
         raise Norikra::ArgumentError, "target name '#{target_name}' not found"
@@ -171,10 +174,12 @@ module Norikra
     end
 
     def register(query)
-      info "registering query", :name => query.name, :targets => query.targets, :expression => query.expression
+      info "registering query", name: query.name, targets: query.targets, expression: query.expression
       raise Norikra::ClientError, "query name '#{query.name}' already exists" if @queries.select{|q| q.name == query.name }.size > 0
       raise Norikra::ClientError, "query name '#{query.name}' already exists in suspended" if @suspended_queries.select{|q| q.name == query.name }.size > 0
-      raise Norikra::ClientError, "query '#{query.name}' is invalid query for Norikra" if query.invalid?
+      if reason = query.invalid?
+        raise Norikra::ClientError, "invalid query '#{query.name}': #{reason}"
+      end
 
       query.targets.each do |target_name|
         open(target_name) unless @targets.any?{|t| t.name == target_name}
@@ -183,7 +188,7 @@ module Norikra
     end
 
     def deregister(query_name)
-      info "de-registering query", :name => query_name
+      info "de-registering query", name: query_name
       queries = @queries.select{|q| q.name == query_name }
       s_queries = @suspended_queries.select{|q| q.name == query_name }
 
@@ -242,12 +247,12 @@ module Norikra
     end
 
     def send(target_name, events)
-      trace "send messages", :target => target_name, :events => events
+      trace "send messages", target: target_name, events: events
 
       @statistics[:events][:input] += events.size
 
       unless @targets.any?{|t| t.name == target_name} # discard events for target not registered
-        trace "messages skipped for non-opened target", :target => target_name
+        trace "messages skipped for non-opened target", target: target_name
         return
       end
       return if events.size < 1
@@ -255,19 +260,19 @@ module Norikra
       target = @targets.select{|t| t.name == target_name}.first
 
       if @typedef_manager.lazy?(target.name)
-        info "opening lazy target", :target => target
+        info "opening lazy target", target: target
 
         first_event = event_filter(events.first)
         if first_event.nil? # non-hash object
           raise Norikra::ClientError, "Input data must be JSON object"
         end
-        debug "generating base fieldset from event", :target => target.name, :event => first_event
+        debug "generating base fieldset from event", target: target.name, event: first_event
         base_fieldset = @typedef_manager.generate_base_fieldset(target.name, first_event)
 
-        debug "registering base fieldset", :target => target.name, :base => base_fieldset
+        debug "registering base fieldset", target: target.name, base: base_fieldset
         register_base_fieldset(target.name, base_fieldset)
 
-        info "target successfully opened with fieldset", :target => target, :base => base_fieldset
+        info "target successfully opened with fieldset", target: target, base: base_fieldset
       end
 
       registered_data_fieldset = @registered_fieldsets[target_name][:data]
@@ -282,18 +287,19 @@ module Norikra
 
         unless registered_data_fieldset[fieldset.summary]
           # register waiting queries including this fieldset, and this fieldset itself
-          debug "registering unknown fieldset", :target => target_name, :fieldset => fieldset
+          debug "registering unknown fieldset", target: target_name, fieldset: fieldset
           register_fieldset(target_name, fieldset)
           debug "successfully registered"
 
           # fieldset should be refined, when waiting_queries rewrite inheritance structure and data fieldset be renewed.
           fieldset = @typedef_manager.refer(target_name, event, strict_refer)
+          debug "re-referred data fieldset", target: target_name, fieldset: fieldset
         end
 
-        trace "calling sendEvent with bound fieldset (w/ valid event_type_name)", :target => target_name, :event => event
-        trace "This is assert for valid event_type_name", :event_type_name => fieldset.event_type_name
+        trace "calling sendEvent with bound fieldset (w/ valid event_type_name)", target: target_name, event: event
+        trace("This is assert for valid event_type_name"){ { event_type_name: fieldset.event_type_name } }
         formed = fieldset.format(event)
-        trace "sendEvent", :data => formed
+        trace "sendEvent", data: formed
         @runtime.sendEvent(formed.to_java, fieldset.event_type_name)
       end
       target.update!
@@ -301,8 +307,13 @@ module Norikra
       nil
     end
 
-    def load(plugin_klass)
-      load_udf(plugin_klass)
+    def load(type, plugin_klass)
+      case type
+      when :udf then load_udf(plugin_klass)
+      when :listener then load_listener(plugin_klass)
+      else
+        raise "BUG: unknown plugin type: #{type}"
+      end
     end
 
     private
@@ -312,7 +323,7 @@ module Norikra
         return false if @targets.include?(target)
 
         @typedef_manager.add_target(target.name, target.fields)
-        @registered_fieldsets[target.name] = {:base => {}, :query => {}, :data => {}}
+        @registered_fieldsets[target.name] = {base: {}, query: {}, data: {}}
 
         unless @typedef_manager.lazy?(target.name)
           base_fieldset = @typedef_manager.base_fieldset(target.name)
@@ -352,7 +363,7 @@ module Norikra
     def update_inherits_graph(target_name, query_fieldset)
       # replace registered data fieldsets with new fieldset inherits this query fieldset
       @typedef_manager.supersets(target_name, query_fieldset).each do |set|
-        rebound = set.rebind(true) # update event_type_name with new inheritations
+        rebound = set.rebind(true, query_fieldset) # update event_type_name with new inheritations & nullable fields
 
         register_fieldset_actually(target_name, rebound, :data, true) # replacing on esper engine
         @typedef_manager.replace_fieldset(target_name, set, rebound)
@@ -362,26 +373,28 @@ module Norikra
 
     def register_query(query)
 
-      if lo_target_name = Norikra::Query.loopback(query.group)
+      if lo_target_name = Norikra::Listener::Loopback.check(query.group)
         raise "Invalid loopback target name should be checked before. THIS IS BUG." unless Norikra::Target.valid?(lo_target_name)
 
         target = Norikra::Target.new(lo_target_name)
         unless @targets.include?(target)
-          info "opening loopback target", :target => lo_target_name
+          info "opening loopback target", target: lo_target_name
           open_target(target)
         end
       end
 
       @mutex.synchronize do
         raise Norikra::ClientError, "query '#{query.name}' already exists" unless @queries.select{|q| q.name == query.name }.empty?
-        raise Norikra::ClientError, "query '#{query.name}' is invalid query for Norikra" if query.invalid?
-        if lo_target_name = Norikra::Query.loopback(query.group)
+        if reason = query.invalid?
+          raise Norikra::ClientError, "invalid query '#{query.name}': #{reason}"
+        end
+        if lo_target_name = Norikra::Listener::Loopback.check(query.group)
           raise Norikra::ClientError, "loopback target '#{lo_target_name}'" unless Norikra::Target.valid?(lo_target_name)
         end
 
         unless @typedef_manager.ready?(query)
           @waiting_queries.push(query)
-          trace "waiting query fields", :targets => query.targets, :fields => query.targets.map{|t| query.fields(t)}
+          trace("waiting query fields"){ { targets: query.targets, fields: query.targets.map{|t| query.fields(t)} } }
           @typedef_manager.register_waiting_fields(query)
           @queries.push(query)
           return
@@ -389,7 +402,9 @@ module Norikra
 
         mapping = @typedef_manager.generate_fieldset_mapping(query)
         mapping.each do |target_name, query_fieldset|
+          trace "binding query fieldset", fieldset: query_fieldset
           @typedef_manager.bind_fieldset(target_name, :query, query_fieldset)
+          trace "registering query fieldset", fieldset: query_fieldset
           register_fieldset_actually(target_name, query_fieldset, :query)
           update_inherits_graph(target_name, query_fieldset)
           query.fieldsets[target_name] = query_fieldset
@@ -453,7 +468,9 @@ module Norikra
       ready.each do |query|
         mapping = @typedef_manager.generate_fieldset_mapping(query)
         mapping.each do |target_name, query_fieldset|
+          trace "binding query fieldset for waiting query", query: query, target: target_name, fieldset: query_fieldset
           @typedef_manager.bind_fieldset(target_name, :query, query_fieldset)
+          trace "registering query fieldset", target: target_name, fieldset: query_fieldset
           register_fieldset_actually(target_name, query_fieldset, :query)
           update_inherits_graph(target_name, query_fieldset)
           query.fieldsets[target_name] = query_fieldset
@@ -464,12 +481,30 @@ module Norikra
 
     def register_fieldset(target_name, fieldset)
       @mutex.synchronize do
+        trace "binding data fieldset", fieldset: fieldset # to prepare pickup waiting queries by newly comming fields
         @typedef_manager.bind_fieldset(target_name, :data, fieldset)
 
         if @waiting_queries.size > 0
           register_waiting_queries
         end
-        debug "registering data fieldset", :target => target_name, :fields => fieldset.fields
+
+        diff_nullable_fields = []
+
+        @typedef_manager.subsets(target_name, fieldset).each do |query_fieldset|
+          next unless query_fieldset.level == :query
+          # fill nullable fields of all required query fieldsets
+          diff_nullable_fields += fieldset.nullable_diff(query_fieldset)
+        end
+
+        unless diff_nullable_fields.empty?
+          trace "query fieldset has nullable diff", diff: diff_nullable_fields
+          fieldset.update(diff_nullable_fields, true) # nullable fields are always optional
+          trace "rebinding data fieldset w/ nullable fields", fieldset: fieldset
+          rebound = fieldset.rebind(false) # type_name is not required to be updated because it is not registered yet
+          @typedef_manager.replace_fieldset(target_name, fieldset, rebound)
+          fieldset = rebound
+        end
+        debug "registering data fieldset", target: target_name, fieldset: fieldset
         register_fieldset_actually(target_name, fieldset, :data)
       end
     end
@@ -485,6 +520,10 @@ module Norikra
       end
     end
 
+    def load_listener(listener_klass)
+      @listeners.push(listener_klass)
+    end
+
     # this method should be protected with @mutex lock
     def register_query_actually(query, mapping)
       # 'mapping' argument is {target => fieldset}
@@ -498,13 +537,19 @@ module Norikra
       statement_model = administrator.compileEPL(query.expression)
       Norikra::Query.rewrite_query(statement_model, event_type_name_map)
 
-      listener = if Norikra::Query.loopback(query.group)
-                   Norikra::LoopbackListener.new(self, query.name, query.group, @statistics[:events])
-                 elsif Norikra::Query.stdout?(query.group)
-                   Norikra::StdoutListener.new(self, query.name, query.group, @statistics[:events])
-                 else
-                   Norikra::Listener.new(query.name, query.group, @output_pool, @statistics[:events])
-                 end
+      listener = nil
+      @listeners.each do |klass|
+        trace("checking listeners"){ {target: klass, result: klass.check(query.group)} }
+        if klass.check(query.group)
+          listener = klass.new(query.name, query.group, @statistics[:events])
+          break
+        end
+      end
+      raise "BUG: no listener is selected" unless listener
+      listener.engine = self if listener.respond_to?(:engine=)
+      listener.output_pool = @output_pool if listener.respond_to?(:output_pool=)
+      listener.start
+      @running_listeners[query.name] = listener
 
       epl = administrator.create(statement_model)
       epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], listener
@@ -522,9 +567,11 @@ module Norikra
       epl = administrator.getStatement(query.statement_name)
       return unless epl
 
-      @output_pool.remove(query.name, query.group)
       epl.stop unless epl.isStopped
       epl.destroy unless epl.isDestroyed
+      listener = @running_listeners.delete(query.name)
+      listener.shutdown
+      @output_pool.remove(query.name, query.group)
     end
 
     # this method should be protected with @mutex lock
@@ -537,15 +584,15 @@ module Norikra
       #   .addEventType("AccountUpdate", accountUpdateDef, new String[] {"BaseUpdate"});
       case level
       when :base
-        debug "add event type", :target => target_name, :level => 'base', :event_type => fieldset.event_type_name
+        debug("add event type"){ { target: target_name, level: 'base', event_type: fieldset.event_type_name } }
         @config.addEventType(fieldset.event_type_name, fieldset.definition)
       when :query
         base_name = @typedef_manager.base_fieldset(target_name).event_type_name
-        debug "add event type", :target => target_name, :level => 'query', :event_type => fieldset.event_type_name, :base => base_name
+        debug("add event type"){ { target: target_name, level: 'query', event_type: fieldset.event_type_name, base: base_name } }
         @config.addEventType(fieldset.event_type_name, fieldset.definition, [base_name].to_java(:string))
-      else
+      else # :data
         subset_names = @typedef_manager.subsets(target_name, fieldset).map(&:event_type_name)
-        debug "add event type", :target => target_name, :level => 'data', :event_type => fieldset.event_type_name, :inherit => subset_names
+        debug("add event type"){ { target: target_name, level: 'data', event_type: fieldset.event_type_name, inherit: subset_names } }
         @config.addEventType(fieldset.event_type_name, fieldset.definition, subset_names.to_java(:string))
 
         @registered_fieldsets[target_name][level][fieldset.summary] = fieldset
@@ -559,7 +606,7 @@ module Norikra
 
       # DON'T check @registered_fieldsets[target_name][level][fieldset.summary]
       # removed fieldset should be already replaced with register_fieldset_actually w/ replace flag
-      debug "remove event type", :target => target_name, :event_type => event_type_name
+      debug "remove event type", target: target_name, event_type: event_type_name
       @config.removeEventType(event_type_name, true)
     end
 
@@ -567,7 +614,7 @@ module Norikra
     FILTER_OPTIMIZABLE_ENUM = com.espertech.esper.client.ConfigurationPlugInSingleRowFunction::FilterOptimizable
 
     def load_udf_actually(udf)
-      debug "importing class into config object", :name => udf.class.to_s
+      debug "importing class into config object", name: udf.class.to_s
 
       functionName, className, methodName = udf.definition
 
@@ -575,17 +622,17 @@ module Norikra
       filterOptimizable = udf.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
       rethrowExceptions = udf.rethrow_exceptions
 
-      debug "adding SingleRowFunction", :function => functionName, :javaClass => className, :javaMethod => methodName
+      debug "adding SingleRowFunction", function: functionName, javaClass: className, javaMethod: methodName
       @config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
       functionName
     end
 
     def load_udf_aggregation_actually(udf)
-      debug "importing class into config object", :name => udf.class.to_s
+      debug "importing class into config object", name: udf.class.to_s
 
       functionName, factoryClassName = udf.definition
 
-      debug "adding AggregationSingleFactory", :function => functionName, :javaClass => factoryClassName
+      debug "adding AggregationSingleFactory", function: functionName, javaClass: factoryClassName
       @config.addPlugInAggregationFunctionFactory(functionName, factoryClassName)
       functionName
     end