norikra 1.1.2-java → 1.2.0-java

data/lib/norikra/field.rb

@@ -35,7 +35,7 @@ module Norikra
     ### expected java.lang.Class or java.util.Map or the name of a previously-declared Map or ObjectArray type
     #### Correct type name is 'int'. see and run 'junks/esper-test.rb'
 
-    attr_accessor :name, :type, :esper_type, :optional, :escaped_name, :container_name, :container_type
+    attr_accessor :name, :type, :esper_type, :optional, :nullable, :escaped_name, :container_name, :container_type
 
     def self.esper_type_map(type)
       case type.to_s.downcase
@@ -75,10 +75,11 @@ module Norikra
       end
     end
 
-    def initialize(name, type, optional=nil)
+    def initialize(name, type, optional=nil, nullable=false)
       @name = name.to_s
       @type = self.class.valid_type(type)
       @optional = optional
+      @nullable = nullable
 
       @escaped_name = self.class.escape_name(@name)
 
@@ -154,24 +155,28 @@ module Norikra
 
     def to_hash(sym=false)
       if sym
-        {name: @name, type: @type, optional: @optional}
+        {name: @name, type: @type, optional: @optional, nullable: @nullable}
       else
-        {'name' => @name, 'type' => @type, 'optional' => @optional}
+        {'name' => @name, 'type' => @type, 'optional' => @optional, 'nullable' => @nullable}
       end
     end
 
     def dup(optional=nil)
-      self.class.new(@name, @type, optional.nil? ? @optional : optional)
+      self.class.new(@name, @type, optional.nil? ? @optional : optional, @nullable)
     end
 
     def ==(other)
-      self.name == other.name && self.type == other.type && self.optional == other.optional
+      self.name == other.name && self.type == other.type && self.optional == other.optional && self.nullable == other.nullable
     end
 
     def optional? # used outside of FieldSet
       @optional
     end
 
+    def nullable?
+      @nullable
+    end
+
     # def value(event) # by define_value_accessor
 
     def format(value, element_path=nil) #element_path ex: 'fname.fchild', 'fname.$0', 'f.fchild.$2'
@@ -228,7 +233,7 @@ module Norikra
       self.instance_eval do
         def safe_fetch(v, accessor)
           unless accessor.is_a?(String) || accessor.is_a?(Fixnum)
-            raise ArgumentError, "container_accessor must be a String or Interger, but #{accessor.class.to_s}"
+            raise ::ArgumentError, "container_accessor must be a String or Interger, but #{accessor.class.to_s}"
           end
           if v.is_a?(Hash)
             # v[accessor] || v[accessor.to_s]

data/lib/norikra/fieldset.rb

@@ -17,11 +17,11 @@ module Norikra
         elsif data.is_a?(Hash)
           type = data[:type].to_s
           optional = data.has_key?(:optional) ? data[:optional] : default_optional
-          @fields[key.to_s] = Field.new(key.to_s, type, optional)
+          @fields[key.to_s] = Field.new(key.to_s, type, optional, !!data[:nullable])
         elsif data.is_a?(String) || data.is_a?(Symbol)
           @fields[key.to_s] = Field.new(key.to_s, data.to_s, default_optional)
         else
-          raise ArgumentError, "FieldSet.new argument class unknown: #{fields.class}"
+          raise ::ArgumentError, "FieldSet.new argument class unknown: #{fields.class}"
         end
       end
       self.update_summary
@@ -34,7 +34,7 @@ module Norikra
     end
 
     def dup
-      fields = Hash[@fields.map{|key,field| [key, {:type => field.type, :optional => field.optional}]}]
+      fields = Hash[@fields.map{|key,field| [key, {type: field.type, optional: field.optional, nullable: field.nullable?}]}]
       self.class.new(fields, nil, @rebounds, @query_unique_keys)
     end
 
@@ -75,13 +75,16 @@ module Norikra
       dig.call(container)
     end
 
+    #### field_names_key is for lookup FieldSet from event data
+    # field_names_key: a,b,c,d
+    ### comma separated field names, which contains valid values (null fields are not included)
     def self.field_names_key(data, fieldset=nil, strict=false, additional_fields=[])
       if !fieldset && strict
         raise RuntimeError, "strict(true) cannot be specified with fieldset=nil"
       end
 
       unless fieldset
-        return data.keys.sort.join(',')
+        return data.reject{|k,v| v.respond_to?(:nullable?) && v.nullable? }.keys.sort.join(',')
       end
 
       keys = []
@@ -113,8 +116,9 @@ module Norikra
       self.class.field_names_key(@fields)
     end
 
+    # summary is for checks whether FieldSet is already registered or not
     def update_summary
-      @summary = @fields.keys.sort.map{|k| @fields[k].escaped_name + ':' + @fields[k].type}.join(',')
+      @summary = @fields.keys.sort.map{|k| f = @fields[k]; "#{f.escaped_name}:#{f.type}" + (f.nullable? ? ':nullable' : '')}.join(',')
       self
     end
 
@@ -125,7 +129,10 @@ module Norikra
       self.update_summary
     end
 
-    #TODO: have a bug?
+    def nullable_diff(fieldset) # data_fieldset.nullable_diff(query_fieldset)
+      fieldset.fields.select{|fname, f| !self.fields[fname] && f.nullable? }.values
+    end
+
     def ==(other)
       return false if self.class != other.class
       self.summary == other.summary && self.query_unique_keys == other.query_unique_keys
@@ -139,15 +146,16 @@ module Norikra
       d
     end
 
-    def subset?(other) # self is subset of other (or not)
-      (self.fields.keys - other.fields.keys).size == 0
+    def subset?(other) # self is subset of other (or not)  ### query_fieldset.subset?(fieldset_referred_from_event_data)
+      ### nullable fields can be ignored (to be updated later, with nullable fields ignored here)
+      (self.fields.keys - other.fields.keys).reject{|fname| @fields[fname].nullable? }.size < 1
     end
 
     def event_type_name
       @event_type_name.dup
     end
 
-    def bind(target, level, update_type_name=false)
+    def bind(target, level, type_name_update=false)
       @target = target
       @level = level
       prefix = case level
@@ -155,17 +163,22 @@ module Norikra
                when :query then 'q_'
                when :data then 'e_' # event
                else
-                 raise ArgumentError, "unknown fieldset bind level: #{level}, for target #{target}"
+                 raise ::ArgumentError, "unknown fieldset bind level: #{level}, for target #{target}"
                end
-      @rebounds += 1 if update_type_name
+      @rebounds += 1 if type_name_update
       query_unique_key = @query_unique_keys ? @query_unique_keys.join("\t") : ''
 
       @event_type_name = prefix + Digest::MD5.hexdigest([target, level.to_s, @rebounds.to_s, query_unique_key, @summary].join("\t"))
       self
     end
 
-    def rebind(update_type_name)
-      self.dup.bind(@target, @level, update_type_name)
+    def rebind(type_name_update, query_fieldset=nil)
+      renew = self.dup
+      if query_fieldset
+        diff = self.nullable_diff(query_fieldset)
+        renew.update(diff, true) unless diff.empty? # all nullable fields are optional
+      end
+      renew.bind(@target, @level, type_name_update)
     end
 
     def format(data)

data/lib/norikra/listener.rb

@@ -4,90 +4,221 @@ require 'esper/lib/commons-logging-1.1.3.jar'
 require 'esper/lib/antlr-runtime-4.1.jar'
 require 'esper/lib/cglib-nodep-3.1.jar'
 
+require 'rubygems'
+
 require 'norikra/field'
 require 'norikra/query'
 
+require 'norikra/logger'
+include Norikra::Log
+
 require 'json'
 
 module Norikra
-  class Listener
-    include com.espertech.esper.client.UpdateListener
-
-    def initialize(query_name, query_group, output_pool, events_statistics)
-      @query_name = query_name
-      @query_group = query_group
-      @output_pool = output_pool
-      @events_statistics = events_statistics
+  module Listener
+    def self.listup
+      return unless defined? Gem
+
+      plugins = Gem.find_latest_files('norikra/listener/*.rb')
+      plugins.each do |plugin|
+        begin
+          debug "plugin file found!", file: plugin
+          rbpath = plugin.dup
+          4.times do
+            rbpath = File.dirname( rbpath )
+          end
+          files = Dir.entries( rbpath )
+          gemname = files.select{|f| f=~ /\.gemspec$/ }.first.sub(/\.gemspec$/, '')
+          trace "Loading listener gem", gemname: gemname, path: plugin
+          require gemname
+          load plugin
+        rescue => e
+          warn "Failed to load norikra listener plugin", plugin: plugin.to_s, error_class: e.class, error: e.message
+          e.backtrace.each do |t|
+            warn "  " + t
+          end
+        end
+      end
+
+      known_consts = [:Base, :MemoryPool, :Loopback, :Stdout]
+      listeners = [Norikra::Listener::Stdout, Norikra::Listener::Loopback]
+      self.constants.each do |c|
+        next if known_consts.include?(c)
+
+        klass = Norikra::Listener.const_get(c)
+        if klass.is_a?(Class) && klass.superclass == Norikra::Listener::Base
+          listeners.push(klass)
+        end
+      end
+      listeners.push(Norikra::Listener::MemoryPool)
+      listeners
     end
 
-    def type_convert(value)
-      if value.respond_to?(:getUnderlying)
-        value = value.getUnderlying
+    class Base
+      include com.espertech.esper.client.UpdateListener
+
+      DEFAULT_ASYNC_INTERVAL = 0.1
+
+      def self.check(group_name)
+        raise NotImplementedError
       end
 
-      trace "converting", :value => value
+      def initialize(query_name, query_group, events_statistics)
+        @query_name = query_name
+        @query_group = query_group
+        @events_statistics = events_statistics
 
-      if value.nil?
-        value
-      elsif value.respond_to?(:to_hash)
-        Hash[ value.to_hash.map{|k,v| [ Norikra::Field.unescape_name(k), type_convert(v)] } ]
-      elsif value.respond_to?(:to_a)
-        value.to_a.map{|v| type_convert(v) }
-      elsif value.respond_to?(:force_encoding)
-        value.force_encoding('UTF-8')
-      else
-        value
+        @async_interval = DEFAULT_ASYNC_INTERVAL
+
+        @thread = nil
+        @events = []
+        @mutex = Mutex.new
+        @running = true
       end
-    end
 
-    def update(new_events, old_events)
-      t = Time.now.to_i
-      events = new_events.map{|e| [t, type_convert(e)]}
-      trace "updated event", :query => @query_name, :group => @query_group, :event => events
-      @output_pool.push(@query_name, @query_group, events)
-      @events_statistics[:output] += events.size
-    end
-  end
+      # def engine=(engine)
+      #   @engine = engine
+      # end
+
+      # def output_pool=(output_pool)
+      #   @output_pool = output_pool
+      # end
+
+      def start
+        if self.respond_to?(:process_async)
+          trace "starting thread to process events in background", query: @query_name
+          @thread = Thread.new(&method(:background))
+        end
+      end
+
+      def background
+        trace "backgroupd thread starts", query: @query_name
+        while @running
+          events_empty = true
+          events = nil
+          @mutex.synchronize do
+            events = @events
+            @events = []
+          end
+          unless events.empty?
+            events_empty = false
+            trace("calling #process_async"){ {listener: self.class, query: @query_name, events: events.size} }
+            process_async(events)
+          end
+          sleep @async_interval if events_empty
+        end
+      rescue => e
+        error "exception in listener background thread, stopped", listener: self.class, query: @query_name, error: e
+      end
+
+      def push(events)
+        @mutex.synchronize do
+          @events += events
+        end
+      end
+
+      # def process_async
+      # end
 
-  class LoopbackListener < Listener
-    def initialize(engine, query_name, query_group, events_statistics)
-      @engine = engine
-      @query_name = query_name
-      @query_group = query_group
-      @events_statistics = events_statistics
-      @loopback_target = Norikra::Query.loopback(query_group)
+      def shutdown
+        trace "stopping listener", query: @query_name
+        @running = false
+        @thread.join if @thread
+        @thread = nil
+      end
+
+      def type_convert(value)
+        if value.respond_to?(:getUnderlying)
+          value = value.getUnderlying
+        end
+
+        trace("converting"){ { value: value } }
+
+        if value.nil?
+          value
+        elsif value.respond_to?(:to_hash)
+          Hash[ value.to_hash.map{|k,v| [ Norikra::Field.unescape_name(k), type_convert(v)] } ]
+        elsif value.respond_to?(:to_a)
+          value.to_a.map{|v| type_convert(v) }
+        elsif value.respond_to?(:force_encoding)
+          value.force_encoding('UTF-8')
+        else
+          value
+        end
+      end
+
+      def update(new_events, old_events)
+        t = Time.now.to_i
+        events = new_events.map{|e| [t, type_convert(e)]}
+        trace("updated event"){ { query: @query_name, group: @query_group, event: events } }
+        push(events)
+        @events_statistics[:output] += events.size
+      end
     end
 
-    def update(new_events, old_events)
-      event_list = new_events.map{|e| type_convert(e) }
-      trace "loopback event", :query => @query_name, :group => @query_group, :event => event_list
-      @events_statistics[:output] += event_list.size
-      #
-      # We does NOT convert 'container.$0' into container['field'].
-      # Use escaped names like 'container__0'. That is NOT so confused.
-      @engine.send(@loopback_target, event_list)
+    class MemoryPool < Base
+      def self.check(group_name)
+        true
+      end
+
+      def output_pool=(output_pool)
+        @output_pool = output_pool
+      end
+
+      def update(new_events, old_events)
+        t = Time.now.to_i
+        events = new_events.map{|e| [t, type_convert(e)]}
+        trace("updated event"){ { query: @query_name, group: @query_group, event: events } }
+        @output_pool.push(@query_name, @query_group, events)
+        @events_statistics[:output] += events.size
+      end
     end
-  end
 
-  class StdoutListener < Listener
-    def initialize(engine, query_name, query_group, events_statistics)
-      raise "BUG: query group is not 'STDOUT()'" unless query_group == 'STDOUT()'
+    class Loopback < Base
+      def self.check(group_name)
+        group_name && group_name =~ /^LOOPBACK\((.+)\)$/ && $1
+      end
 
-      @engine = engine
-      @query_name = query_name
-      @query_group = query_group
-      @events_statistics = events_statistics
+      def initialize(query_name, query_group, events_statistics)
+        super
+        @loopback_target = Loopback.check(query_group)
+        raise "BUG: query group is not 'LOOPBACK(...)'" unless @loopback_target
+      end
+
+      def engine=(engine)
+        @engine = engine
+      end
 
-      @stdout = STDOUT
+      def update(new_events, old_events)
+        event_list = new_events.map{|e| type_convert(e) }
+        trace("loopback event"){ { query: @query_name, group: @query_group, event: event_list } }
+        @events_statistics[:output] += event_list.size
+        #
+        # We does NOT convert 'container.$0' into container['field'].
+        # Use escaped names like 'container__0'. That is NOT so confused.
+        @engine.send(@loopback_target, event_list)
+      end
     end
 
-    def update(new_events, old_events)
-      event_list = new_events.map{|e| type_convert(e) }
-      trace "stdout event", :query => @query_name, :event => event_list
-      @events_statistics[:output] += event_list.size
+    class Stdout < Base
+      def self.check(group_name)
+        group_name && group_name == "STDOUT()"
+      end
+
+      def initialize(query_name, query_group, events_statistics)
+        super
+        raise "BUG: query group is not 'STDOUT()'" unless Stdout.check(query_group)
+        @stdout = STDOUT
+      end
+
+      def update(new_events, old_events)
+        event_list = new_events.map{|e| type_convert(e) }
+        trace("stdout event"){ { query: @query_name, event: event_list } }
+        @events_statistics[:output] += event_list.size
 
-      event_list.each do |e|
-        @stdout.puts @query_name + "\t" + JSON.dump(e)
+        event_list.each do |e|
+          @stdout.puts @query_name + "\t" + JSON.dump(e)
+        end
       end
     end
   end

data/lib/norikra/logger.rb

@@ -55,7 +55,7 @@ module Norikra
       end
 
       level = level.upcase
-      raise ArgumentError, "unknown log level: #{level}" unless LOG_LEVELS.include?(level)
+      raise ::ArgumentError, "unknown log level: #{level}" unless LOG_LEVELS.include?(level)
 
       p = java.util.Properties.new
       p.setProperty('log4j.appender.default.layout', 'org.apache.log4j.PatternLayout')
@@ -122,32 +122,44 @@ module Norikra
 
     def self.logger; @@logger ; end
 
-    def trace(message, data=nil)
+    def trace(message, data=nil, &block)
+      return unless @@logger.enabled?(LEVEL_TRACE)
+      data ||= block
       from = @@devmode ? caller_locations(1,1) : nil
       @@logger.trace(message, data, from)
     end
 
-    def debug(message, data=nil)
+    def debug(message, data=nil, &block)
+      return unless @@logger.enabled?(LEVEL_DEBUG)
+      data ||= block
       from = @@devmode ? caller_locations(1,1) : nil
       @@logger.debug(message, data, from)
     end
 
-    def info(message, data=nil)
+    def info(message, data=nil, &block)
+      return unless @@logger.enabled?(LEVEL_INFO)
+      data ||= block
       from = @@devmode ? caller_locations(1,1) : nil
       @@logger.info(message, data, from)
     end
 
-    def warn(message, data=nil)
+    def warn(message, data=nil, &block)
+      return unless @@logger.enabled?(LEVEL_WARN)
+      data ||= block
       from = @@devmode ? caller_locations(1,1) : nil
       @@logger.warn(message, data, from)
     end
 
-    def error(message, data=nil)
+    def error(message, data=nil, &block)
+      return unless @@logger.enabled?(LEVEL_ERROR)
+      data ||= block
       from = @@devmode ? caller_locations(1,1) : nil
       @@logger.error(message, data, from)
     end
 
-    def fatal(message, data=nil)
+    def fatal(message, data=nil, &block)
+      # always enabled
+      data ||= block
       from = @@devmode ? caller_locations(1,1) : nil
       @@logger.fatal(message, data, from)
     end
@@ -172,6 +184,17 @@ module Norikra
       @buffer << [Time.now.strftime(TIME_FORMAT), level, line]
     end
 
+    def enabled?(level)
+      case level
+      when LEVEL_TRACE then @log4j.isTraceEnabled
+      when LEVEL_DEBUG then @log4j.isDebugEnabled
+      when LEVEL_INFO  then @log4j.isInfoEnabled
+      when LEVEL_WARN  then @log4j.isWarnEnabled
+      when LEVEL_ERROR then @log4j.isErrorEnabled
+      else true
+      end
+    end
+
     def trace(message, data=nil, from=nil)
       return unless @log4j.isTraceEnabled
       line = format(from, message, data)
@@ -225,7 +248,7 @@ module Norikra
 
       #, status:404, message:'content not found'
       if data.is_a?(Proc)
-        ', ' + format_data(data.call)
+        format_data(data.call)
       elsif data.is_a?(Hash)
         ', ' + data.map{|k,v| "#{k}:#{v.inspect}"}.join(', ')
       else
@@ -245,14 +268,15 @@ module Norikra
     FORMAT_SIMULATED = "%s [%s] %s\n"
     attr_accessor :logs, :output
     def initialize
-      @logs = { :TRACE => [], :DEBUG => [], :INFO => [], :WARN => [], :ERROR => [], :FATAL => [] }
+      @logs = { TRACE: [], DEBUG: [], INFO: [], WARN: [], ERROR: [], FATAL: [] }
       @output = []
     end
     def log(level, message, data, from)
-      @logs[level].push({:message => message, :data => data, :from => from})
+      @logs[level].push({message: message, data: data, from: from})
       formatted = sprintf(FORMAT_SIMULATED, Time.now.strftime(TIME_FORMAT), level.to_s, format(from, message, data))
       @output.push(formatted)
     end
+    def enabled?(level); true; end
     def trace(m,d,f); self.log(:TRACE,m,d,f); end
     def debug(m,d,f); self.log(:DEBUG,m,d,f); end
     def info(m,d,f) ; self.log(:INFO, m,d,f); end

data/lib/norikra/output_pool.rb

@@ -21,7 +21,7 @@ module Norikra
           group.delete(query_name)
         end
         @groups.delete(query_name)
-      @pool.delete(query_name)
+        @pool.delete(query_name)
       end
       nil
     end