norikra 0.0.1-java

data/junks/type_conversion.rb

@@ -0,0 +1,49 @@
+require 'pp'
+
+require 'norikra/query'
+require 'norikra/output_pool'
+require 'norikra/engine'
+
+pool = Norikra::OutputPool.new
+
+query1 = Norikra::Query.new(
+  :name => 'OrderEvent over 150 price counts per 10secs',
+  :tablename => 'OrderEvent',
+  :expression => 'select count(*) AS cnt from OrderEvent.win:time_batch(10 seconds) where price > 150'
+)
+query2 = Norikra::Query.new(
+  :name => 'OrderEvent events with 2 or more count',
+  :tablename => 'OrderEvent',
+  :expression => 'select * from OrderEvent where count > 1'
+)
+engine = Norikra::Engine.new(pool)
+
+engine.register(query1)
+engine.register(query2)
+
+flag = true
+
+thread = Thread.new do
+  while flag
+    sleep 1
+    results = pool.pop('OrderEvent over 150 price counts per 10secs')
+    pp(results) if results.size > 0
+    results = pool.pop('OrderEvent events with 2 or more count')
+    pp(results) if results.size > 0
+  end
+end
+
+t = Time.now + 35
+while Time.now < t
+  3.times do
+    engine.send('OrderEvent', [{"itemName"=>"test","price"=>150,"count"=>1}, {"itemName"=>"test","price"=>200,"count"=>1}])
+    sleep 3
+  end
+  engine.send('OrderEvent', [{"itemName"=>"test2","price"=>300}])
+  engine.send('OrderEvent', [{"itemName"=>"test2","price"=>300,"count"=>3000000000}])
+  engine.send('OrderEvent', [{"itemName"=>"test2","price"=>300,"count"=>3000}])
+end
+
+flag = false
+thread.join
+

data/junks/type_inherit.rb

@@ -0,0 +1,67 @@
+#!/usr/bin/env jruby
+
+require 'java'
+require 'esper/esper-4.9.0.jar'
+require 'esper/esper/lib/commons-logging-1.1.1.jar'
+require 'esper/esper/lib/antlr-runtime-3.2.jar'
+require 'esper/esper/lib/cglib-nodep-2.2.jar'
+require 'pp'
+
+# Lets get the epService provider
+ep_service = com.espertech.esper.client.EPServiceProviderManager.getDefaultProvider
+
+# And the configuration
+ep_config = ep_service.getEPAdministrator.getConfiguration
+
+class BaseListener
+  include com.espertech.esper.client.UpdateListener
+  def initialize(name)
+    @name = name
+  end
+  def update(newEvents, oldEvents)
+    puts "#{@name} #{Time.now}: " + newEvents.map{|e| e.getUnderlying.inspect}.join(' ')
+  end
+end
+
+# Create an unmatched listener
+class UnmatchedListener
+  include com.espertech.esper.client.UnmatchedListener
+
+  def update(event)
+    # puts "unmatched:\n- " + event.getProperties.inspect
+    # ignore
+  end
+end
+
+# Prepare the OrderEvent type
+base = {
+  "itemName" => "string",
+  "price" => "double"
+}
+ep_config.addEventType('b_test', base)
+
+ep_config.addEventType('q_test1', base, ['b_test'].to_java(:string))
+query1 = "SELECT count(*) AS cnt FROM q_test1.win:time_batch(3 seconds) WHERE price > 1.0"
+ep_service.getEPAdministrator.createEPL(query1).addListener(BaseListener.new('q_test1'))
+
+ep_config.addEventType('q_test2', base.merge({'amount' => 'int'}), ['b_test'].to_java(:string))
+query2 = "SELECT count(*) AS cnt FROM q_test2.win:time_batch(3 seconds) WHERE price > 1.0 AND amount > 1"
+ep_service.getEPAdministrator.createEPL(query2).addListener(BaseListener.new('q_test2'))
+
+ep_service.getEPRuntime.setUnmatchedListener(UnmatchedListener.new)
+
+# And finally process the event
+epr_runtime = ep_service.getEPRuntime
+
+ep_config.addEventType('e_v1', base, ['q_test1', 'b_test'].to_java(:string))
+ep_config.addEventType('e_v2', base, ['q_test2', 'q_test1', 'b_test'].to_java(:string))
+
+t = Time.now + 15
+while Time.now < t
+  epr_runtime.sendEvent({"itemName"=>"test","price"=>100}, "e_v1")
+  epr_runtime.sendEvent({"itemName"=>"test","amount"=>1,"price"=>300}, "e_v2")
+  epr_runtime.sendEvent({"itemName"=>"test","amount"=>2,"price"=>300}, "e_v2")
+  epr_runtime.sendEvent({"itemName"=>"test","price"=>200}, "e_v1")
+  epr_runtime.sendEvent({"itemName"=>"test","amount"=>3,"price"=>300}, "e_v2")
+  sleep 1
+end

data/lib/norikra.rb

@@ -0,0 +1,13 @@
+unless RUBY_PLATFORM =~ /java/
+  raise LoadError, 'Only supports JRuby'
+end
+
+require 'norikra/version'
+
+require 'norikra/typedef'
+require 'norikra/query'
+
+require 'norikra/typedef_manager'
+require 'norikra/output_pool'
+require 'norikra/engine'
+require 'norikra/rpc'

data/lib/norikra/cli.rb

@@ -0,0 +1,24 @@
+require 'norikra/server'
+require 'thor'
+
+module Norikra
+  class CLI < Thor
+    desc "start", "start Norikra server process"
+    option :host, :type => :string, :default => '0.0.0.0', :aliases => "-H", :desc => 'host address that server listen [0.0.0.0]'
+    option :port, :type => :numeric, :default => 26571, :aliases => "-P", :desc => 'port that server uses [26571]'
+    # option :config, :type => :string, :default => nil, :aliases => "-c", :desc => 'configuration file to define target/query [none]'
+    # option :daemonize, :type => :boolean, :default => false, :aliases => "-d", :desc => 'daemonize Norikra server [false]'
+    # option :pidfile, :type => :string, :default => '/var/run/norikra.pid', :aliases => "-p", :desc => "pidfile path when daemonized [/var/run/norikra.pid]"
+    # option :logfile, :type => :string, :default => '/var/log/norikra.log', :aliases => "-l", :desc => "logfile path when daemonized [/var/log/norikra.log]"
+
+    #TODO: configuration file to init
+    #TODO: daemonize
+    #  TODO: pidcheck
+    #  TODO: open logfile & write
+    #  TODO: logfile reopen
+    def start
+      server = Norikra::Server.new(options[:host], options[:port])
+      server.run
+    end
+  end
+end

data/lib/norikra/engine.rb

@@ -0,0 +1,231 @@
+require 'java'
+require 'esper-4.9.0.jar'
+require 'esper/lib/commons-logging-1.1.1.jar'
+require 'esper/lib/antlr-runtime-3.2.jar'
+require 'esper/lib/cglib-nodep-2.2.jar'
+
+require 'norikra/typedef_manager'
+
+module Norikra
+  class Engine
+    attr_reader :targets, :queries, :output_pool, :typedef_manager
+
+    def initialize(output_pool, typedef_manager=nil)
+      @output_pool = output_pool
+      @typedef_manager = typedef_manager || Norikra::TypedefManager.new()
+
+      @service = com.espertech.esper.client.EPServiceProviderManager.getDefaultProvider
+      @config = @service.getEPAdministrator.getConfiguration
+
+      @mutex = Mutex.new
+
+      # fieldsets already registered into @runtime
+      @registered_fieldsets = {} # {target => {fieldset_summary => Fieldset}
+
+      @targets = []
+      @queries = []
+
+      @waiting_queries = {} # target => [query]
+    end
+
+    def start
+      @runtime = @service.getEPRuntime
+    end
+
+    def stop
+      #TODO: stop to @runtime
+    end
+
+    def open(target, fields=nil)
+      return if @targets.include?(target)
+      open_target(target, fields)
+    end
+
+    def close(target)
+      #TODO: write
+      raise NotImplementedError
+    end
+
+    def reserve(target, field, type)
+      @typedef_manager.reserve(target, field, type)
+    end
+
+    def register(query)
+      unless @targets.include?(query.target)
+        open(query.target) # open as lazy defined target
+      end
+      register_query(query)
+    end
+
+    def deregister(query_name)
+      #TODO: write
+      raise NotImplementedError
+    end
+
+    def send(target, events)
+      #TODO: trace log
+      #p events
+      return unless @targets.include?(target) # discard events for target not registered
+      return if events.size < 1
+
+      if @typedef_manager.lazy?(target)
+        base_fieldset = @typedef_manager.generate_base_fieldset(target, events.first)
+        register_base_fieldset(target, base_fieldset)
+      end
+
+      registered_data_fieldset = @registered_fieldsets[target][:data]
+
+      events.each do |event|
+        fieldset = @typedef_manager.refer(target, event)
+
+        unless registered_data_fieldset[fieldset.summary]
+          # register waiting queries including this fieldset, and this fieldset itself
+          register_fieldset(target, fieldset)
+        end
+        #TODO: trace log
+        #p "sendEvent eventType:#{fieldset.event_type_name}, event:#{event.inspect}"
+        @runtime.sendEvent(@typedef_manager.format(target, event).to_java, fieldset.event_type_name)
+      end
+      nil
+    end
+
+    class Listener
+      include com.espertech.esper.client.UpdateListener
+
+      def initialize(query_name, output_pool)
+        @query_name = query_name
+        @output_pool = output_pool
+      end
+
+      def update(new_events, old_events)
+        #TODO: trace log
+        #p "updated event query:#{@query_name}, event:#{new_events.inspect}"
+        @output_pool.push(@query_name, new_events)
+      end
+    end
+    ##### Unmatched events are simply ignored
+    # class UnmatchedListener
+    #   include com.espertech.esper.client.UnmatchedListener
+    #   def update(event)
+    #     # puts "unmatched:\n- " + event.getProperties.inspect
+    #     # ignore
+    #   end
+    # end
+
+    private
+
+    def open_target(target, fields)
+      # from open
+      @mutex.synchronize do
+        return if @targets.include?(target)
+        @typedef_manager.add_target(target, fields)
+        @registered_fieldsets[target] = {:base => {}, :query => {}, :data => {}}
+
+        unless @typedef_manager.lazy?(target)
+          base_fieldset = @typedef_manager.base_fieldset(target)
+
+          @typedef_manager.bind_fieldset(target, :base, base_fieldset)
+          register_fieldset_actually(target, base_fieldset, :base)
+        end
+
+        @targets.push(target)
+      end
+    end
+
+    def register_base_fieldset(target, fieldset)
+      # for lazy target, with generated fieldset from sent events.first
+      @mutex.synchronize do
+        return unless @typedef_manager.lazy?(target)
+
+        @typedef_manager.bind_fieldset(target, :base, fieldset)
+        register_fieldset_actually(target, fieldset, :base)
+
+        @typedef_manager.activate(target, fieldset)
+      end
+      nil
+    end
+
+    def register_query(query)
+      target = query.target
+      @mutex.synchronize do
+        if @typedef_manager.lazy?(target) || @typedef_manager.fields_defined?(target, query.fields)
+          @waiting_queries[target] ||= []
+          @waiting_queries[target].push(query)
+        else
+          query_fieldset = @typedef_manager.generate_query_fieldset(target, query.fields)
+          @typedef_manager.bind_fieldset(target, :query, query_fieldset)
+          register_fieldset_actually(target, query_fieldset, :query)
+          register_query_actually(target, query_fieldset.event_type_name, query)
+        end
+        @queries.push(query)
+      end
+    end
+
+    def register_waiting_queries(target)
+      waitings = @waiting_queries.delete(target) || []
+      not_registered = []
+
+      waitings.each do |query|
+        if @typedef_manager.fields_defined?(target, query.fields)
+          query_fieldset = @typedef_manager.generate_query_fieldset(target, query.fields)
+          @typedef_manager.bind_fieldset(target, :query, query_fieldset)
+          register_fieldset_actually(target, query_fieldset, :query)
+          register_query_actually(target, query_fieldset.event_type_name, query)
+        else
+          not_registered.push(query)
+        end
+      end
+
+      @waiting_queries[target] = not_registered if not_registered.size > 0
+    end
+
+    def register_fieldset(target, fieldset)
+      @mutex.synchronize do
+        @typedef_manager.bind_fieldset(target, :data, fieldset)
+
+        if @waiting_queries[target]
+          register_waiting_queries(target)
+        end
+
+        register_fieldset_actually(target, fieldset, :data)
+      end
+    end
+
+    # this method should be protected with @mutex lock
+    def register_query_actually(target, stream_name, query)
+      query = query.dup_with_stream_name(stream_name)
+      epl = @service.getEPAdministrator.createEPL(query.expression)
+      epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], Listener.new(query.name, @output_pool)
+      #TODO: debug log
+      #p "addListener target:#{target}, query_name:#{query.name}, query:#{query.expression}"
+    end
+
+    # this method should be protected with @mutex lock
+    def register_fieldset_actually(target, fieldset, level)
+      return if @registered_fieldsets[target][level][fieldset.summary]
+
+      # Map Supertype (target) and Subtype (typedef name, like TARGET_TypeDefName)
+      # http://esper.codehaus.org/esper-4.9.0/doc/reference/en-US/html/event_representation.html#eventrep-map-supertype
+      # epService.getEPAdministrator().getConfiguration()
+      #   .addEventType("AccountUpdate", accountUpdateDef, new String[] {"BaseUpdate"});
+      case level
+      when :base
+        @config.addEventType(fieldset.event_type_name, fieldset.definition)
+        #TODO: debug log
+        #p "addEventType target:#{target}, level:base, eventType:#{fieldset.event_type_name}"
+      when :query
+        base_name = @typedef_manager.base_fieldset(target).event_type_name
+        @config.addEventType(fieldset.event_type_name, fieldset.definition, [base_name].to_java(:string))
+        #TODO: debug log
+        #p "addEventType target:#{target}, level:query, eventType:#{fieldset.event_type_name}, base:#{base_name}"
+      else
+        subset_names = @typedef_manager.subsets(target, fieldset).map(&:event_type_name)
+        @config.addEventType(fieldset.event_type_name, fieldset.definition, subset_names.to_java(:string))
+        #TODO: debug log
+        #p "addEventType target:#{target}, level:data, eventType:#{fieldset.event_type_name}, inherit:#{subset_names.join(',')}"
+      end
+      @registered_fieldsets[target][level][fieldset.summary] = fieldset
+      nil
+    end
+  end
+end

data/lib/norikra/output_pool.rb

@@ -0,0 +1,45 @@
+module Norikra
+  class OutputPool
+    attr_accessor :pool
+
+    def initialize
+      @pool = {}
+      @mutex = Mutex.new
+    end
+
+    def queries
+      @pool.keys
+    end
+
+    def push(query_name, events)
+      t = Time.now.to_i
+      events = events.map{|e| [t, (e.respond_to?(:getUnderlying) ? e.getUnderlying : e).to_hash]}
+      @mutex.synchronize do
+        @pool[query_name] ||= []
+        @pool[query_name].push(events) if events.size > 0
+      end
+    end
+
+    # returns [[time(int from epoch), event], ...], event: hash
+    def pop(query_name)
+      events = @mutex.synchronize do
+        @pool.delete(query_name) || []
+      end
+      events.reduce(&:+) || []
+    end
+
+    # returns {query_name => [[time, event], ...]}
+    def sweep
+      ret = {}
+      sweep_pool = @mutex.synchronize do
+        sweeped = @pool
+        @pool = {}
+        sweeped
+      end
+      sweep_pool.keys.each do |k|
+        ret[k] = sweep_pool[k].reduce(&:+) || []
+      end
+      ret
+    end
+  end
+end

data/lib/norikra/query.rb

@@ -0,0 +1,173 @@
+require 'java'
+require 'esper-4.9.0.jar'
+require 'esper/lib/commons-logging-1.1.1.jar'
+require 'esper/lib/antlr-runtime-3.2.jar'
+require 'esper/lib/cglib-nodep-2.2.jar'
+
+module Norikra
+  class Query
+    attr_accessor :name, :expression
+
+    def initialize(param={})
+      @name = param[:name]
+      @expression = param[:expression]
+      @ast = nil
+      @target = nil
+      @fields = nil
+    end
+
+    def dup
+      self.class.new(:name => @name, :expression => @expression.dup)
+    end
+
+    def dup_with_stream_name(actual_name)
+      target = self.target
+      query = self.dup
+      query.expression = self.expression.gsub(/(\s[Ff][Rr][Oo][Mm]\s+)#{target}(\.|\s)/, '\1' + actual_name + '\2')
+      if query.target != actual_name
+        raise RuntimeError, 'failed to replace query target into stream name:' + self.expression
+      end
+      query
+    end
+
+    def to_hash
+      {'name' => @name, 'expression' => @expression, 'target' => self.target}
+    end
+
+    def target
+      return @target if @target
+      #TODO: this code doesn't care JOINs.
+      @target = self.ast.find('STREAM_EXPR').find('EVENT_FILTER_EXPR').children.first.name
+      @target
+    end
+
+    def fields
+      return @fields if @fields
+      #TODO: this code doesn't care JOINs.
+
+      # Norikra::Query.new(
+      #   :name => 'hoge',
+      #   :expression => 'select count(*) AS cnt from www.win:time_batch(10 seconds) where path="/" AND search.length() > 0').ast.to_a
+      # ["EPL_EXPR",
+      #  ["SELECTION_EXPR", ["SELECTION_ELEMENT_EXPR", ["count"], ["cnt"]]],
+      #  ["STREAM_EXPR",
+      #   ["EVENT_FILTER_EXPR", ["www"]],
+      #   ["VIEW_EXPR",
+      #    ["win"],
+      #    ["time_batch"],
+      #    ["TIME_PERIOD", ["SECOND_PART", ["10"]]]]],
+      #  ["WHERE_EXPR",
+      #   ["EVAL_AND_EXPR",
+      #    ["EVAL_EQUALS_EXPR",
+      #     ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["path"]]],
+      #     ["\"/\""]],
+      #    [">",
+      #     ["LIB_FUNC_CHAIN", ["LIB_FUNCTION", ["search"], ["length"], ["("]]],
+      #     ["0"]]]]]
+
+      ast = self.ast
+      names_simple = ast.listup('EVENT_PROP_SIMPLE').map{|p| p.child.name}
+      names_chain_root = ast.listup('LIB_FUNC_CHAIN').map{|c| c.child.child.name}.select{|n| not self.class.imported_java_class?(n)}
+      @fields = (names_simple + names_chain_root).uniq.sort
+      @fields
+    end
+
+    def self.imported_java_class?(name)
+      return false unless name =~ /^[A-Z]/
+      # Esper auto-imports the following Java library packages:
+      # java.lang.* -> Java::JavaLang::*
+      # java.math.* -> Java::JavaMath::*
+      # java.text.* -> Java::JavaText::*
+      # java.util.* -> Java::JavaUtil::*
+      java_class('Java::JavaLang::'+name) || java_class('Java::JavaMath::'+name) ||
+        java_class('Java::JavaText::'+name) || java_class('Java::JavaUtil::'+name) || false
+    end
+    def self.java_class(const_name)
+      begin
+        c = eval(const_name)
+        c.class == Kernel ? nil : c
+      rescue NameError
+        return nil
+      end
+    end
+
+    class ParseRuleSelectorImpl
+      include com.espertech.esper.epl.parse.ParseRuleSelector
+      def invokeParseRule(parser)
+        parser.startEPLExpressionRule().getTree()
+      end
+    end
+
+    class ASTNode
+      attr_accessor :name, :children
+      def initialize(name, children)
+        @name = name
+        @children = children
+      end
+      def to_a
+        [@name] + @children.map(&:to_a)
+      end
+      def child
+        @children.first
+      end
+      def find(node_name) # only one, depth-first search
+        return self if @name == node_name
+        @children.each do |c|
+          r = c.find(node_name)
+          return r if r
+        end
+        nil
+      end
+      def listup(node_name)
+        result = []
+        result.push(self) if @name == node_name
+        @children.each do |c|
+          result.push(*c.listup(node_name))
+        end
+        result
+      end
+    end
+
+    def ast
+      #TODO: test
+      #TODO: take care for parse error
+      return @ast if @ast
+      rule = ParseRuleSelectorImpl.new
+      target = @expression.dup
+      forerrmsg = @expression.dup
+      result = com.espertech.esper.epl.parse.ParseHelper.parse(target, forerrmsg, true, rule, false)
+
+      def convSubTree(tree)
+        ASTNode.new(tree.text, (tree.children ? tree.children.map{|c| convSubTree(c)} : []))
+      end
+      @ast = convSubTree(result.getTree)
+      @ast
+    end
+
+    ### select max(price) as maxprice from HogeTable.win:time_batch(10 sec) where cast(amount, double) > 2 and price > 50
+    # query.ast.to_a
+    # ["EPL_EXPR",
+    #  ["SELECTION_EXPR",
+    #   ["SELECTION_ELEMENT_EXPR",
+    #    ["LIB_FUNC_CHAIN",
+    #     ["LIB_FUNCTION",
+    #      ["max"],
+    #      ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["price"]]],
+    #      ["("]]],
+    #    ["maxprice"]]],
+    #  ["STREAM_EXPR",
+    #   ["EVENT_FILTER_EXPR", ["HogeTable"]],
+    #   ["VIEW_EXPR",
+    #    ["win"],
+    #    ["time_batch"],
+    #    ["TIME_PERIOD", ["SECOND_PART", ["10"]]]]],
+    #  ["WHERE_EXPR",
+    #   ["EVAL_AND_EXPR",
+    #    [">",
+    #     ["cast",
+    #      ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["amount"]]],
+    #      ["double"]],
+    #     ["2"]],
+    #    [">", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["price"]]], ["50"]]]]]
+  end
+end