norikra 0.0.5-java → 0.0.6-java

data/lib/norikra/engine.rb

@@ -29,7 +29,7 @@ module Norikra
       @targets = []
       @queries = []
 
-      @waiting_queries = {} # target => [query]
+      @waiting_queries = []
     end
 
     def start
@@ -58,8 +58,8 @@ module Norikra
 
     def register(query)
       info "registering query", :name => query.name, :targets => query.targets, :expression => query.expression
-      unless @targets.include?(query.targets.first)
-        open(query.targets.first) # open as lazy defined target
+      query.targets.each do |target|
+        open(target) unless @targets.include?(target)
       end
       register_query(query)
     end
@@ -120,8 +120,7 @@ module Norikra
       end
 
       def update(new_events, old_events)
-        #TODO: trace log
-        #p "updated event query:#{@query_name}, event:#{new_events.inspect}"
+        trace "updated event", :query => @query_name, :event => new_events
         @output_pool.push(@query_name, new_events)
       end
     end
@@ -161,25 +160,27 @@ module Norikra
         return false unless @typedef_manager.lazy?(target)
 
         @typedef_manager.activate(target, fieldset)
-        # @typedef_manager.bind_fieldset(target, :base, fieldset)
         register_fieldset_actually(target, fieldset, :base)
       end
       nil
     end
 
     def register_query(query)
-      #TODO: support JOINs
-      target = query.targets.first
       @mutex.synchronize do
-        if @typedef_manager.lazy?(target) || !@typedef_manager.fields_defined?(target, query.fields)
-          @waiting_queries[target] ||= []
-          @waiting_queries[target].push(query)
-        else
-          query_fieldset = @typedef_manager.generate_query_fieldset(target, query.fields)
+        unless @typedef_manager.ready?(query)
+          @waiting_queries.push(query)
+          @queries.push(query)
+          return
+        end
+
+        mapping = @typedef_manager.generate_fieldset_mapping(query)
+        mapping.each do |target, query_fieldset|
           @typedef_manager.bind_fieldset(target, :query, query_fieldset)
           register_fieldset_actually(target, query_fieldset, :query)
-          register_query_actually(target, query_fieldset.event_type_name, query)
+        end
+        register_query_actually(query, mapping)
 
+        mapping.each do |target, query_fieldset|
           # replace registered data fieldsets with new fieldset inherits this query fieldset
           @typedef_manager.supersets(target, query_fieldset).each do |set|
             rebound = set.rebind(true) # update event_type_name with new inheritations
@@ -189,33 +190,38 @@ module Norikra
             remove_fieldset_actually(target, set, :data)
           end
         end
+
         @queries.push(query)
       end
     end
 
     def register_waiting_queries(target)
-      waitings = @waiting_queries.delete(target) || []
-      not_registered = []
+      ready = []
+      not_ready = []
+      @waiting_queries.each do |q|
+        if @typedef_manager.ready?(q)
+          ready.push(q)
+        else
+          not_ready.push(q)
+        end
+      end
+      @waiting_queries = not_ready
 
-      waitings.each do |query|
-        if @typedef_manager.fields_defined?(target, query.fields)
-          query_fieldset = @typedef_manager.generate_query_fieldset(target, query.fields)
+      ready.each do |query|
+        mapping = @typedef_manager.generate_fieldset_mapping(query)
+        mapping.each do |target, query_fieldset|
           @typedef_manager.bind_fieldset(target, :query, query_fieldset)
           register_fieldset_actually(target, query_fieldset, :query)
-          register_query_actually(target, query_fieldset.event_type_name, query)
-        else
-          not_registered.push(query)
         end
+        register_query_actually(query, mapping)
       end
-
-      @waiting_queries[target] = not_registered if not_registered.size > 0
     end
 
     def register_fieldset(target, fieldset)
       @mutex.synchronize do
         @typedef_manager.bind_fieldset(target, :data, fieldset)
 
-        if @waiting_queries[target]
+        if @waiting_queries.size > 0
           register_waiting_queries(target)
         end
 
@@ -224,12 +230,20 @@ module Norikra
     end
 
     # this method should be protected with @mutex lock
-    def register_query_actually(target, stream_name, query)
-      query = query.dup_with_stream_name(stream_name)
-      epl = @service.getEPAdministrator.createEPL(query.expression)
+    def register_query_actually(query, mapping)
+      # 'mapping' argument is {target => fieldset}
+      event_type_name_map = {}
+      mapping.keys.each do |key|
+        event_type_name_map[key] = mapping[key].event_type_name
+      end
+
+      administrator = @service.getEPAdministrator
+
+      statement_model = administrator.compileEPL(query.expression)
+      Norikra::Query.rewrite_event_type_name(statement_model, event_type_name_map)
+
+      epl = administrator.create(statement_model)
       epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], Listener.new(query.name, @output_pool)
-      #TODO: debug log
-      #p "addListener target:#{target}, query_name:#{query.name}, query:#{query.expression}"
     end
 
     # this method should be protected with @mutex lock
@@ -242,19 +256,16 @@ module Norikra
       #   .addEventType("AccountUpdate", accountUpdateDef, new String[] {"BaseUpdate"});
       case level
       when :base
+        debug "add event type", :target => target, :level => 'base', :event_type => fieldset.event_type_name
         @config.addEventType(fieldset.event_type_name, fieldset.definition)
-        #TODO: debug log
-        #p "addEventType target:#{target}, level:base, eventType:#{fieldset.event_type_name}"
       when :query
         base_name = @typedef_manager.base_fieldset(target).event_type_name
+        debug "add event type", :target => target, :level => 'query', :event_type => fieldset.event_type_name, :base => base_name
         @config.addEventType(fieldset.event_type_name, fieldset.definition, [base_name].to_java(:string))
-        #TODO: debug log
-        #p "addEventType target:#{target}, level:query, eventType:#{fieldset.event_type_name}, base:#{base_name}"
       else
         subset_names = @typedef_manager.subsets(target, fieldset).map(&:event_type_name)
+        debug "add event type", :target => target, :level => 'data', :event_type => fieldset.event_type_name, :inherit => subset_names
         @config.addEventType(fieldset.event_type_name, fieldset.definition, subset_names.to_java(:string))
-        #TODO: debug log
-        #p "addEventType target:#{target}, level:data, eventType:#{fieldset.event_type_name}, inherit:#{subset_names.join(',')}"
 
         @registered_fieldsets[target][level][fieldset.summary] = fieldset
       end
@@ -267,7 +278,7 @@ module Norikra
 
       # DON'T check @registered_fieldsets[target][level][fieldset.summary]
       # removed fieldset should be already replaced with register_fieldset_actually w/ replace flag
-      #TODO: debug log
+      debug "remove event type", :target => target, :event_type => fieldset.event_type_name
       @config.removeEventType(fieldset.event_type_name, true)
     end
   end

data/lib/norikra/query.rb

@@ -4,6 +4,8 @@ require 'esper/lib/commons-logging-1.1.1.jar'
 require 'esper/lib/antlr-runtime-3.2.jar'
 require 'esper/lib/cglib-nodep-2.2.jar'
 
+require 'norikra/query/ast'
+
 module Norikra
   class Query
     attr_accessor :name, :expression
@@ -13,6 +15,7 @@ module Norikra
       @expression = param[:expression]
       @ast = nil
       @targets = nil
+      @subqueries = nil
       @fields = nil
     end
 
@@ -36,139 +39,168 @@ module Norikra
 
     def targets
       return @targets if @targets
-      #TODO: test with JOINs.
-      @targets = self.ast.find('STREAM_EXPR').find('EVENT_FILTER_EXPR').children.map(&:name)
+      @targets = (self.ast.listup(:stream).map(&:target) + self.subqueries.map(&:targets).flatten).sort.uniq
       @targets
     end
 
-    def fields
-      return @fields if @fields
-      #TODO: this code doesn't care JOINs.
-      ### fields -> {'target_name' => fields} ?
-
-      # Norikra::Query.new(
-      #   :name => 'hoge',
-      #   :expression => 'select count(*) AS cnt from www.win:time_batch(10 seconds) where path="/" AND search.length() > 0').ast.to_a
-      # ["EPL_EXPR",
-      #  ["SELECTION_EXPR", ["SELECTION_ELEMENT_EXPR", ["count"], ["cnt"]]],
-      #  ["STREAM_EXPR",
-      #   ["EVENT_FILTER_EXPR", ["www"]],
-      #   ["VIEW_EXPR",
-      #    ["win"],
-      #    ["time_batch"],
-      #    ["TIME_PERIOD", ["SECOND_PART", ["10"]]]]],
-      #  ["WHERE_EXPR",
-      #   ["EVAL_AND_EXPR",
-      #    ["EVAL_EQUALS_EXPR",
-      #     ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["path"]]],
-      #     ["\"/\""]],
-      #    [">",
-      #     ["LIB_FUNC_CHAIN", ["LIB_FUNCTION", ["search"], ["length"], ["("]]],
-      #     ["0"]]]]]
-
-      ast = self.ast
-      names_simple = ast.listup('EVENT_PROP_SIMPLE').map{|p| p.child.name}
-      names_chain_root = ast.listup('LIB_FUNC_CHAIN').map{|c| c.child.child.name}.select{|n| not self.class.imported_java_class?(n)}
-      @fields = (names_simple + names_chain_root).uniq.sort
-      @fields
+    def subqueries
+      return @subqueries if @subqueries
+      @subqueries = self.ast.listup(:subquery).map{|n| Norikra::SubQuery.new(n)}
+      @subqueries
     end
 
-    def self.imported_java_class?(name)
-      return false unless name =~ /^[A-Z]/
-      # Esper auto-imports the following Java library packages:
-      # java.lang.* -> Java::JavaLang::*
-      # java.math.* -> Java::JavaMath::*
-      # java.text.* -> Java::JavaText::*
-      # java.util.* -> Java::JavaUtil::*
-      java_class('Java::JavaLang::'+name) || java_class('Java::JavaMath::'+name) ||
-        java_class('Java::JavaText::'+name) || java_class('Java::JavaUtil::'+name) || false
-    end
-    def self.java_class(const_name)
-      begin
-        c = eval(const_name)
-        c.class == Kernel ? nil : c
-      rescue NameError
-        return nil
-      end
-    end
+    def explore(outer_targets=[], alias_overridden={})
+      fields = {}
+      alias_map = {}.merge(alias_overridden)
 
-    class ParseRuleSelectorImpl
-      include com.espertech.esper.epl.parse.ParseRuleSelector
-      def invokeParseRule(parser)
-        parser.startEPLExpressionRule().getTree()
+      all = []
+      unknowns = []
+      self.ast.listup(:stream).each do |node|
+        #TODO: raise error for same name of target/alias
+        if node.alias
+          alias_map[node.alias] = node.target
+        end
+        fields[node.target] = []
       end
-    end
 
-    class ASTNode
-      attr_accessor :name, :children
-      def initialize(name, children)
-        @name = name
-        @children = children
-      end
-      def to_a
-        [@name] + @children.map(&:to_a)
+      default_target = fields.keys.size == 1 ? fields.keys.first : nil
+
+      outer_targets.each do |t|
+        fields[t] ||= []
       end
-      def child
-        @children.first
+
+      field_bag = []
+      self.subqueries.each do |subquery|
+        field_bag.push(subquery.explore(fields.keys, alias_map))
       end
-      def find(node_name) # only one, depth-first search
-        return self if @name == node_name
-        @children.each do |c|
-          r = c.find(node_name)
-          return r if r
+
+      self.ast.fields(default_target).each do |field_def|
+        f = field_def[:f]
+        all.push(f)
+
+        if field_def[:t]
+          t = alias_map[field_def[:t]] || field_def[:t]
+          unless fields[t]
+            raise "unknown target alias name for: #{field_def[:t]}.#{field_def[:f]}"
+          end
+          fields[t].push(f)
+
+        else
+          unknowns.push(f)
         end
-        nil
       end
-      def listup(node_name)
-        result = []
-        result.push(self) if @name == node_name
-        @children.each do |c|
-          result.push(*c.listup(node_name))
+
+      field_bag.each do |bag|
+        all += bag['']
+        unknowns += bag[nil]
+        bag.keys.each do |t|
+          fields[t] ||= []
+          fields[t] += bag[t]
         end
-        result
+      end
+
+      fields.keys.each do |target|
+        fields[target] = fields[target].sort.uniq
+      end
+      fields[''] = all.sort.uniq
+      fields[nil] = unknowns.sort.uniq
+
+      fields
+    end
+
+    def fields(target='')
+      # target '': fields for all targets (without target name)
+      # target nil: fields for unknown targets
+      return @fields[target] if @fields
+
+      @fields = explore()
+      @fields[target]
+    end
+
+    class ParseRuleSelectorImpl
+      include com.espertech.esper.epl.parse.ParseRuleSelector
+      def invokeParseRule(parser)
+        parser.startEPLExpressionRule().getTree()
       end
     end
 
     def ast
-      #TODO: test
-      #TODO: take care for parse error
+      #TODO: take care for parse error(com.espertech.esper.client.EPStatementSyntaxException)
       return @ast if @ast
       rule = ParseRuleSelectorImpl.new
       target = @expression.dup
       forerrmsg = @expression.dup
       result = com.espertech.esper.epl.parse.ParseHelper.parse(target, forerrmsg, true, rule, false)
 
-      def convSubTree(tree)
-        ASTNode.new(tree.text, (tree.children ? tree.children.map{|c| convSubTree(c)} : []))
-      end
-      @ast = convSubTree(result.getTree)
+      @ast = astnode(result.getTree)
       @ast
     end
 
-    ### select max(price) as maxprice from HogeTable.win:time_batch(10 sec) where cast(amount, double) > 2 and price > 50
-    # query.ast.to_a
-    # ["EPL_EXPR",
-    #  ["SELECTION_EXPR",
-    #   ["SELECTION_ELEMENT_EXPR",
-    #    ["LIB_FUNC_CHAIN",
-    #     ["LIB_FUNCTION",
-    #      ["max"],
-    #      ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["price"]]],
-    #      ["("]]],
-    #    ["maxprice"]]],
-    #  ["STREAM_EXPR",
-    #   ["EVENT_FILTER_EXPR", ["HogeTable"]],
-    #   ["VIEW_EXPR",
-    #    ["win"],
-    #    ["time_batch"],
-    #    ["TIME_PERIOD", ["SECOND_PART", ["10"]]]]],
-    #  ["WHERE_EXPR",
-    #   ["EVAL_AND_EXPR",
-    #    [">",
-    #     ["cast",
-    #      ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["amount"]]],
-    #      ["double"]],
-    #     ["2"]],
-    #    [">", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", ["price"]]], ["50"]]]]]
+    def self.rewrite_event_type_name(statement_model, mapping)
+      # mapping: {target_name => query_event_type_name}
+
+      ### esper-4.9.0/esper/doc/reference/html/epl_clauses.html#epl-subqueries
+      # Subqueries can only consist of a select clause, a from clause and a where clause.
+      # The group by and having clauses, as well as joins, outer-joins and output rate limiting are not permitted within subqueries.
+
+      # model.getFromClause.getStreams[0].getFilter.setEventTypeName("hoge")
+
+      # model.getSelectClause.getSelectList[1].getExpression => #<Java::ComEspertechEsperClientSoda::SubqueryExpression:0x3344c133>
+      # model.getSelectClause.getSelectList[1].getExpression.getModel.getFromClause.getStreams[0].getFilter.getEventTypeName
+      # model.getWhereClause.getChildren[1]                 .getModel.getFromClause.getStreams[0].getFilter.getEventTypeName
+
+      statement_model.getFromClause.getStreams.each do |stream|
+        target_name = stream.getFilter.getEventTypeName
+        unless mapping[target_name]
+          raise RuntimeError, "target missing in mapping, maybe BUG"
+        end
+        stream.getFilter.setEventTypeName(mapping[target_name])
+      end
+
+      dig = lambda {|node|
+        if node.is_a?(Java::ComEspertechEsperClientSoda::SubqueryExpression)
+          Norikra::Query.rewrite_event_type_name(node.getModel, mapping)
+        elsif node.getChildren.size > 0
+          node.getChildren.each do |c|
+            dig.call(c)
+          end
+        end
+      }
+
+      if statement_model.getSelectClause
+        statement_model.getSelectClause.getSelectList.each do |item|
+          dig.call(item.getExpression)
+        end
+      end
+
+      if statement_model.getWhereClause
+        statement_model.getWhereClause.getChildren.each do |child|
+          dig.call(child)
+        end
+      end
+
+      statement_model
+    end
+  end
+
+  class SubQuery < Query
+    def initialize(ast_nodetree)
+      @ast = ast_nodetree
+      @targets = nil
+      @subqueries = nil
+    end
+
+    def ast; @ast; end
+
+    def subqueries
+      return @subqueries if @subqueries
+      @subqueries = @ast.children.map{|c| c.listup(:subquery)}.reduce(&:+).map{|n| Norikra::SubQuery.new(n)}
+      @subqueries
+    end
+
+    def name; ''; end
+    def expression; ''; end
+    def dup; self; end
+    def dup_with_stream_name(actual_name); self; end
   end
 end

data/lib/norikra/query/ast.rb

@@ -0,0 +1,237 @@
+module Norikra
+  class Query
+    ### SELECT MAX(size) AS maxsize, fraud.aaa,bbb
+    ### FROM FraudWarningEvent.win:keepall() AS fraud,
+    ###      PINChangeEvent(size > 10).win:time(20 sec)
+    ### WHERE fraud.accountNumber.substr(0,8) = substr(PINChangeEvent.accountNumber, 0, 8)
+    ###   AND cast(PINChangeEvent.size,double) > 10.5
+    #
+    # ["EPL_EXPR",
+    #  ["SELECTION_EXPR",
+    #   ["SELECTION_ELEMENT_EXPR",
+    #    ["LIB_FUNC_CHAIN",
+    #     ["LIB_FUNCTION",
+    #      "max",
+    #      ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]],
+    #      "("]],
+    #    "maxsize"],
+    #   ["SELECTION_ELEMENT_EXPR",
+    #    ["EVENT_PROP_EXPR",
+    #     ["EVENT_PROP_SIMPLE", "fraud"],
+    #     ["EVENT_PROP_SIMPLE", "aaa"]]],
+    #   ["SELECTION_ELEMENT_EXPR",
+    #    ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "bbb"]]]],
+    #  ["STREAM_EXPR",
+    #   ["EVENT_FILTER_EXPR", "FraudWarningEvent"],
+    #   ["VIEW_EXPR", "win", "keepall"],
+    #   "fraud"],
+    #  ["STREAM_EXPR",
+    #   ["EVENT_FILTER_EXPR",
+    #    "PINChangeEvent",
+    #    [">", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]], "10"]],
+    #   ["VIEW_EXPR", "win", "time", ["TIME_PERIOD", ["SECOND_PART", "20"]]]],
+    #  ["WHERE_EXPR",
+    #   ["EVAL_AND_EXPR",
+    #    ["EVAL_EQUALS_EXPR",
+    #     ["LIB_FUNC_CHAIN",
+    #      ["LIB_FUNCTION", "fraud.accountNumber", "substr", "0", "8", "("]],
+    #     ["LIB_FUNC_CHAIN",
+    #      ["LIB_FUNCTION",
+    #       "substr",
+    #       ["EVENT_PROP_EXPR",
+    #        ["EVENT_PROP_SIMPLE", "PINChangeEvent"],
+    #        ["EVENT_PROP_SIMPLE", "accountNumber"]],
+    #       "0",
+    #       "8",
+    #       "("]]],
+    #    [">",
+    #     ["cast",
+    #      ["EVENT_PROP_EXPR",
+    #       ["EVENT_PROP_SIMPLE", "PINChangeEvent"],
+    #       ["EVENT_PROP_SIMPLE", "size"]],
+    #      "double"],
+    #     "10.5"]]]]
+
+    def astnode(tree)
+      children = if tree.children
+                   tree.children.map{|c| astnode(c)}
+                 else
+                   []
+                 end
+      case tree.text
+      when 'EVENT_PROP_EXPR'
+        ASTEventPropNode.new(tree.text, children)
+      when 'LIB_FUNCTION'
+        ASTLibFunctionNode.new(tree.text, children)
+      when 'STREAM_EXPR'
+        ASTStreamNode.new(tree.text, children)
+      when 'SUBSELECT_EXPR'
+        ASTSubSelectNode.new(tree.text, children)
+      else
+        ASTNode.new(tree.text, children)
+      end
+    end
+
+    class ASTNode
+      attr_accessor :name, :children
+
+      def initialize(name, children)
+        @name = name
+        @children = children
+      end
+
+      def nodetype?(*sym)
+        false
+      end
+
+      def to_a
+        [@name] + @children.map{|c| c.children.size > 0 ? c.to_a : c.name}
+      end
+
+      def child
+        @children.first
+      end
+
+      def find(type) # only one, depth-first search
+        return self if type.is_a?(String) && @name == type || nodetype?(type)
+
+        @children.each do |c|
+          next if type != :subquery && c.nodetype?(:subquery)
+          r = c.find(type)
+          return r if r
+        end
+        nil
+      end
+
+      def listup(type) # search all nodes that has 'type'
+        result = []
+        result.push(self) if type.is_a?(String) && @name == type || nodetype?(type)
+
+        @children.each do |c|
+          next if type != :subquery && c.nodetype?(:subquery)
+          result.push(*c.listup(type))
+        end
+        result
+      end
+
+      def fields(default_target=nil)
+        @children.map{|c| c.nodetype?(:subquery) ? [] : c.fields(default_target)}.reduce(&:+) || []
+      end
+    end
+
+    class ASTEventPropNode < ASTNode # EVENT_PROP_EXPR
+      # ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "bbb"]]
+      # ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "fraud"], ["EVENT_PROP_SIMPLE", "aaa"]]
+
+      def nodetype?(*sym)
+        sym.include?(:prop) || sym.include?(:property)
+      end
+
+      def fields(default_target=nil)
+        props = self.listup('EVENT_PROP_SIMPLE')
+        if props.size > 1 # alias.fieldname
+          [ {:f => props[1].child.name, :t => props[0].child.name} ]
+        else # fieldname (default target)
+          [ {:f => props[0].child.name, :t => default_target } ]
+        end
+      end
+    end
+
+    class ASTLibFunctionNode < ASTNode # LIB_FUNCTION
+      # ["LIB_FUNCTION", "now", "("]                 #### now()
+      # ["LIB_FUNCTION", "hoge", "length", "("] #    #### hoge.length()
+      # ["LIB_FUNCTION", "hoge", "substr", "0", "("] #### hoge.substr(0)
+      # ["LIB_FUNCTION", "substr", "10", "0", "("]   #### substr(10,0)
+      # ["LIB_FUNCTION", "hoge", "substr", "0", "8", "("] #### hoge.substr(0,8)
+      # ["LIB_FUNCTION", "max", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]], "("] #### max(size)
+
+      def nodetype?(*sym)
+        sym.include?(:lib) || sym.include?(:libfunc)
+      end
+
+      def fields(default_target=nil)
+        if @children.size <= 2
+          # single function like 'now()', function-name and "("
+          []
+
+        elsif @children[1].nodetype?(:prop, :lib, :subquery)
+          # first element should be func name if second element is property, library call or subqueries
+          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+
+        elsif @children[1].name =~ /^(-)?\d+(\.\d+)$/ || @children[1].name =~ /^'[^']*'$/ || @children[1].name =~ /^"[^"]*"$/
+          # first element should be func name if secod element is number/string literal
+          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+
+        elsif Norikra::Query.imported_java_class?(@children[0].name)
+          # Java imported class name (ex: 'Math.abs(-1)')
+          self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+
+        else
+          # first element may be property (simple 'fieldname.funcname()' or fully qualified 'target.fieldname.funcname()')
+          target,fieldname = if @children[0].name.include?('.')
+                               @children[0].name.split('.', 2)
+                             else
+                               [default_target,@children[0].name]
+                             end
+          children_list = self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target)}.reduce(&:+) || []
+          [{:f => fieldname, :t => target}] + children_list
+        end
+      end
+    end
+
+    class ASTStreamNode < ASTNode # STREAM_EXPR
+      #  ["STREAM_EXPR",
+      #   ["EVENT_FILTER_EXPR", "FraudWarningEvent"],
+      #   ["VIEW_EXPR", "win", "keepall"],
+      #   "fraud"],
+      #  ["STREAM_EXPR",
+      #   ["EVENT_FILTER_EXPR",
+      #    "PINChangeEvent",
+      #    [">", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]], "10"]],
+      #   ["VIEW_EXPR", "win", "time", ["TIME_PERIOD", ["SECOND_PART", "20"]]]],
+
+      def nodetype?(*sym)
+        sym.include?(:stream)
+      end
+
+      def target
+        self.find('EVENT_FILTER_EXPR').child.name
+      end
+
+      def alias
+        @children.last.children.size < 1 ? @children.last.name : nil
+      end
+
+      def fields(default_target=nil)
+        this_target = self.target
+        self.listup('EVENT_PROP_EXPR').map{|p| p.fields(this_target)}.reduce(&:+) || []
+      end
+    end
+
+    class ASTSubSelectNode < ASTNode # SUBSELECT_EXPR
+      def nodetype?(*sym)
+        sym.include?(:subquery)
+      end
+    end
+
+    def self.imported_java_class?(name)
+      return false unless name =~ /^[A-Z]/
+      # Esper auto-imports the following Java library packages:
+      # java.lang.* -> Java::JavaLang::*
+      # java.math.* -> Java::JavaMath::*
+      # java.text.* -> Java::JavaText::*
+      # java.util.* -> Java::JavaUtil::*
+      java_class('Java::JavaLang::'+name) || java_class('Java::JavaMath::'+name) ||
+        java_class('Java::JavaText::'+name) || java_class('Java::JavaUtil::'+name) || false
+    end
+
+    def self.java_class(const_name)
+      begin
+        c = eval(const_name)
+        c.class == Kernel ? nil : c
+      rescue NameError
+        return nil
+      end
+    end
+  end
+end

data/lib/norikra/typedef_manager.rb

@@ -41,6 +41,33 @@ module Norikra
       @typedefs[target].field_defined?(field_name_list)
     end
 
+    def ready?(query)
+      query.targets.all?{|t| @typedefs[t] && ! @typedefs[t].lazy? && @typedefs[t].field_defined?(query.fields(t))} &&
+        query.fields(nil).all?{|f| query.targets.any?{|t| @typedefs[t].field_defined?([f])}}
+    end
+
+    def generate_fieldset_mapping(query)
+      fields_set = {}
+      query.targets.each do |target|
+        fields_set[target] = query.fields(target)
+      end
+      query.fields(nil).each do |field|
+        assumed = query.targets.select{|t| @typedefs[t].field_defined?([field])}
+        if assumed.size != 1
+          #TODO exception class
+          raise "cannot determine target for field #{field}"
+        end
+        fields_set[assumed.first].push(field)
+      end
+
+      mapping = {}
+      fields_set.each do |target,fields|
+        mapping[target] = generate_query_fieldset(target, fields.sort.uniq)
+      end
+
+      mapping
+    end
+
     def bind_fieldset(target, level, fieldset)
       fieldset.bind(target, level)
       @typedefs[target].push(level, fieldset)

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/spec/query_spec.rb

@@ -6,26 +6,107 @@ describe Norikra::Query do
   context 'when instanciate' do
     describe '#initialize' do
       context 'with simple query' do
-        expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) WHERE path="/" AND size > 100 and param.length() > 0'
-        subject { Norikra::Query.new(
+        it 'returns query instances collectly parsed' do
+          expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) WHERE path="/" AND size > 100 and param.length() > 0'
+          q = Norikra::Query.new(
             :name => 'TestTable query1', :expression => expression
-        ) }
-        its(:name){ should == 'TestTable query1' }
-        its(:expression){ should == expression }
-        its(:targets){ should == ['TestTable'] }
-        its(:fields){ should == ['param', 'path', 'size'] }
+          )
+          expect(q.name).to eql('TestTable query1')
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['TestTable'])
+
+          expect(q.fields).to eql(['param', 'path', 'size'].sort)
+          expect(q.fields('TestTable')).to eql(['param','path','size'].sort)
+          expect(q.fields(nil)).to eql([])
+        end
+      end
+
+      context 'with query including Static lib call' do
+        it 'returns query instances collectly parsed' do
+          expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) AS source WHERE source.path="/" AND Math.abs(-1 * source.size) > 3'
+          q = Norikra::Query.new(
+            :name => 'TestTable query2', :expression => expression
+          )
+          expect(q.name).to eql('TestTable query2')
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['TestTable'])
+
+          expect(q.fields).to eql(['path', 'size'].sort)
+          expect(q.fields('TestTable')).to eql(['path', 'size'].sort)
+          expect(q.fields(nil)).to eql([])
+        end
+      end
+
+      context 'with query with join' do
+        it 'returns query instances collectly parsed' do
+          expression = 'select product, max(sta.size) as maxsize from StreamA.win:keepall() as sta, StreamB(size > 10).win:time(20 sec) as stb where sta.data.substr(0,8) = stb.header AND Math.abs(sta.size) > 3'
+          q = Norikra::Query.new(
+            :name => 'TestTable query3', :expression => expression
+          )
+          expect(q.name).to eql('TestTable query3')
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['StreamA', 'StreamB'])
+
+          expect(q.fields).to eql(['product', 'size', 'data', 'header'].sort)
+          expect(q.fields('StreamA')).to eql(['size','data'].sort)
+          expect(q.fields('StreamB')).to eql(['size','header'].sort)
+          expect(q.fields(nil)).to eql(['product'])
+        end
+      end
+
+      context 'with query with subquery (where clause)' do
+        it 'returns query instances collectly parsed' do
+          expression = 'select * from RfidEvent as RFID where "Dock 1" = (select name from Zones.std:unique(zoneName) where zoneId = RFID.zoneId)'
+          q = Norikra::Query.new(
+            :name => 'TestTable query4', :expression => expression
+          )
+          expect(q.name).to eql('TestTable query4')
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['RfidEvent', 'Zones'])
+
+          expect(q.fields).to eql(['name','zoneName','zoneId'].sort)
+          expect(q.fields('RfidEvent')).to eql(['zoneId'])
+          expect(q.fields('Zones')).to eql(['name','zoneName','zoneId'].sort)
+          expect(q.fields(nil)).to eql([])
+        end
+      end
+
+      context 'with query with subquery (select clause)' do
+        it 'returns query instances collectly parsed' do
+          expression = 'select zoneId, (select name from Zones.std:unique(zoneName) where zoneId = RfidEvent.zoneId) as name from RfidEvent'
+          q = Norikra::Query.new(
+            :name => 'TestTable query5', :expression => expression
+          )
+          expect(q.name).to eql('TestTable query5')
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['RfidEvent', 'Zones'].sort)
+
+          expect(q.fields).to eql(['name','zoneName','zoneId'].sort)
+          expect(q.fields('RfidEvent')).to eql(['zoneId'])
+          expect(q.fields('Zones')).to eql(['name','zoneName','zoneId'].sort)
+          expect(q.fields(nil)).to eql([])
+        end
+      end
+
+      context 'with query with subquery (from clause)' do
+        it 'returns query instances collectly parsed' do
+          expression = "select * from BarData(ticker='MSFT', sub(closePrice, (select movAgv from SMA20Stream(ticker='MSFT').std:lastevent())) > 0)"
+          q = Norikra::Query.new(
+            :name => 'TestTable query6', :expression => expression
+          )
+          expect(q.name).to eql('TestTable query6')
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['BarData', 'SMA20Stream'].sort)
+
+          expect(q.fields).to eql(['ticker','closePrice','movAgv'].sort)
+          expect(q.fields('BarData')).to eql(['ticker','closePrice'].sort)
+          expect(q.fields('SMA20Stream')).to eql(['movAgv','ticker'].sort)
+          expect(q.fields(nil)).to eql([])
+        end
       end
-      context 'with query including Static lib call'
-        expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) WHERE path="/" AND Math.abs(-1 * size) > 3'
-        subject { Norikra::Query.new(
-            :name => 'TestTable query1', :expression => expression
-        ) }
-        its(:name){ should == 'TestTable query1' }
-        its(:expression){ should == expression }
-        its(:targets){ should == ['TestTable'] }
-        its(:fields){ should == ['path', 'size'] }
     end
-    describe '#dup_with_stream_name' do
+
+   describe '#dup_with_stream_name' do
       context 'with simple query' do
         expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) WHERE path="/" AND size > 100 and param.length() > 0'
         it 'returns duplicated object, with replaced ' do
@@ -37,6 +118,7 @@ describe Norikra::Query do
           )
         end
       end
+
       context 'with query with newlines' do
         expression = <<EOQ
 SELECT
@@ -61,6 +143,7 @@ EOQ
       end
     end
   end
+
   describe '.imported_java_class?' do
     it 'can do judge passed name exists under java package tree or not' do
       expect(Norikra::Query.imported_java_class?('String')).to be_true

data/spec/typedef_manager_spec.rb

@@ -41,6 +41,8 @@ describe Norikra::TypedefManager do
   context 'when instanciated with a target with fields definition' do
     manager = Norikra::TypedefManager.new
     manager.add_target('sample', {'a'=>'string','b'=>'string','c'=>'double'})
+    manager.reserve('sample', 'z', 'boolean')
+    manager.add_target('sample_next', {'a'=>'string','b'=>'string','c'=>'double','d'=>'double'})
 
     set_query_base = Norikra::FieldSet.new({'a'=>'string','b'=>'string','c'=>'double'})
 
@@ -68,9 +70,63 @@ describe Norikra::TypedefManager do
     describe '#fields_defined?' do
       it 'does not fail' do
         expect(manager.fields_defined?('sample', ['a','b','x'])).to be_true
-        expect(manager.fields_defined?('sample', ['a','b','z'])).to be_false
+        expect(manager.fields_defined?('sample', ['a','b','y'])).to be_false
       end
     end
+
+    describe '#ready?' do
+      context 'with query with single target' do
+        it 'returns boolean which matches target or not' do
+          q1 = Norikra::Query.new(:name => 'test', :expression => 'select a from sample.win:time(5 sec) where c > 1.0 and z')
+          expect(manager.ready?(q1)).to be_true
+          q2 = Norikra::Query.new(:name => 'test', :expression => 'select a from sample.win:time(5 sec) where c > 1.0 and d > 2.0')
+          expect(manager.ready?(q2)).to be_false
+          q3 = Norikra::Query.new(:name => 'test', :expression => 'select a from sample2.win:time(5 sec) where c > 1.0 and d > 2.0')
+          expect(manager.ready?(q3)).to be_false
+        end
+      end
+
+      context 'with query with multi targets, including unexisting target' do
+        it 'returns false' do
+          q = Norikra::Query.new(:name => 'test', :expression => 'select x.a,y.a from sample.win:time(5 sec) as x, sample2.win:time(5 sec) as y where x.c > 1.0 and y.d > 1.0')
+          expect(manager.ready?(q)).to be_false
+        end
+      end
+
+      context 'with query with multi targets, all of them are exisitng' do
+        it 'returns true' do
+          q = Norikra::Query.new(:name => 'test', :expression => 'select x.a,d from sample.win:time(5 sec) as x, sample_next.win:time(5 sec) as y where x.c > 1.0 and y.d > 1.0')
+          expect(manager.ready?(q)).to be_true
+        end
+      end
+    end
+
+    describe '#generate_fieldset_mapping' do
+      it 'retuns collect mapping for fieldsets' do
+        q1 = Norikra::Query.new(:name => 'test', :expression => 'select a from sample.win:time(5 sec) where c > 1.0 and z')
+        map1 = manager.generate_fieldset_mapping(q1)
+        expect(map1.keys).to eql(['sample'])
+        expect(map1.values.size).to eql(1)
+        expect(map1.values.first).to be_a(Norikra::FieldSet)
+        expect(map1.values.first.fields.size).to eql(4)
+        expect(map1.values.first.fields.keys).to eql(['a','b','c','z']) # a,b,c is non-optional fields
+
+        q2 = Norikra::Query.new(:name => 'test', :expression => 'select a from sample.win:time(5 sec) where c > 1.0')
+        map2 = manager.generate_fieldset_mapping(q2)
+        expect(map2.keys).to eql(['sample'])
+        expect(map2.values.size).to eql(1)
+        expect(map2.values.first).to be_a(Norikra::FieldSet)
+        expect(map2.values.first.fields.size).to eql(3)
+        expect(map2.values.first.fields.keys).to eql(['a','b','c']) # a,b,c is non-optional fields
+
+        q3 = Norikra::Query.new(:name => 'test', :expression => 'select x.a, z from sample.win:time(5 sec) as x, sample_next.win:time(5 sec) as y where x.c > 1.0 and y.d > 1.0')
+        map3 = manager.generate_fieldset_mapping(q3)
+        expect(map3.keys).to eql(['sample', 'sample_next'])
+        expect(map3['sample'].fields.keys).to eql(['a','b','c','z'])
+        expect(map3['sample_next'].fields.keys).to eql(['a','b','c','d'])
+      end
+    end
+
     describe '#bind_fieldset' do
       it 'does not fail' do
         manager.bind_fieldset('sample', :query, set_query_base)

metadata

@@ -2,14 +2,14 @@
 name: norikra
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.0.5
+  version: 0.0.6
 platform: java
 authors:
 - TAGOMORI Satoshi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-07-01 00:00:00.000000000 Z
+date: 2013-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mizuno
@@ -266,6 +266,7 @@ files:
 - lib/norikra/logger_mizuno_patch.rb
 - lib/norikra/output_pool.rb
 - lib/norikra/query.rb
+- lib/norikra/query/ast.rb
 - lib/norikra/rpc.rb
 - lib/norikra/rpc/handler.rb
 - lib/norikra/rpc/http.rb