norikra 0.0.13-java → 0.0.14-java

data/lib/norikra/fieldset.rb

@@ -0,0 +1,167 @@
+require 'digest'
+require 'norikra/field'
+
+module Norikra
+  class FieldSet
+    attr_accessor :summary, :fields
+    attr_accessor :target, :level
+
+    # fieldset doesn't have container fields
+    def initialize(fields, default_optional=nil, rebounds=0)
+      @fields = {}
+      # fields.keys are raw key for container access chains
+      fields.keys.each do |key|
+        data = fields[key]
+        if data.is_a?(Norikra::Field)
+          @fields[data.name] = data
+        elsif data.is_a?(Hash)
+          type = data[:type].to_s
+          optional = data.has_key?(:optional) ? data[:optional] : default_optional
+          @fields[key.to_s] = Field.new(key.to_s, type, optional)
+        elsif data.is_a?(String) || data.is_a?(Symbol)
+          @fields[key.to_s] = Field.new(key.to_s, data.to_s, default_optional)
+        else
+          raise ArgumentError, "FieldSet.new argument class unknown: #{fields.class}"
+        end
+      end
+      self.update_summary
+
+      @target = nil
+      @level = nil
+      @rebounds = rebounds
+      @event_type_name = nil
+    end
+
+    def dup
+      fields = Hash[@fields.map{|key,field| [key, {:type => field.type, :optional => field.optional}]}]
+      self.class.new(fields, nil, @rebounds)
+    end
+
+    def self.leaves(container)
+      # returns list of [ [key-chain-items-flatten-list, value] ]
+      dig = Proc.new do |obj|
+        if obj.is_a?(Array)
+          ary = []
+          obj.each_with_index do |v,i|
+            if v.is_a?(Hash) || v.is_a?(Array)
+              ary += dig.call(v).map{|chain| [i] + chain}
+            else
+              ary.push([i, v])
+            end
+          end
+          ary
+        else # Hash
+          obj.map {|k,v|
+            if v.is_a?(Hash) || v.is_a?(Array)
+              dig.call(v).map{|chain| [k] + chain}
+            else
+              [[k, v]]
+            end
+          }.reduce(:+)
+        end
+      end
+      dig.call(container)
+    end
+
+    def self.field_names_key(data, fieldset=nil, strict=false, additional_fields=[])
+      if !fieldset && strict
+        raise RuntimeError, "strict(true) cannot be specified with fieldset=nil"
+      end
+
+      unless fieldset
+        return data.keys.sort.join(',')
+      end
+
+      keys = []
+      optionals = []
+
+      fieldset.fields.each do |key,field|
+        if field.optional?
+          optionals.push(field.name)
+        else
+          keys.push(field.name)
+        end
+      end
+      optionals += additional_fields
+
+      Norikra::FieldSet.leaves(data).each do |chain|
+        value = chain.pop
+        key = Norikra::Field.regulate_key_chain(chain).join('.')
+        unless keys.include?(key)
+          if optionals.include?(key) || (!strict && chain.size == 1)
+            keys.push(key)
+          end
+        end
+      end
+
+      keys.sort.join(',')
+    end
+
+    def field_names_key
+      self.class.field_names_key(@fields)
+    end
+
+    def update_summary
+      @summary = @fields.keys.sort.map{|k| @fields[k].escaped_name + ':' + @fields[k].type}.join(',')
+      self
+    end
+
+    def update(fields, optional_flag)
+      fields.each do |field|
+        @fields[field.name] = field.dup(optional_flag)
+      end
+      self.update_summary
+    end
+
+    #TODO: have a bug?
+    def ==(other)
+      return false if self.class != other.class
+      self.summary == other.summary
+    end
+
+    def definition
+      d = {}
+      @fields.each do |key, field|
+        d[field.escaped_name] = field.type
+      end
+      d
+    end
+
+    def subset?(other) # self is subset of other (or not)
+      (self.fields.keys - other.fields.keys).size == 0
+    end
+
+    def event_type_name
+      @event_type_name.dup
+    end
+
+    def bind(target, level, update_type_name=false)
+      @target = target
+      @level = level
+      prefix = case level
+               when :base then 'b_'
+               when :query then 'q_'
+               when :data then 'e_' # event
+               else
+                 raise ArgumentError, "unknown fieldset bind level: #{level}, for target #{target}"
+               end
+      @rebounds += 1 if update_type_name
+
+      @event_type_name = prefix + Digest::MD5.hexdigest([target, level.to_s, @rebounds.to_s, @summary].join("\t"))
+      self
+    end
+
+    def rebind(update_type_name)
+      self.dup.bind(@target, @level, update_type_name)
+    end
+
+    def format(data)
+      # all keys of data should be already known at #format (before #format, do #refer)
+      ret = {}
+      @fields.each do |key,field|
+        ret[field.escaped_name] = field.format(field.value(data))
+      end
+      ret
+    end
+  end
+end

data/lib/norikra/logger.rb

@@ -41,6 +41,8 @@ module Norikra
     @@levelnum = nil
     @@devmode = false
 
+    @@test_flag = false
+
     @@mon = Monitor.new
 
     def self.init(level, logdir, opts, devmode=false)
@@ -49,6 +51,11 @@ module Norikra
       #         else => RollingFileAppender (output: directory path)
       # http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/RollingFileAppender.html
 
+      if level.upcase == 'TEST' # with opts[:logger] as DummyLogger instance
+        level = LOG_LEVEL_DEFAULT
+        @@test_flag = true
+      end
+
       @@level = level.upcase
       raise ArgumentError, "unknown log level: #{@@level}" unless LOG_LEVELS.include?(@@level)
       @@levelnum = LOG_LEVELS.index(@@level)
@@ -67,30 +74,38 @@ module Norikra
       p.setProperty('log4j.appender.builtin.layout', 'org.apache.log4j.PatternLayout')
       p.setProperty('log4j.appender.builtin.layout.ConversionPattern', LOG_LOG4J_BUILTIN_FORMAT)
 
-      if logdir.nil?
-        p.setProperty('log4j.appender.default', 'org.apache.log4j.ConsoleAppender')
-        p.setProperty('log4j.appender.builtin', 'org.apache.log4j.ConsoleAppender')
-      else
-        # DailyRollingFileAppender ?
-        # http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/DailyRollingFileAppender.html
-        norikra_log = File.join(logdir, 'norikra.log')
-        p.setProperty('log4j.appender.default', 'org.apache.log4j.RollingFileAppender')
-        p.setProperty('log4j.appender.default.File', norikra_log)
-        p.setProperty('log4j.appender.default.MaxFileSize', opts[:filesize] || LOGFILE_DEFAULT_MAX_SIZE)
-        p.setProperty('log4j.appender.default.MaxBackupIndex', opts[:backups].to_s || LOGFILE_DEFAULT_MAX_BACKUP_INDEX.to_s)
-
-        builtin_log = File.join(logdir, 'builtin.log')
-        p.setProperty('log4j.appender.builtin', 'org.apache.log4j.RollingFileAppender')
-        p.setProperty('log4j.appender.builtin.File', builtin_log)
-        p.setProperty('log4j.appender.builtin.MaxFileSize', opts[:filesize] || LOGFILE_DEFAULT_MAX_SIZE)
-        p.setProperty('log4j.appender.builtin.MaxBackupIndex', opts[:backups].to_s || LOGFILE_DEFAULT_MAX_BACKUP_INDEX.to_s)
+      unless @@test_flag
+        if logdir.nil?
+          p.setProperty('log4j.appender.default', 'org.apache.log4j.ConsoleAppender')
+          p.setProperty('log4j.appender.builtin', 'org.apache.log4j.ConsoleAppender')
+        else
+          # DailyRollingFileAppender ?
+          # http://logging.apache.org/log4j/1.2/apidocs/org/apache/log4j/DailyRollingFileAppender.html
+          norikra_log = File.join(logdir, 'norikra.log')
+          p.setProperty('log4j.appender.default', 'org.apache.log4j.RollingFileAppender')
+          p.setProperty('log4j.appender.default.File', norikra_log)
+          p.setProperty('log4j.appender.default.MaxFileSize', opts[:filesize] || LOGFILE_DEFAULT_MAX_SIZE)
+          p.setProperty('log4j.appender.default.MaxBackupIndex', opts[:backups].to_s || LOGFILE_DEFAULT_MAX_BACKUP_INDEX.to_s)
+
+          builtin_log = File.join(logdir, 'builtin.log')
+          p.setProperty('log4j.appender.builtin', 'org.apache.log4j.RollingFileAppender')
+          p.setProperty('log4j.appender.builtin.File', builtin_log)
+          p.setProperty('log4j.appender.builtin.MaxFileSize', opts[:filesize] || LOGFILE_DEFAULT_MAX_SIZE)
+          p.setProperty('log4j.appender.builtin.MaxBackupIndex', opts[:backups].to_s || LOGFILE_DEFAULT_MAX_BACKUP_INDEX.to_s)
+        end
+        p.setProperty('log4j.rootLogger', "#{@@level},default")
+        org.apache.log4j.PropertyConfigurator.configure(p)
+
+        @@logger = Logger.new('norikra.log')
+
+      else # for test(rspec)
+        p.setProperty('log4j.appender.default', 'org.apache.log4j.varia.NullAppender')
+        p.setProperty('log4j.appender.builtin', 'org.apache.log4j.varia.NullAppender')
+        p.setProperty('log4j.rootLogger', "#{@@level},default")
+        org.apache.log4j.PropertyConfigurator.configure(p)
+        @@logger = opts[:logger]
       end
 
-      p.setProperty('log4j.rootLogger', "#{@@level},default")
-      org.apache.log4j.PropertyConfigurator.configure(p)
-
-      @@logger = Logger.new('norikra.log')
-
       @@devmode = devmode
     end
 

data/lib/norikra/query.rb

@@ -6,6 +6,7 @@ require 'esper/lib/cglib-nodep-2.2.jar'
 
 require 'norikra/error'
 require 'norikra/query/ast'
+require 'norikra/field'
 
 module Norikra
   class Query
@@ -19,6 +20,7 @@ module Norikra
       @fieldsets = {} # { target => fieldset }
       @ast = nil
       @targets = nil
+      @aliases = nil
       @subqueries = nil
       @fields = nil
     end
@@ -27,16 +29,6 @@ module Norikra
       self.class.new(:name => @name, :group => @group, :expression => @expression.dup)
     end
 
-    def dup_with_stream_name(actual_name)
-      first_target = self.targets.first
-      query = self.dup
-      query.expression = self.expression.gsub(/(\s[Ff][Rr][Oo][Mm]\s+)#{first_target}(\.|\s)/, '\1' + actual_name + '\2')
-      if query.targets.first != actual_name
-        raise RuntimeError, 'failed to replace query target into stream name:' + self.expression
-      end
-      query
-    end
-
     def to_hash
       {'name' => @name, 'group' => @group, 'expression' => @expression, 'targets' => self.targets}
     end
@@ -47,6 +39,12 @@ module Norikra
       @targets
     end
 
+    def aliases
+      return @aliases if @aliases
+      @aliases = (self.ast.listup(:stream).map(&:alias) + self.subqueries.map(&:aliases).flatten).sort.uniq
+      @aliases
+    end
+
     def subqueries
       return @subqueries if @subqueries
       @subqueries = self.ast.listup(:subquery).map{|n| Norikra::SubQuery.new(n)}
@@ -82,7 +80,8 @@ module Norikra
         field_bag.push(subquery.explore(fields.keys, alias_map))
       end
 
-      self.ast.fields(default_target).each do |field_def|
+      known_targets_aliases = fields.keys + alias_map.keys
+      self.ast.fields(default_target, known_targets_aliases).each do |field_def|
         f = field_def[:f]
         all.push(f)
 
@@ -145,6 +144,68 @@ module Norikra
       raise Norikra::QueryError, e.message
     end
 
+    def self.rewrite_query(statement_model, mapping)
+      rewrite_event_type_name(statement_model, mapping)
+      rewrite_event_field_name(statement_model, mapping)
+    end
+
+    def self.rewrite_event_field_name(statement_model, mapping)
+      # mapping: {target_name => query_event_type_name}
+      #  mapping is for target name rewriting of fully qualified field name access
+
+
+      # model.getFromClause.getStreams[0].getViews[0].getParameters[0].getPropertyName
+
+      # model.getSelectClause.getSelectList[0].getExpression.getPropertyName
+      # model.getSelectClause.getSelectList[0].getExpression.getChildren[0].getPropertyName #=> 'field.key1.$0'
+
+      # model.getWhereClause.getChildren[1].getChildren[0].getPropertyName #=> 'field.key1.$1'
+      # model.getWhereClause.getChildren[2].getChildren[0].getChain[0].getName #=> 'opts.num.$0' from opts.num.$0.length()
+
+      query = Norikra::Query.new(:expression => statement_model.toEPL)
+      targets = query.targets
+      fqfs_prefixes = targets + query.aliases
+
+      default_target = (targets.size == 1 ? targets.first : nil)
+
+      rewrite_name = lambda {|node,getter,setter|
+        name = node.send(getter)
+        if name && name.index('.')
+          prefix = nil
+          body = nil
+          first_part = name.split('.').first
+          if fqfs_prefixes.include?(first_part) or mapping.has_key?(first_part) # fully qualified field specification
+            prefix = first_part
+            if mapping[prefix]
+              prefix = mapping[prefix]
+            end
+            body = name.split('.')[1..-1].join('.')
+          elsif default_target # default target field (outside of join context)
+            body = name
+          else
+            raise Norikra::QueryError, "target cannot be determined for field '#{name}'"
+          end
+          encoded = (prefix ? "#{prefix}." : "") + Norikra::Field.escape_name(body)
+          node.send(setter, encoded)
+        end
+      }
+
+      rewriter = lambda {|node|
+        if node.respond_to?(:getPropertyName)
+          rewrite_name.call(node, :getPropertyName, :setPropertyName)
+        elsif node.respond_to?(:getChain)
+          node.getChain.each do |chain|
+            rewrite_name.call(chain, :getName, :setName)
+          end
+        end
+      }
+      recaller = lambda {|node|
+        Norikra::Query.rewrite_event_field_name(node.getModel, mapping)
+      }
+
+      traverse_fields(rewriter, recaller, statement_model)
+    end
+
     def self.rewrite_event_type_name(statement_model, mapping)
       # mapping: {target_name => query_event_type_name}
 
@@ -166,19 +227,96 @@ module Norikra
         stream.getFilter.setEventTypeName(mapping[target_name])
       end
 
+      rewriter = lambda {|node|
+        # nothing for query expression clauses
+      }
+      recaller = lambda {|node|
+        Norikra::Query.rewrite_event_type_name(node.getModel, mapping)
+      }
+      traverse_fields(rewriter, recaller, statement_model)
+    end
+
+    # model.methods.select{|m| m.to_s.start_with?('get')}
+    # :getContextName,
+    # :getCreateContext,
+    # :getCreateDataFlow,
+    # :getCreateExpression,
+    # :getCreateIndex,
+    # :getCreateSchema,
+    # :getCreateVariable,
+    # :getCreateWindow,
+    # :getExpressionDeclarations,
+    # :getFireAndForgetClause,
+    # :getForClause,
+    # (*) :getFromClause,
+    # :getGroupByClause,
+    # :getHavingClause,
+    # :getInsertInto,
+    # :getMatchRecognizeClause,
+    # :getOnExpr,
+    # :getOrderByClause,
+    # :getOutputLimitClause,
+    # :getRowLimitClause,
+    # :getScriptExpressions,
+    # (*) :getSelectClause,
+    # :getTreeObjectName,
+    # :getUpdateClause,
+    # (*) :getWhereClause,
+
+    def self.traverse_fields(rewriter, recaller, statement_model)
+      #NOTICE: SQLStream is not supported yet.
+      #TODO: other clauses with fields, especially: OrderBy, Having, GroupBy, For
+
       dig = lambda {|node|
+        rewriter.call(node)
+
         if node.is_a?(Java::ComEspertechEsperClientSoda::SubqueryExpression)
-          Norikra::Query.rewrite_event_type_name(node.getModel, mapping)
-        elsif node.getChildren.size > 0
+          recaller.call(node)
+        end
+        if node.respond_to?(:getFilter)
+          dig.call(node.getFilter)
+        end
+        if node.respond_to?(:getChildren)
           node.getChildren.each do |c|
             dig.call(c)
           end
         end
+        if node.respond_to?(:getParameters)
+          node.getParameters.each do |p|
+            dig.call(p)
+          end
+        end
+        if node.respond_to?(:getChain)
+          node.getChain.each do |c|
+            dig.call(c)
+          end
+        end
       }
 
+      statement_model.getFromClause.getStreams.each do |stream|
+        if stream.respond_to?(:getExpression) # PatternStream < ProjectedStream
+          dig.call(stream.getExpression)
+        end
+        if stream.respond_to?(:getFilter) # Filter < ProjectedStream
+          dig.call(stream.getFilter.getFilter) #=> Expression
+        end
+        if stream.respond_to?(:getParameterExpressions) # MethodInvocationStream
+          dig.call(stream.getParameterExpressions)
+        end
+        if stream.respond_to?(:getViews) # ProjectedStream
+          stream.getViews.each do |view|
+            view.getParameters.each do |parameter|
+              dig.call(parameter)
+            end
+          end
+        end
+      end
+
       if statement_model.getSelectClause
         statement_model.getSelectClause.getSelectList.each do |item|
-          dig.call(item.getExpression)
+          if item.respond_to?(:getExpression)
+            dig.call(item.getExpression)
+          end
         end
       end