norikra 0.1.4-java → 0.1.5-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d5ba6d34ed7bba46947b8434b0ac32b8cb7ba2d
-  data.tar.gz: 2c6a5434cfc175aab9de2f89d3ab10198a75a06f
+  metadata.gz: 30a26c93805e62b832b47ade931907e800c35c01
+  data.tar.gz: f8fe030eacb414c84aeebd01b40351965391a53b
 SHA512:
-  metadata.gz: 48a696a0f5611dc7657c82e1e119a10355da055eadfd4417b0456508a345ce4a18bc8617bd4aa46435cf49c85c7e0863ef42e01747923de89a0695e9ea7a86db
-  data.tar.gz: 9b04d5197dd10a38c6d66dd09ae580509f2469c1d6727f774e203e61400796accc5c3de164df2262b0d0521959c49bb80bd8241648e339f234d75881b8994a71
+  metadata.gz: 243a44cf8003f1843df7137c4c78c9e72765e3da51f7f33e961531cd9017c612389d44051fdf2a1cf98c57149b97a9978c250dacc98a4cd324206e9866d4cdbd
+  data.tar.gz: 7351f6ee5139a5323adfb9708abe8291f253e47a783d2efcc4fdf682c35f706ea48ae7c179dcfa8bb6bea73137eb95bdd2b578c164ec4af4de24e237cb2efe9b

data/README.md

@@ -69,6 +69,29 @@ See: http://norikra.github.io/
 
 ## Changes
 
+* v0.1.5:
+ * Add RPC port monitoring handler (GET /)
+ * Add `Pattern` support
+ * Changes for GC/JVM
+   * Server starts with default(recommended) JVM options for GC configurations
+   * Add option not to use default JVM options
+   * Add option to configure JVM to print GC logs
+ * Stats file changes
+   * Changed to use SIGUSR2 (SIGUSR1 is used by JVM on linux platform)
+   * Changed no to contain server process configurations (ports, threads, logs ...)
+ * WebUI improvements
+   * Add button to close target on WebUI
+   * Add link to download stats file
+ * Fix bugs
+   * Fieldname handlings about names with non-alphabetic chars
+   * Query parser bug for some built-in functions (ex: `rate(10)`)
+   * Write stats file in more safe way
+   * WebUI: Incorrect memory bar
+* v0.1.4:
+ * Stat dump option on runtime per specified intervals (`--dump-stat-interval`)
+ * Stat dump on demand by SIGUSR1
+ * `norikra-client event see` command (and API call) to get query results, but not remove it
+ * Last modified time for each target on WebUI
 * v0.1.3:
  * Fix critical bug about query de-registration
 * v0.1.2:

data/lib/norikra/cli.rb

@@ -13,11 +13,12 @@ module Norikra
         ### Server options
         option :host, :type => :string, :default => nil, :aliases => "-H", :desc => 'host address that server listen [0.0.0.0]'
         option :port, :type => :numeric, :default => nil, :aliases => "-P", :desc => 'port that server uses [26571]'
+        option :'ui-port', :type => :numeric, :default => nil, :desc => 'port that Web UI server uses [26578]'
 
         option :stats, :type => :string, :default => nil, :aliases => "-s", \
-                       :desc => 'status file path to load/dump targets, queries and server configurations [none]'
+                       :desc => 'status file path to load/dump targets and queries [none]'
         option :'suppress-dump-stat', :type => :boolean, :default => false, \
-                       :desc => 'specify not to update stat file with updated targets/queries/configurations on runtime [false]'
+                       :desc => 'specify not to update stat file with updated targets/queries on runtime [false]'
         option :'dump-stat-interval', :type => :numeric, :default => nil, \
                        :desc => 'interval(seconds) of status file dumps on runtime [none (on shutdown only)]'
 
@@ -29,6 +30,10 @@ module Norikra
         option :outfile, :type => :string, :default => nil, \
                          :desc => "stdout redirect file when daemonized [${logdir}/norikra.out]"
 
+        ### JVM options
+        option :'bare-jvm', :type => :boolean, :default => false, :desc => "use JVM without any recommended options"
+        option :'gc-log', :type => :string, :default => nil, :desc => "output gc logs on specified file path"
+
         ### Performance options
         # performance predefined configuration sets
         option :micro, :type => :boolean, :default => false, \
@@ -71,6 +76,40 @@ module Norikra
   end
 
   class CLI < Thor
+    # JVM defaults w/ ConcurrentGC:
+    #    java -XX:+UseConcMarkSweepGC -XX:+PrintFlagsFinal -version
+    #    (jruby does not modify flags)
+    #
+    #   CMSIncrementalMode: not recommended in Java8
+    #
+    #   CMSInitiatingOccupancyFraction: CMS threshold
+    #   CMSInitiatingOccupancyFraction = 100 - MinHeapFreeRatio + CMSTriggerRatio * MinHeapFreeRatio / 100
+    #     default: MinHeapFreeRatio=40 CMSTriggerRatio=80 -> CMSInitiatingOccupancyFraction=92
+    #
+    #   NewRatio=7 (New:Old = 1:7)
+    #   InitialSurvivorRatio=8     MinSurvivorRatio=3     SurvivorRatio=8 (Eden:Survivor0:survivor1 = 8:1:1)
+    #
+    #   MaxTenuringThreshold=4
+    #   TargetSurvivorRatio=50
+    #    (InitialTenuringThreshold=7 is for Parallel GC/UseAdaptiveSizePolicy)
+    #
+    #   SoftRefLRUPolicyMSPerMB=1000
+    #    ( gc_interval > free_heap * ms_per_mb : clear softref )
+
+    JVM_OPTIONS = [
+      '-XX:-UseGCOverheadLimit',
+      '-XX:+UseConcMarkSweepGC', '-XX:+UseCompressedOops',
+      '-XX:CMSInitiatingOccupancyFraction=70', '-XX:+UseCMSInitiatingOccupancyOnly',
+      '-XX:NewRatio=1',
+      '-XX:SurvivorRatio=2', '-XX:MaxTenuringThreshold=15', '-XX:TargetSurvivorRatio=80',
+      '-XX:SoftRefLRUPolicyMSPerMB=200',
+    ]
+
+    JVM_GC_OPTIONS = [
+      '-verbose:gc', '-XX:+PrintGCDetails', '-XX:+PrintGCDateStamps',
+      '-XX:+HeapDumpOnOutOfMemoryError',
+    ]
+
     ### 'start' and 'serverprocess' have almost same option set (for parse/help)
     ### DIFF: jvm options (-X)
     Norikra::CLIUtil.register_common_start_options(self)
@@ -85,11 +124,22 @@ module Norikra
       ARGV.shift # shift head "start"
 
       argv = ["serverproc"]
-      jruby_options = ['-J-server', '-J-XX:-UseGCOverheadLimit']
+      jruby_options = ['-J-server']
+
+      unless options[:'bare-jvm']
+        jruby_options += JVM_OPTIONS.map{|opt| '-J' + opt }
+      end
+
+      if options[:'gc-log']
+        jruby_options += JVM_GC_OPTIONS.map{|opt| '-J' + opt }
+        jruby_options.push "-J-Xloggc:#{options[:'gc-log']}"
+      end
 
       ARGV.each do |arg|
         if arg =~ /^-X(.+)$/
           jruby_options.push('-J-X' + $1)
+        elsif arg =~ /^-verbose:gc$/
+          jruby_options.push('-J-verbose:gc')
         else
           argv.push(arg)
         end
@@ -171,7 +221,13 @@ module Norikra
         level: loglevel, dir: options[:logdir], filesize: options[:'log-filesize'], backups: options[:'log-backups'],
       }
 
-      server = Norikra::Server.new( options[:host], options[:port], conf )
+      server_options = {
+        host: options[:host],
+        port: options[:port],
+        ui_port: options[:'ui-port'],
+      }
+
+      server = Norikra::Server.new( server_options, conf )
       server.run
       server.shutdown
     end

data/lib/norikra/field.rb

@@ -218,7 +218,23 @@ module Norikra
             raise ArgumentError, "container_accessor must be a String or Interger, but #{accessor.class.to_s}"
           end
           if v.is_a?(Hash)
-            v[accessor] || v[accessor.to_s] # hash[string] is valid, and hash[int] is also valid, and hash["int"] is valid too.
+            # v[accessor] || v[accessor.to_s]
+            if v.has_key?(accessor)
+              v[accessor]
+            elsif v.has_key?(accessor.to_s)
+              v[accessor.to_s]
+            elsif accessor.is_a?(String)
+              val = v.keys.reduce(nil) do |r, key|
+                if key.is_a?(String)
+                  r || ( key.gsub(/[^$_a-zA-Z0-9]/, '_') == accessor.to_s ? v[key] : nil )
+                else
+                  r
+                end
+              end
+              val # valid value or nil
+            else # v does not have key "#{accessor}" and accessor is Fixnum
+              nil
+            end
           elsif v.is_a?(Array)
             if accessor.is_a?(Fixnum)
               v[accessor]

data/lib/norikra/query.rb

@@ -58,13 +58,13 @@ module Norikra
 
     def targets
       return @targets if @targets
-      @targets = (self.ast.listup(:stream).map(&:target) + self.subqueries.map(&:targets).flatten).sort.uniq
+      @targets = (self.ast.listup(:stream).map(&:targets).flatten + self.subqueries.map(&:targets).flatten).sort.uniq
       @targets
     end
 
     def aliases
       return @aliases if @aliases
-      @aliases = (self.ast.listup(:stream).map(&:alias) + self.subqueries.map(&:aliases).flatten).sort.uniq
+      @aliases = (self.ast.listup(:stream).map{|s| s.aliases.map(&:first) }.flatten + self.subqueries.map(&:aliases).flatten).sort.uniq
       @aliases
     end
 
@@ -81,10 +81,12 @@ module Norikra
       all = []
       unknowns = []
       self.ast.listup(:stream).each do |node|
-        if node.alias
-          alias_map[node.alias] = node.target
+        node.aliases.each do |alias_name, target|
+          alias_map[alias_name] = target
+        end
+        node.targets.each do |target|
+          fields[target] ||= []
         end
-        fields[node.target] = []
       end
 
       dup_aliases = (alias_map.keys & fields.keys)

data/lib/norikra/query/ast.rb

@@ -79,34 +79,84 @@ module Norikra
     #     ["LIB_FUNC_CHAIN", ["LIB_FUNCTION", "opts.num.seq", "length", "("]],
     #     "0"]]]]
 
+    ### SELECT a.name, a.content, b.content
+    ### FROM pattern[every a=EventA -> b=EventA(name = a.name, type = 'TYPE') WHERE timer:within(1 min)].win:time(2 hour)
+    ### WHERE a.source in ('A', 'B')
+
+    # ["EPL_EXPR",
+    #   ["SELECTION_EXPR",
+    #     ["SELECTION_ELEMENT_EXPR",
+    #       ["EVENT_PROP_EXPR",
+    #         ["EVENT_PROP_SIMPLE", "a"],
+    #         ["EVENT_PROP_SIMPLE", "name"]]],
+    #     ["SELECTION_ELEMENT_EXPR",
+    #       ["EVENT_PROP_EXPR",
+    #         ["EVENT_PROP_SIMPLE", "a"],
+    #         ["EVENT_PROP_SIMPLE", "content"]]],
+    #     ["SELECTION_ELEMENT_EXPR",
+    #       ["EVENT_PROP_EXPR",
+    #         ["EVENT_PROP_SIMPLE", "b"],
+    #         ["EVENT_PROP_SIMPLE", "content"]]]],
+    #   ["STREAM_EXPR",
+    #     ["PATTERN_INCL_EXPR",
+    #       ["FOLLOWED_BY_EXPR",
+    #         ["FOLLOWED_BY_ITEM", ["every", ["PATTERN_FILTER_EXPR", "a", "EventA"]]],
+    #         ["FOLLOWED_BY_ITEM",
+    #           ["GUARD_EXPR",
+    #             ["PATTERN_FILTER_EXPR",
+    #               "b",
+    #               "EventA",
+    #               ["EVAL_EQUALS_EXPR",
+    #                 ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "name"]],
+    #                 ["EVENT_PROP_EXPR",
+    #                   ["EVENT_PROP_SIMPLE", "a"],
+    #                   ["EVENT_PROP_SIMPLE", "name"]]],
+    #               ["EVAL_EQUALS_EXPR",
+    #                 ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "type"]],
+    #                 "'TYPE'"]],
+    #             "timer",
+    #             "within",
+    #             ["TIME_PERIOD", ["MINUTE_PART", "1"]]]]]],
+    #     ["VIEW_EXPR", "win", "time", ["TIME_PERIOD", ["HOUR_PART", "2"]]]],
+    #   ["WHERE_EXPR",
+    #     ["in",
+    #       ["EVENT_PROP_EXPR",
+    #         ["EVENT_PROP_SIMPLE", "a"],
+    #         ["EVENT_PROP_SIMPLE", "source"]],
+    #       "(",
+    #       "'A'",
+    #       "'B'",
+    #       ")"]]]
+
     def astnode(tree)
       children = if tree.children
                    tree.children.map{|c| astnode(c)}
                  else
                    []
                  end
-      case tree.text
-      when 'EVENT_PROP_EXPR'
-        ASTEventPropNode.new(tree.text, children)
-      when 'SELECTION_ELEMENT_EXPR'
-        ASTSelectionElementNode.new(tree.text, children)
-      when 'LIB_FUNCTION'
-        ASTLibFunctionNode.new(tree.text, children)
-      when 'STREAM_EXPR'
-        ASTStreamNode.new(tree.text, children)
-      when 'SUBSELECT_EXPR'
-        ASTSubSelectNode.new(tree.text, children)
+      cls = case tree.text
+            when 'EVENT_PROP_EXPR' then ASTEventPropNode
+            when 'SELECTION_ELEMENT_EXPR' then ASTSelectionElementNode
+            when 'LIB_FUNCTION' then ASTLibFunctionNode
+            when 'STREAM_EXPR' then ASTStreamNode
+            when 'PATTERN_FILTER_EXPR' then ASTPatternNode
+            when 'SUBSELECT_EXPR' then ASTSubSelectNode
+            else ASTNode
+            end
+      if cls.respond_to?(:generate)
+        cls.generate(tree.text, children, tree)
       else
-        ASTNode.new(tree.text, children)
+        cls.new(tree.text, children, tree)
       end
     end
 
     class ASTNode
-      attr_accessor :name, :children
+      attr_accessor :name, :children, :tree
 
-      def initialize(name, children)
+      def initialize(name, children, tree)
         @name = name
         @children = children
+        @tree = tree
       end
 
       def nodetype?(*sym)
@@ -132,7 +182,12 @@ module Norikra
         nil
       end
 
-      def listup(type) # search all nodes that has 'type'
+      def listup(*type) # search all nodes that has 'type'
+        if type.size > 1
+          return type.map{|t| self.listup(t) }.reduce(&:+)
+        end
+        type = type.first
+
         result = []
         result.push(self) if type.is_a?(String) && @name == type || nodetype?(type)
 
@@ -184,15 +239,22 @@ module Norikra
     end
 
     class ASTLibFunctionNode < ASTNode # LIB_FUNCTION
-      # "now()"            => ["LIB_FUNCTION", "now", "("]
-      # "hoge.length()"    => ["LIB_FUNCTION", "hoge", "length", "("]
-      # "hoge.substr(0)"   => ["LIB_FUNCTION", "hoge", "substr", "0", "("]
-      # "substr(10,0)"     => ["LIB_FUNCTION", "substr", "10", "0", "("]
-      # "hoge.substr(0,8)" => ["LIB_FUNCTION", "hoge", "substr", "0", "8", "("]
+      ### foo is function
+      # "foo()"     => ["LIB_FUNCTION", "foo", "("]
+      # "foo(10)"   => ["LIB_FUNCTION", "foo", "10", "("]
+      # "foo(10,0)" => ["LIB_FUNCTION", "foo", "10", "0", "("]
+      # "foo(bar)"  => ["LIB_FUNCTION", "foo", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "bar"]], "("]
+
+      ### foo is property
+      # "foo.bar()"    => ["LIB_FUNCTION", "foo", "bar", "("]
+      # "foo.bar(0)"   => ["LIB_FUNCTION", "foo", "bar", "0", "("]
+      # "foo.bar(0,8)" => ["LIB_FUNCTION", "foo", "bar", "0", "8", "("]
 
-      # "opts.num.$0.length()" => ["LIB_FUNCTION", "opts.num.$0", "length", "("]
+      ### nested field access
+      # "foo.bar.$0.baz()" => ["LIB_FUNCTION", "foo.bar.$0", "baz", "("]
 
-      # "max(size)"             => ["LIB_FUNCTION", "max", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]], "("]
+      # 2nd child is bare-word (like [a-z][a-z0-9]*) -> this is function -> 1st child is receiver -> property
+      # 2nd child is literal or property or none -> 1st child is built-in function
 
       def nodetype?(*sym)
         sym.include?(:lib) || sym.include?(:libfunc)
@@ -207,7 +269,7 @@ module Norikra
           # first element should be func name if second element is property, library call or subqueries
           self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
 
-        elsif @children[1].name =~ /^(-)?\d+(\.\d+)$/ || @children[1].name =~ /^'[^']*'$/ || @children[1].name =~ /^"[^"]*"$/
+        elsif @children[1].name =~ /^(-)?\d+(\.\d+)?$/ || @children[1].name =~ /^'[^']*'$/ || @children[1].name =~ /^"[^"]*"$/
           # first element should be func name if secod element is number/string literal
           self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
 
@@ -216,7 +278,7 @@ module Norikra
           self.listup('EVENT_PROP_EXPR').map{|c| c.fields(default_target, known_targets_aliases)}.reduce(&:+) || []
 
         else
-          # first element may be property
+          # first element may be property ########## or function
           #  * simple 'fieldname.funcname()'
           #  * fully qualified 'target.fieldname.funcname()'
           #  * simple/fully-qualified container field access 'fieldname.key.$0.funcname()' or 'target.fieldname.$1.funcname()'
@@ -237,31 +299,146 @@ module Norikra
     end
 
     class ASTStreamNode < ASTNode # STREAM_EXPR
+      def self.generate(name, children, tree)
+        if children.first.name == 'EVENT_FILTER_EXPR'
+          ASTStreamEventNode.new(name, children, tree)
+        elsif children.first.name == 'PATTERN_INCL_EXPR'
+          ASTStreamPatternNode.new(name, children, tree)
+        else
+          raise "unexpected stream node type! report to norikra developer!: #{children.map(&:name).join(',')}"
+        end
+      end
+
+      def nodetype?(*sym)
+        sym.include?(:stream)
+      end
+
+      def targets
+        # ["TARGET_NAME"]
+        raise NotImplementedError, "ASTStreamNode#targets MUST be overridden by subclass"
+      end
+
+      def aliases
+        # [ [ "ALIAS_NAME", "TARGET_NAME" ], ... ]
+        raise NotImplementedError, "ASTStreamNode#aliases MUST be overridden by subclass"
+      end
+
+      def fields(default_target=nil, known_targets_aliases=[])
+        raise NotImplementedError, "ASTStreamNode#fields MUST be overridden by subclass"
+      end
+    end
+
+    class ASTStreamEventNode < ASTStreamNode
+      ##### from stream_def [as name] [unidirectional] [retain-union | retain-intersection],
+      #####      [ stream_def ... ]
+      #
+      # single Event stream name ( ex: FROM events.win:time(...) AS e )
+      #
       #  ["STREAM_EXPR",
       #   ["EVENT_FILTER_EXPR", "FraudWarningEvent"],
       #   ["VIEW_EXPR", "win", "keepall"],
       #   "fraud"],
+      #
       #  ["STREAM_EXPR",
       #   ["EVENT_FILTER_EXPR",
       #    "PINChangeEvent",
       #    [">", ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "size"]], "10"]],
       #   ["VIEW_EXPR", "win", "time", ["TIME_PERIOD", ["SECOND_PART", "20"]]]],
 
+      NON_ALIAS_NODES = ['EVENT_FILTER_EXPR','VIEW_EXPR','unidirectional','retain-union','retain-intersection']
+
+      def targets
+        [self.find('EVENT_FILTER_EXPR').child.name]
+      end
+
+      def aliases
+        alias_nodes = children.select{|n| not NON_ALIAS_NODES.include?(n.name) }
+        if alias_nodes.size < 1
+          []
+        elsif alias_nodes.size > 1
+          raise "unexpected FROM clause (includes 2 or more alias words): #{alias_nodes.map(&:name).join(',')}"
+        else
+          [ [ alias_nodes.first.name, self.targets.first ] ]
+        end
+      end
+
+      def fields(default_target=nil, known_targets_aliases=[])
+        this_target = self.targets.first
+        self.listup('EVENT_PROP_EXPR').map{|p| p.fields(this_target,known_targets_aliases)}.reduce(&:+) || []
+      end
+    end
+
+    class ASTStreamPatternNode < ASTStreamNode
+      ## MEMO: Pattern itself can have alias name, but it makes no sense. So we ignore it.
+      ##       ('x' is ignored): pattern [... ] AS x
+      #
+      # pattern ( ex: FROM pattern[ every a=events1 -> b=Events1(name=a.name, type='T') where timer:within(1 min) ].win:time(2 hour) )
+      #
+      # ["STREAM_EXPR",
+      #   ["PATTERN_INCL_EXPR",
+      #     ["FOLLOWED_BY_EXPR",
+      #       ["FOLLOWED_BY_ITEM", ["every", ["PATTERN_FILTER_EXPR", "a", "EventA"]]],
+      #       ["FOLLOWED_BY_ITEM",
+      #         ["GUARD_EXPR",
+      #           ["PATTERN_FILTER_EXPR",
+      #             "b",
+      #             "EventA",
+      #             ["EVAL_EQUALS_EXPR",
+      #               ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "name"]],
+      #               ["EVENT_PROP_EXPR",
+      #                 ["EVENT_PROP_SIMPLE", "a"],
+      #                 ["EVENT_PROP_SIMPLE", "name"]]],
+      #             ["EVAL_EQUALS_EXPR",
+      #               ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "type"]],
+      #               "'TYPE'"]],
+      #           "timer",
+      #           "within",
+      #           ["TIME_PERIOD", ["MINUTE_PART", "1"]]]]]],
+      #   ["VIEW_EXPR", "win", "time", ["TIME_PERIOD", ["HOUR_PART", "2"]]]],
+
+      def targets
+        self.listup(:pattern).map(&:target)
+      end
+
+      def aliases
+        self.listup(:pattern).map{|p| [ p.alias, p.target ] }
+      end
+
+      def fields(default_target=nil, known_targets_aliases=[])
+        self.listup(:pattern).map{|p| p.fields(default_target, known_targets_aliases) }.reduce(&:+) || []
+      end
+    end
+
+    class ASTPatternNode < ASTNode
+      # ["PATTERN_FILTER_EXPR", "a", "EventA"]
+      #
+      # ["PATTERN_FILTER_EXPR",
+      #   "b",
+      #   "EventA",
+      #   ["EVAL_EQUALS_EXPR",
+      #     ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "name"]],
+      #     ["EVENT_PROP_EXPR",
+      #       ["EVENT_PROP_SIMPLE", "a"],
+      #       ["EVENT_PROP_SIMPLE", "name"]]],
+      #   ["EVAL_EQUALS_EXPR",
+      #     ["EVENT_PROP_EXPR", ["EVENT_PROP_SIMPLE", "type"]],
+      #     "'TYPE'"]],
+
       def nodetype?(*sym)
-        sym.include?(:stream)
+        sym.include?(:pattern)
       end
 
       def target
-        self.find('EVENT_FILTER_EXPR').child.name
+        children[1].name
       end
 
       def alias
-        @children.last.children.size < 1 ? @children.last.name : nil
+        children[0].name
       end
 
       def fields(default_target=nil, known_targets_aliases=[])
         this_target = self.target
-        self.listup('EVENT_PROP_EXPR').map{|p| p.fields(this_target,known_targets_aliases)}.reduce(&:+) || []
+        self.listup('EVENT_PROP_EXPR').map{|p| p.fields(this_target, known_targets_aliases) }.reduce(&:+) || []
       end
     end
 

data/lib/norikra/rpc/gatekeeper.rb

@@ -0,0 +1,18 @@
+require 'norikra/logger'
+include Norikra::Log
+
+require 'norikra/rpc'
+
+class Norikra::RPC::Gatekeeper
+  # handler of "GET /" to respond for monitor tools
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    if env["REQUEST_METHOD"] == 'GET' && env["PATH_INFO"] == '/'
+      return [ 200, {'Content-Type' => 'text/plain'}, ['Norikra alive!'] ]
+    end
+    return @app.call(env)
+  end
+end

data/lib/norikra/rpc/http.rb

@@ -3,6 +3,7 @@ require 'rack/builder'
 require 'msgpack-rpc-over-http-jruby'
 
 require_relative 'handler'
+require_relative 'gatekeeper'
 
 require 'norikra/logger'
 include Norikra::Log
@@ -25,6 +26,7 @@ module Norikra::RPC
       @threads = opts[:threads] || DEFAULT_THREADS
       handler = Norikra::RPC::Handler.new(@engine)
       @app = Rack::Builder.new {
+        use Norikra::RPC::Gatekeeper
         run MessagePack::RPCOverHTTP::Server.app(handler)
       }
     end

data/lib/norikra/server.rb

@@ -43,13 +43,13 @@ module Norikra
       :web => { threads: 2 },
     }
 
-    def self.threading_configuration(conf, stats)
+    def self.threading_configuration(conf)
       threads = case conf[:predefined]
                 when :micro then MICRO_PREDEFINED
                 when :small then SMALL_PREDEFINED
                 when :middle then MIDDLE_PREDEFINED
                 when :large then LARGE_PREDEFINED
-                else (stats ? stats.threads : MICRO_PREDEFINED)
+                else MICRO_PREDEFINED
                 end
       [:inbound, :outbound, :route_exec, :timer_exec].each do |type|
         [:threads, :capacity].each do |item|
@@ -61,15 +61,15 @@ module Norikra
       threads
     end
 
-    def self.log_configuration(conf, stats)
-      logconf = stats ? stats.log : { level: nil, dir: nil, filesize: nil, backups: nil }
+    def self.log_configuration(conf)
+      logconf = { level: nil, dir: nil, filesize: nil, backups: nil }
       [:level, :dir, :filesize, :backups].each do |sym|
         logconf[sym] = conf[sym] if conf[sym]
       end
       logconf
     end
 
-    def initialize(host, port, conf={})
+    def initialize(server_options, conf={})
       if conf[:daemonize]
         outfile_path = conf[:daemonize][:outfile] || File.join(conf[:log][:dir], 'norikra.out')
         Dir.chdir("/")
@@ -89,12 +89,12 @@ module Norikra
                  nil
                end
 
-      @host = host || (@stats ? @stats.host : nil)
-      @port = port || (@stats ? @stats.port : nil)
-      @ui_port = @stats ? @stats.ui_port : nil
+      @host = server_options[:host] || Norikra::RPC::HTTP::DEFAULT_LISTEN_HOST
+      @port = server_options[:port] || Norikra::RPC::HTTP::DEFAULT_LISTEN_PORT
+      @ui_port = server_options[:ui_port] || Norikra::WebUI::HTTP::DEFAULT_LISTEN_PORT
 
-      @thread_conf = self.class.threading_configuration(conf[:thread], @stats)
-      @log_conf = self.class.log_configuration(conf[:log], @stats)
+      @thread_conf = self.class.threading_configuration(conf[:thread])
+      @log_conf = self.class.log_configuration(conf[:log])
 
       Norikra::Log.init(@log_conf[:level], @log_conf[:dir], {:filesize => @log_conf[:filesize], :backups => @log_conf[:backups]})
 
@@ -144,7 +144,7 @@ module Norikra
       info "Norikra server started."
 
       shutdown_proc = ->{ @running = false }
-      # JVM uses SIGQUIT for thread/heap state dumping
+      # JVM uses SIGQUIT / SIGUSR1 for thread/heap state dumping
       [:INT, :TERM].each do |s|
         Signal.trap(s, shutdown_proc)
       end
@@ -155,14 +155,14 @@ module Norikra
                         else
                           nil
                         end
-      Signal.trap(:USR1, ->{ @dump_stats = true })
+      Signal.trap(:USR2, ->{ @dump_stats = true })
 
-      #TODO: SIGHUP?(dynamic plugin loading?) SIGUSR2?
+      #TODO: SIGHUP?(dynamic plugin loading?)
 
       while @running
         sleep 0.3
 
-        if @stats && !@stats_suppress_dump
+        if @stats_path && !@stats_suppress_dump
           if @dump_stats || (@dump_next_time && Time.now > @dump_next_time)
             dump_stats
             @dump_stats = false
@@ -203,21 +203,7 @@ module Norikra
     end
 
     def dump_stats
-      stats = Norikra::Stats.new(
-        host: @host,
-        port: @port,
-        threads: @thread_conf,
-        log: @log_conf,
-        targets: @engine.targets.map{|t|
-          {
-            :name => t.name,
-            :fields => @engine.typedef_manager.dump_target(t.name),
-            :auto_field => t.auto_field
-          }
-        },
-        queries: @engine.queries.map(&:dump)
-      )
-      stats.dump(@stats_path)
+      Norikra::Stats.generate(@engine).dump(@stats_path)
       info "Current status saved", :path => @stats_path
     end
   end

data/lib/norikra/stats.rb

@@ -2,27 +2,40 @@ require 'json'
 
 module Norikra
   class Stats
-    attr_accessor :host, :port, :ui_port, :threads, :log
     attr_accessor :targets, :queries
 
+    def self.generate(engine)
+      Norikra::Stats.new(
+        targets: engine.targets.map{|t|
+          {
+            :name => t.name,
+            :fields => engine.typedef_manager.dump_target(t.name),
+            :auto_field => t.auto_field
+          }
+        },
+        queries: engine.queries.map(&:dump)
+      )
+    end
+
     def initialize(opts={})
-      @host = opts[:host]
-      @port = opts[:port]
-      @ui_port = opts[:ui_port]
-      @threads = opts[:threads]
-      @log = opts[:log]
       @targets = opts[:targets] || []
       @queries = opts[:queries] || []
     end
 
     def to_hash
-      {host: @host, port: @port, ui_port: @ui_port, threads: @threads, log: @log, targets: @targets, queries: @queries}
+      {targets: @targets, queries: @queries}
+    end
+
+    def to_json
+      JSON.pretty_generate(self.to_hash)
     end
 
     def dump(path)
-      File.open(path, 'w') do |file|
-        file.write(JSON.pretty_generate(self.to_hash))
+      tmp_path = path + '.tmp'
+      File.open(tmp_path, 'w') do |file|
+        file.write(self.to_json)
       end
+      File.rename(tmp_path, path)
     end
 
     def self.load(path)

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

data/lib/norikra/webui/handler.rb

@@ -4,6 +4,8 @@ require 'norikra/logger'
 
 require 'norikra/webui'
 
+require 'norikra/stats'
+
 require 'sinatra/base'
 require 'sinatra/json'
 require 'erubis'
@@ -87,6 +89,14 @@ class Norikra::WebUI::Handler < Sinatra::Base
     end
   end
 
+  post '/close' do
+    target_name = params[:target]
+    logging(:manage, :close, [target_name]) do
+      engine.close(target_name)
+      redirect '/'
+    end
+  end
+
   post '/register' do
     query_name,query_group,expression = params[:query_name], params[:query_group], params[:expression]
 
@@ -120,6 +130,15 @@ class Norikra::WebUI::Handler < Sinatra::Base
     end
   end
 
+  get '/stat/dump' do
+    logging(:show, :stat_dump) do
+      stats = Norikra::Stats.generate(engine)
+      date = Time.now.strftime('%Y-%m-%d')
+      attachment "stats.dump.#{date}.json"
+      stats.to_json
+    end
+  end
+
   get '/json/query/:name' do
     query_name = params[:name]
     logging(:show, :json_query, [query_name]) do

data/spec/field_spec.rb

@@ -275,5 +275,18 @@ describe Norikra::Field do
       f.define_value_accessor('foo.bar.1.baz', true)
       expect(f.value({'foo' => {'bar' => [{'baz' => 'value1'}, {'baz' => 'value2'}]}})).to eql('value2')
     end
+
+    it 'defines chain value accessor with field name including escaped field' do
+      f = Norikra::Field.new('foo', 'hash')
+      f.define_value_accessor('foo.bar_baz.bar_baz', true)
+      data = {
+        'foo' => {
+          'bar baz' => {
+            'bar/baz' => 'ichi',
+          }
+        }
+      }
+      expect(f.value(data)).to eql('ichi')
+    end
   end
 end

data/spec/query_spec.rb

@@ -24,6 +24,23 @@ describe Norikra::Query do
         end
       end
 
+      context 'with top-level built-in functions' do
+        it 'returns query instances collectly parsed' do
+          expression = 'SELECT rate(10) FROM TestTable output snapshot every 2 sec'
+          q = Norikra::Query.new(
+            :name => 'TestTable query1', :expression => expression
+          )
+          expect(q.name).to eql('TestTable query1')
+          expect(q.group).to be_nil
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['TestTable'])
+
+          expect(q.fields).to eql([])
+          expect(q.fields('TestTable')).to eql([])
+          expect(q.fields(nil)).to eql([])
+        end
+      end
+
       context 'with order by' do
         it 'returns query instances, collectly parsed, without AS_names for fields' do
           expression = 'SELECT name.string, count(*) AS cnt FROM TestTable.win:time_batch(10 sec) WHERE path="/" AND size > 100 and param.length() > 0 GROUP BY name.string ORDER BY cnt'
@@ -159,6 +176,24 @@ describe Norikra::Query do
           expect(q.fields(nil)).to eql([])
         end
       end
+
+      context 'with query with patterns' do
+        it 'returns query instances collectly parsed' do
+
+          expression = "select a.name, a.content, b.content from pattern [every a=EventA -> b=EventA(name = a.name, type = 'TYPE') where timer:within(1 min)].win:time(2 hour) where a.source in ('A', 'B')"
+          q = Norikra::Query.new(
+            :name => 'TestTable query9', :expression => expression
+          )
+          expect(q.name).to eql('TestTable query9')
+          expect(q.group).to be_nil
+          expect(q.expression).to eql(expression)
+          expect(q.targets).to eql(['EventA'])
+          expect(q.aliases).to eql(['a', 'b'])
+          expect(q.fields).to eql(['name', 'content', 'type', 'source'].sort)
+          expect(q.fields('EventA')).to eql(['name', 'content', 'type', 'source'].sort)
+          expect(q.fields(nil)).to eql([])
+        end
+      end
     end
 
     describe '#dup' do

data/spec/stats_spec.rb

@@ -9,11 +9,6 @@ describe Norikra::Stats do
   describe '#to_hash' do
     it 'returns internal stats as hash with symbolized keys' do
       args = {
-        host: nil,
-        port: nil,
-        ui_port: nil,
-        threads: Norikra::Server::SMALL_PREDEFINED,
-        log: {level: nil, dir: nil, filesize: nil, backups: nil},
         targets: [],
         queries: [],
       }
@@ -27,11 +22,6 @@ describe Norikra::Stats do
       Dir.mktmpdir do |dir|
         File.open("#{dir}/stats.json", 'w') do |file|
           args = {
-            host: nil,
-            port: nil,
-            ui_port: nil,
-            threads: Norikra::Server::LARGE_PREDEFINED,
-            log: {level: 'WARN', dir: '/var/log/norikra', filesize: '50MB', backups: 300},
             targets: [
               { name: 'test1', fields: { id: { name: 'id', type: 'int', optional: false}, data: { name: 'data', type: 'string', optional: true } } },
             ],

data/views/index.erb

@@ -14,7 +14,7 @@
       <div class="progress-bar progress-bar-<%= usage_bar_color %>" style="width: <%= used_p.to_i %>%;">
         <span class="sr-only"><%= used_p %> % used</span>
       </div>
-      <div class="progress-bar progress-bar-success" style="width: <%= committed_p.to_i %>%;">
+      <div class="progress-bar progress-bar-success" style="width: <%= (committed_p - used_p).to_i %>%;">
         <span class="sr-only"><%= committed_p %> % used</span>
       </div>
     </div>
@@ -151,8 +151,8 @@
 
 <% if targets.size > 0 %>
 <table class="table">
-  <tr><th>Target</th><th>Auto field</th><th>Fields</th><th></th><th>modified</th></tr>
-  <% targets.each do |target| %>
+  <tr><th>Target</th><th>Auto field</th><th>Fields</th><th>modified</th><th></th></tr>
+  <% targets.each_with_index do |target, index| %>
   <tr>
     <td><%= target[:name] %></td>
     <td><%= target[:auto_field] %></td>
@@ -163,11 +163,44 @@
         (lazy target)
       <% end %>
     </td>
-    <td style="font-size: small; color: #c0c0c0;">TODO: change auto_fields, reserve fields, remove targets</td>
     <td style="text-align: right;"><%= target[:modified] %></td>
+    <td style="font-size: small; color: #c0c0c0;">
+      <a class="btn btn-danger btn-xs" data-toggle="modal" href="#removeTarget<%= index %>">
+      <span class="glyphicon glyphicon-trash"></span>
+      </a>
+      <div class="modal fade"
+        id="removeTarget<%= index %>"
+        tabindex="-1" role="dialog" aria-labelledby="removeTargetLabel<%= index %>" aria-hidden="true">
+        <div class="modal-dialog">
+          <div class="modal-content">
+            <div class="modal-header">
+              <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+              <h4 class="modal-title">Remove Target <%= target[:name] %></h4>
+            </div>
+            <div class="modal-body">
+              <p>Remove all definitions and queries?</p>
+              <p>name: <%= target[:name] %></p>
+            </div>
+            <div class="modal-footer">
+              <form class="form-inline" action="/close" method="POST">
+                <input type="hidden" name="target" value="<%= target[:name] %>" />
+                <button type="button" class="btn btn-default" data-dismiss="modal">Cancel</button>
+                <button type="submit" class="btn btn-danger">Remove</button>
+              </form>
+            </div>
+          </div><!-- /.modal-content -->
+        </div><!-- /.modal-dialog -->
+      </div><!-- /.modal -->
+    </td>
   </tr>
   <% end %>
 </table>
 <% else %>
 <p>No targets found.</p>
 <% end %>
+
+<div class="page-header">
+  <h1 id="stats">Target/Query stat dump</h1>
+</div>
+
+<a href="/stat/dump" class="btn btn-info btn-lg" role="button">Download JSON</a>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: norikra
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: java
 authors:
 - TAGOMORI Satoshi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-03-07 00:00:00.000000000 Z
+date: 2014-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mizuno
@@ -292,6 +292,7 @@ files:
 - lib/norikra/query/ast.rb
 - lib/norikra/rpc.rb
 - lib/norikra/rpc/error.rb
+- lib/norikra/rpc/gatekeeper.rb
 - lib/norikra/rpc/handler.rb
 - lib/norikra/rpc/http.rb
 - lib/norikra/rubyudf.rb