norikra 0.0.16-java → 0.0.17-java

data/.gitignore

@@ -15,3 +15,4 @@ test/tmp
 test/version_tmp
 tmp
 vendor/
+dump.json

data/Rakefile

@@ -6,5 +6,16 @@ RSpec::Core::RakeTask.new(:spec) do |t|
   t.pattern = 'spec/**/*_spec.rb'
 end
 
+task :devstart do |t|
+  require 'pathname'
+  libs = ['lib', 'esper'].map{|p| Pathname.new(__FILE__).dirname.join('.', p).expand_path}
+  $LOAD_PATH.unshift(*libs.map(&:to_s))
+
+  ARGV.clear
+  ARGV.push("start", "--more-verbose", "-Xmx1500m", "--stats", "dump.json")
+  require 'norikra/cli'
+  Norikra::CLI.start
+end
+
 task :test => :spec
 task :default => :spec

data/lib/norikra/cli.rb

@@ -46,6 +46,7 @@ module Norikra
         option :'timer-thread-capacity',    :type => :numeric, :default => nil
         # Jetty
         option :'rpc-threads', :type => :numeric, :default => nil, :desc => 'number of threads for rpc handlers'
+        option :'web-threads', :type => :numeric, :default => nil, :desc => 'number of threads for WebUI handlers'
 
         ### Logging options
         option :logdir, :type => :string, :default => nil, :aliases => "-l", \
@@ -140,12 +141,14 @@ module Norikra
         micro: options[:micro], small: options[:small], middle: options[:middle], large: options[:large],
         engine: {inbound:{}, outbound:{}, route_exec:{}, timer_exec:{}},
         rpc: {},
+        web: {},
       }
       [:inbound, :outbound, :route_exec, :timer_exec].each do |sym|
         conf[:thread][:engine][sym][:threads] = options[:"#{sym}-threads"] if options[:"#{sym}-threads"]
         conf[:thread][:engine][sym][:capacity] = options[:"#{sym}-thread-capacity"] if options[:"#{sym}-thread-capacity"]
       end
       conf[:thread][:rpc][:threads] = options[:'rpc-threads'] if options[:'rpc-threads']
+      conf[:thread][:web][:threads] = options[:'web-threads'] if options[:'web-threads']
 
       ### logs
       loglevel = case

data/lib/norikra/engine.rb

@@ -18,6 +18,11 @@ module Norikra
     attr_reader :targets, :queries, :output_pool, :typedef_manager
 
     def initialize(output_pool, typedef_manager, opts={})
+      @statistics = {
+        started: Time.now,
+        events: { input: 0, processed: 0, output: 0, },
+      }
+
       @output_pool = output_pool
       @typedef_manager = typedef_manager
 
@@ -36,6 +41,53 @@ module Norikra
       @waiting_queries = []
     end
 
+    def statistics
+      s = @statistics
+      {
+        started: s[:started].httpdate,
+        uptime: self.uptime,
+        memory: self.memory_statistics,
+        input_events: s[:events][:input],
+        processed_events: s[:events][:processed],
+        output_events: s[:events][:output],
+        queries: @queries.size,
+        targets: @targets.size,
+      }
+    end
+
+    def uptime
+      # up 239 days, 20:40
+      seconds = (Time.now - @statistics[:started]).to_i
+      days = seconds / (24*60*60)
+      hours = (seconds - days * (24*60*60)) / (60*60)
+      minutes = (seconds - days * (24*60*60) - hours * (60*60)) / 60
+      "#{days} days, #{sprintf("%02d", hours)}:#{sprintf("%02d", minutes)}"
+    end
+
+    def memory_statistics
+      mb = 1024 * 1024
+
+      memoryBean = Java::JavaLangManagement::ManagementFactory.getMemoryMXBean()
+
+      usage = memoryBean.getHeapMemoryUsage()
+      total = usage.getMax() / mb
+      committed = usage.getCommitted() / mb
+      committed_percent = (committed.to_f / total * 1000).floor / 10.0
+      used = usage.getUsed() / mb
+      used_percent = (used.to_f / total * 1000).floor / 10.0
+      heap = { max: total, committed: committed, committed_percent: committed_percent, used: used, used_percent: used_percent }
+
+      usage = memoryBean.getNonHeapMemoryUsage()
+      total = usage.getMax() / mb
+      committed = usage.getCommitted() / mb
+      committed_percent = (committed.to_f / total * 1000).floor / 10.0
+      used = usage.getUsed() / mb
+      used_percent = (used.to_f / total * 1000).floor / 10.0
+      non_heap = { max: total, committed: committed, committed_percent: committed_percent, used: used, used_percent: used_percent }
+
+      { heap: heap, nonheap: non_heap }
+    end
+
     def camelize(sym)
       sym.to_s.split(/_/).map(&:capitalize).join
     end
@@ -133,6 +185,9 @@ module Norikra
 
     def send(target_name, events)
       trace "send messages", :target => target_name, :events => events
+
+      @statistics[:events][:input] += events.size
+
       unless @targets.any?{|t| t.name == target_name} # discard events for target not registered
         trace "messages skipped for non-opened target", :target => target_name
         return
@@ -175,6 +230,7 @@ module Norikra
         trace "sendEvent", :data => formed
         @runtime.sendEvent(formed.to_java, fieldset.event_type_name)
       end
+      @statistics[:events][:processed] += events.size
       nil
     end
 
@@ -213,6 +269,7 @@ module Norikra
         events = new_events.map{|e| [t, type_convert(e)]}
         trace "updated event", :query => @query_name, :group => @query_group, :event => events
         @output_pool.push(@query_name, @query_group, events)
+        @statistics[:events][:output] += events.size
       end
     end
     ##### Unmatched events are simply ignored

data/lib/norikra/logger.rb

@@ -117,6 +117,8 @@ module Norikra
       end
     end
 
+    def self.logger; @@logger ; end
+
     def trace(message, data=nil)
       return if LEVEL_TRACE < @@levelnum
       from = @@devmode ? caller_locations(1,1) : nil

data/lib/norikra/query.rb

@@ -14,8 +14,11 @@ module Norikra
 
     def initialize(param={})
       @name = param[:name]
+      raise Norikra::ArgumentError, "Query name MUST NOT be blank" if @name.nil? || @name.empty?
       @group = param[:group] # default nil
       @expression = param[:expression]
+      raise Norikra::ArgumentError, "Query expression MUST NOT be blank" if @expression.nil? || @expression.empty?
+
       @statement_name = nil
       @fieldsets = {} # { target => fieldset }
       @ast = nil
@@ -25,6 +28,22 @@ module Norikra
       @fields = nil
     end
 
+    def <=>(other)
+      if @group.nil? || other.group.nil?
+        if @group.nil? && other.group.nil?
+          @name <=> other.name
+        else
+          @group.to_s <=> other.group.to_s
+        end
+      else
+        if @group == other.group
+          self.name <=> other.name
+        else
+          self.group <=> other.group
+        end
+      end
+    end
+
     def dup
       self.class.new(:name => @name, :group => @group, :expression => @expression.dup)
     end
@@ -33,6 +52,10 @@ module Norikra
       {'name' => @name, 'group' => @group, 'expression' => @expression, 'targets' => self.targets}
     end
 
+    def dump
+      {name: @name, group: @group, expression: @expression}
+    end
+
     def targets
       return @targets if @targets
       @targets = (self.ast.listup(:stream).map(&:target) + self.subqueries.map(&:targets).flatten).sort.uniq
@@ -167,7 +190,7 @@ module Norikra
       # model.getWhereClause.getChildren[1].getChildren[0].getPropertyName #=> 'field.key1.$1'
       # model.getWhereClause.getChildren[2].getChildren[0].getChain[0].getName #=> 'opts.num.$0' from opts.num.$0.length()
 
-      query = Norikra::Query.new(:expression => statement_model.toEPL)
+      query = Norikra::Query.new(:name => 'dummy name by .rewrite_event_field_name', :expression => statement_model.toEPL)
       targets = query.targets
       fqfs_prefixes = targets + query.aliases
 

data/lib/norikra/rpc/handler.rb

@@ -9,7 +9,7 @@ class Norikra::RPC::Handler
     @engine = engine
   end
 
-  def logging(type, handler, *args)
+  def logging(type, handler, args=[])
     if type == :manage
       debug "RPC", :handler => handler.to_s, :args => args
     else
@@ -37,21 +37,21 @@ class Norikra::RPC::Handler
   end
 
   def open(target, fields, auto_field)
-    logging(:manage, :open, target, fields){
+    logging(:manage, :open, [target, fields]){
       r = @engine.open(target, fields, auto_field)
       !!r
     }
   end
 
   def close(target)
-    logging(:manage, :close, target){
+    logging(:manage, :close, [target]){
       r = @engine.close(target)
       !!r
     }
   end
 
   def modify(target, auto_field)
-    logging(:manage, :modify, target, auto_field){
+    logging(:manage, :modify, [target, auto_field]){
       r = @engine.modify(target, auto_field)
       !!r
     }
@@ -64,41 +64,41 @@ class Norikra::RPC::Handler
   end
 
   def register(query_name, query_group, expression)
-    logging(:manage, :register, query_name, query_group, expression){
+    logging(:manage, :register, [query_name, query_group, expression]){
       r = @engine.register(Norikra::Query.new(:name => query_name, :group => query_group, :expression => expression))
       !!r
     }
   end
 
   def deregister(query_name)
-    logging(:manage, :deregister, query_name){
+    logging(:manage, :deregister, [query_name]){
       r = @engine.deregister(query_name)
       !!r
     }
   end
 
   def fields(target)
-    logging(:show, :fields, target){
+    logging(:show, :fields, [target]){
       @engine.typedef_manager.field_list(target)
     }
   end
 
   def reserve(target, fieldname, type)
-    logging(:manage, :reserve, target, fieldname, type){
+    logging(:manage, :reserve, [target, fieldname, type]){
       r = @engine.reserve(target, fieldname, type)
       !!r
     }
   end
 
   def send(target, events)
-    logging(:data, :send, target, events){
+    logging(:data, :send, [target, events]){
       r = @engine.send(target, events)
       !!r
     }
   end
 
   def event(query_name)
-    logging(:show, :event, query_name){
+    logging(:show, :event, [query_name]){
       @engine.output_pool.pop(query_name)
     }
   end

data/lib/norikra/rpc/http.rb

@@ -4,6 +4,9 @@ require 'msgpack-rpc-over-http-jruby'
 
 require_relative 'handler'
 
+require 'norikra/logger'
+include Norikra::Log
+
 module Norikra::RPC
   class HTTP
     DEFAULT_LISTEN_HOST = '0.0.0.0'
@@ -27,6 +30,7 @@ module Norikra::RPC
     end
 
     def start
+      info "RPC server #{@host}:#{@port}, #{@threads} threads"
       @thread = Thread.new do
         @mizuno = Mizuno::Server.new
         @mizuno.run(@app, :embedded => true, :threads => @threads, :port => @port, :host => @host)

data/lib/norikra/server.rb

@@ -10,6 +10,8 @@ require 'norikra/typedef'
 require 'norikra/query'
 
 require 'norikra/rpc'
+require 'norikra/webui'
+
 require 'norikra/udf'
 
 module Norikra
@@ -20,21 +22,25 @@ module Norikra
       :engine => { inbound:    { threads: 0, capacity: 0 }, outbound:   { threads: 0, capacity: 0 },
                    route_exec: { threads: 0, capacity: 0 }, timer_exec: { threads: 0, capacity: 0 }, },
       :rpc => { threads: 2 },
+      :web => { threads: 2 },
     }
     SMALL_PREDEFINED = {
       :engine => { inbound:    { threads: 1, capacity: 0 }, outbound:   { threads: 1, capacity: 0 },
                    route_exec: { threads: 1, capacity: 0 }, timer_exec: { threads: 1, capacity: 0 }, },
       :rpc => { threads: 2 },
+      :web => { threads: 2 },
     }
     MIDDLE_PREDEFINED = {
       :engine => { inbound:    { threads: 4, capacity: 0 }, outbound:   { threads: 2, capacity: 0 },
                    route_exec: { threads: 2, capacity: 0 }, timer_exec: { threads: 2, capacity: 0 }, },
       :rpc => { threads: 4 },
+      :web => { threads: 2 },
     }
     LARGE_PREDEFINED = {
       :engine => { inbound:    { threads: 6, capacity: 0 }, outbound:   { threads: 6, capacity: 0 },
                    route_exec: { threads: 4, capacity: 0 }, timer_exec: { threads: 4, capacity: 0 }, },
       :rpc => { threads: 8 },
+      :web => { threads: 2 },
     }
 
     def self.threading_configuration(conf, stats)
@@ -51,6 +57,7 @@ module Norikra
         end
       end
       threads[:rpc][:threads] = conf[:rpc][:threads] if conf[:rpc][:threads]
+      threads[:web][:threads] = conf[:web][:threads] if conf[:web][:threads]
       threads
     end
 
@@ -83,6 +90,7 @@ module Norikra
 
       @host = host || (@stats ? @stats.host : nil)
       @port = port || (@stats ? @stats.port : nil)
+      @ui_port = @stats ? @stats.ui_port : nil
 
       @thread_conf = self.class.threading_configuration(conf[:thread], @stats)
       @log_conf = self.class.log_configuration(conf[:log], @stats)
@@ -96,7 +104,17 @@ module Norikra
       @output_pool = Norikra::OutputPool.new
 
       @engine = Norikra::Engine.new(@output_pool, @typedef_manager, {thread: @thread_conf[:engine]})
-      @rpcserver = Norikra::RPC::HTTP.new(:engine => @engine, :host => @host, :port => @port, :threads => @thread_conf[:rpc][:threads])
+
+      @rpcserver = Norikra::RPC::HTTP.new(
+        :engine => @engine,
+        :host => @host, :port => @port,
+        :threads => @thread_conf[:rpc][:threads]
+      )
+      @webserver = Norikra::WebUI::HTTP.new(
+        :engine => @engine,
+        :host => @host, :port => @ui_port,
+        :threads => @thread_conf[:web][:threads]
+      )
     end
 
     def run
@@ -113,12 +131,13 @@ module Norikra
         end
         if @stats.queries && @stats.queries.size > 0
           @stats.queries.each do |query|
-            @engine.register(Norikra::Query.new(:name => query[:name], :expression => query[:expression]))
+            @engine.register(Norikra::Query.new(:name => query[:name], :group => query[:group], :expression => query[:expression]))
           end
         end
       end
 
       @rpcserver.start
+      @webserver.start
 
       @running = true
       info "Norikra server started."
@@ -153,6 +172,7 @@ module Norikra
 
     def shutdown
       info "Norikra server shutting down."
+      @webserver.stop
       @rpcserver.stop
       @engine.stop
       info "Norikra server stopped."
@@ -170,7 +190,7 @@ module Norikra
               :auto_field => t.auto_field
             }
           },
-          queries: @engine.queries.map{|q| {:name => q.name, :expression => q.expression}}
+          queries: @engine.queries.map(&:dump)
         )
         stats.dump(@stats_path)
         info "Current status saved", :path => @stats_path

data/lib/norikra/stats.rb

@@ -2,12 +2,13 @@ require 'json'
 
 module Norikra
   class Stats
-    attr_accessor :host, :port, :threads, :log
+    attr_accessor :host, :port, :ui_port, :threads, :log
     attr_accessor :targets, :queries
 
     def initialize(opts={})
       @host = opts[:host]
       @port = opts[:port]
+      @ui_port = opts[:ui_port]
       @threads = opts[:threads]
       @log = opts[:log]
       @targets = opts[:targets] || []
@@ -15,7 +16,7 @@ module Norikra
     end
 
     def to_hash
-      {host: @host, port: @port, threads: @threads, log: @log, targets: @targets, queries: @queries}
+      {host: @host, port: @port, ui_port: @ui_port, threads: @threads, log: @log, targets: @targets, queries: @queries}
     end
 
     def dump(path)

data/lib/norikra/target.rb

@@ -12,6 +12,10 @@ module Norikra
       @auto_field = !!auto_field
     end
 
+    def <=>(other)
+      self.name <=> other.name
+    end
+
     def to_hash
       {:name => @name, :auto_field => @auto_field}
     end

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "0.0.16"
+  VERSION = "0.0.17"
 end

data/lib/norikra/webui.rb

@@ -0,0 +1,5 @@
+module Norikra::WebUI
+end
+
+require 'norikra/webui/handler'
+require 'norikra/webui/http'