norikra 0.0.11-java → 0.0.12-java

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - jruby-19mode
+jdk:
+  - oraclejdk7
+  - oraclejdk6
+  - openjdk7
+  - openjdk6

data/lib/norikra/cli.rb

@@ -1,4 +1,5 @@
 require 'norikra/server'
+require 'norikra/error'
 require 'thor'
 
 module Norikra
@@ -6,9 +7,13 @@ module Norikra
     desc "start", "start Norikra server process"
 
     ### Server options
-    option :host, :type => :string, :default => '0.0.0.0', :aliases => "-H", :desc => 'host address that server listen [0.0.0.0]'
-    option :port, :type => :numeric, :default => 26571, :aliases => "-P", :desc => 'port that server uses [26571]'
-    # option :config, :type => :string, :default => nil, :aliases => "-c", :desc => 'configuration file to define target/query [none]'
+    option :host, :type => :string, :default => nil, :aliases => "-H", :desc => 'host address that server listen [0.0.0.0]'
+    option :port, :type => :numeric, :default => nil, :aliases => "-P", :desc => 'port that server uses [26571]'
+
+    option :stats, :type => :string, :default => nil, :aliases => "-s", \
+                   :desc => 'status file path to load/dump targets, queries and server configurations [none]'
+    option :'suppress-dump-stat', :type => :boolean, :default => false, \
+                                  :desc => 'specify not to update stat file with updated targets/queries/configurations on runtime [false]'
 
     ### Execution options
     option :daemonize, :type => :boolean, :default => false, :aliases => "-d", \
@@ -17,20 +22,32 @@ module Norikra
                      :desc => "pidfile path when daemonized [/var/run/norikra.pid]"
 
     ### Performance options
-    option :'inbound-threads',  :type => :numeric, :default => 0, :desc => 'number of threads for inbound data'
-    option :'outbound-threads', :type => :numeric, :default => 0, :desc => 'number of threads for outbound data'
-    option :'route-threads',    :type => :numeric, :default => 0, :desc => 'number of threads for events routing for query execution'
-    option :'timer-threads',    :type => :numeric, :default => 0, :desc => 'number of threads for internal timers for query execution'
-    option :'inbound-thread-capacity',  :type => :numeric, :default => 0
-    option :'outbound-thread-capacity', :type => :numeric, :default => 0
-    option :'route-thread-capacity',    :type => :numeric, :default => 0
-    option :'timer-thread-capacity',    :type => :numeric, :default => 0
+    # performance predefined configuration sets
+    option :micro, :type => :boolean, :default => false, \
+                   :desc => 'development or testing (inbound:0, outbound:0, route:0, timer:0, rpc:2)'
+    option :small, :type => :boolean, :default => false, \
+                   :desc => 'virtual or small scale servers (inbound:1, outbount:1, route:1, timer:1, rpc:2)'
+    option :middle, :type => :boolean, :default => false, \
+                   :desc => 'rackmount servers (inbound:4, outbound:2, route:2, timer:2, rpc:4)'
+    option :large, :type => :boolean, :default => false, \
+                   :desc => 'high performance servers (inbound: 6, outbound: 6, route:4, timer:4, rpc: 8)'
+    # Esper
+    option :'inbound-threads',  :type => :numeric, :default => nil, :desc => 'number of threads for inbound data'
+    option :'outbound-threads', :type => :numeric, :default => nil, :desc => 'number of threads for outbound data'
+    option :'route-threads',    :type => :numeric, :default => nil, :desc => 'number of threads for events routing for query execution'
+    option :'timer-threads',    :type => :numeric, :default => nil, :desc => 'number of threads for internal timers for query execution'
+    option :'inbound-thread-capacity',  :type => :numeric, :default => nil
+    option :'outbound-thread-capacity', :type => :numeric, :default => nil
+    option :'route-thread-capacity',    :type => :numeric, :default => nil
+    option :'timer-thread-capacity',    :type => :numeric, :default => nil
+    # Jetty
+    option :'rpc-threads', :type => :numeric, :default => nil, :desc => 'number of threads for rpc handlers'
 
     ### Logging options
     option :logdir, :type => :string, :default => nil, :aliases => "-l", \
                     :desc => "directory path of logfiles when daemonized [nil (console)]"
-    option :'log-filesize', :type => :string, :default => '10MB'
-    option :'log-backups' , :type => :numeric, :default => 10
+    option :'log-filesize', :type => :string, :default => nil, :desc => 'log rotation size [10MB]'
+    option :'log-backups' , :type => :numeric, :default => nil, :desc => 'log rotation backups [10]'
 
     ### Loglevel options
     option :'more-quiet',   :type => :boolean, :default => false,                   :desc => 'set loglevel as ERROR'
@@ -38,7 +55,6 @@ module Norikra
     option :verbose,        :type => :boolean, :default => false, :aliases => "-v", :desc => 'set loglevel as DEBUG'
     option :'more-verbose', :type => :boolean, :default => false,                   :desc => 'set loglevel as TRACE'
 
-    #TODO: configuration file to init
     def start
       conf = {}
 
@@ -46,23 +62,39 @@ module Norikra
       raise NotImplementedError if options[:daemonize]
       #TODO: pidcheck if daemonize
 
+      ### stat file
+      conf[:stats] = {
+        path: options[:stats], suppress: options[:'suppress-dump-stat'],
+      }
+
+      ### threads
+      predefined_selecteds = [:micro, :small, :middle, :larage].select{|sym| options[sym]}
+      if predefined_selecteds.size > 1
+        raise Norikra::ConfigurationError, "one of micro/small/middle/large should be specified"
+      end
       conf[:thread] = {
-        inbound: {threads: options[:'inbound-threads'], capacity: options[:'inbound-thread-capacity']},
-        outbound: {threads: options[:'outbound-threads'], capacity: options[:'outbound-thread-capacity']},
-        route_exec: {threads: options[:'route-threads'], capacity: options[:'route-thread-capacity']},
-        timer_exec: {threads: options[:'timer-threads'], capacity: options[:'timer-thread-capacity']},
+        predefined: predefined_selecteds.first,
+        micro: options[:micro], small: options[:small], middle: options[:middle], large: options[:large],
+        engine: {inbound:{}, outbound:{}, route_exec:{}, timer_exec:{}},
+        rpc: {},
       }
+      [:inbound, :outbound, :route_exec, :timer_exec].each do |sym|
+        conf[:thread][:engine][sym][:threads] = options[:"#{sym}-threads"] if options[:"#{sym}-threads"]
+        conf[:thread][:engine][sym][:capacity] = options[:"#{sym}-thread-capacity"] if options[:"#{sym}-thread-capacity"]
+      end
+      conf[:thread][:rpc][:threads] = options[:'rpc-threads'] if options[:'rpc-threads']
 
-      conf[:loglevel] = case
-                        when options[:'more-verbose'] then 'TRACE'
-                        when options[:verbose]        then 'DEBUG'
-                        when options[:quiet]          then 'WARN'
-                        when options[:'more-quiet']   then 'ERROR'
-                        else nil # for default (assumed as 'INFO')
-                        end
-      conf[:logdir] = options[:logdir]
-      conf[:logfilesize] = options[:'log-filesize']
-      conf[:logbackups] = options[:'log-backups']
+      ### logs
+      loglevel = case
+                 when options[:'more-verbose'] then 'TRACE'
+                 when options[:verbose]        then 'DEBUG'
+                 when options[:quiet]          then 'WARN'
+                 when options[:'more-quiet']   then 'ERROR'
+                 else nil # for default (assumed as 'INFO')
+                 end
+      conf[:log] = {
+        level: loglevel, dir: options[:logdir], filesize: options[:'log-filesize'], backups: options[:'log-backups'],
+      }
 
       server = Norikra::Server.new( options[:host], options[:port], conf )
       server.run

data/lib/norikra/engine.rb

@@ -1,6 +1,7 @@
 require 'java'
 
 require 'norikra/error'
+require 'norikra/target'
 
 require 'norikra/logger'
 include Norikra::Log
@@ -78,8 +79,11 @@ module Norikra
     end
 
     def open(target, fields=nil)
+      # fields nil || [] => lazy
+      # fields {'fieldname' => 'type'} : type 'string', 'boolean', 'int', 'long', 'float', 'double'
       info "opening target", :target => target, :fields => fields
       return false if @targets.include?(target)
+      raise Norikra::ArgumentError, "invalid target name" unless Norikra::Target.valid?(target)
       open_target(target, fields)
     end
 
@@ -161,8 +165,9 @@ module Norikra
     class Listener
       include com.espertech.esper.client.UpdateListener
 
-      def initialize(query_name, output_pool)
+      def initialize(query_name, query_group, output_pool)
         @query_name = query_name
+        @query_group = query_group
         @output_pool = output_pool
       end
 
@@ -182,8 +187,8 @@ module Norikra
       def update(new_events, old_events)
         t = Time.now.to_i
         events = new_events.map{|e| [t, type_convert(e)]}
-        trace "updated event", :query => @query_name, :event => events
-        @output_pool.push(@query_name, events)
+        trace "updated event", :query => @query_name, :group => @query_group, :event => events
+        @output_pool.push(@query_name, @query_group, events)
       end
     end
     ##### Unmatched events are simply ignored
@@ -357,7 +362,7 @@ module Norikra
       Norikra::Query.rewrite_event_type_name(statement_model, event_type_name_map)
 
       epl = administrator.create(statement_model)
-      epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], Listener.new(query.name, @output_pool)
+      epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], Listener.new(query.name, query.group, @output_pool)
       query.statement_name = epl.getName
       # epl is automatically started.
       # epl.isStarted #=> true

data/lib/norikra/error.rb

@@ -1,4 +1,5 @@
 module Norikra
+  class ConfigurationError < StandardError; end
   class ClientError < StandardError; end
   class ArgumentError < ClientError; end
   class QueryError < ClientError; end

data/lib/norikra/output_pool.rb

@@ -1,9 +1,12 @@
+require 'set'
+
 module Norikra
   class OutputPool
     attr_accessor :pool
 
     def initialize
-      @pool = {}
+      @pool = {} # { query_name => [events] }
+      @groups = {} # { group_name => Set(query_names) }
       @mutex = Mutex.new
     end
 
@@ -11,10 +14,19 @@ module Norikra
       @pool.keys
     end
 
-    def push(query_name, events)
+    def push(query_name, query_group, events) # events must be [time(int), event_record]
+      # called with blank events for window leavings (and/or other situations)
+      return if events.size < 1
+
       @mutex.synchronize do
+        if @groups[query_group]
+          @groups[query_group].add(query_name) # Set is unique set of elements
+        else
+          @groups[query_group] ||= Set.new([query_name])
+        end
+
         @pool[query_name] ||= []
-        @pool[query_name].push(events) if events.size > 0
+        @pool[query_name].push(events)
       end
     end
 
@@ -27,11 +39,15 @@ module Norikra
     end
 
     # returns {query_name => [[time, event], ...]}
-    def sweep
+    def sweep(group=nil)
+      return {} if @groups[group].nil?
+
       ret = {}
       sweep_pool = @mutex.synchronize do
-        sweeped = @pool
-        @pool = {}
+        sweeped = {}
+        @groups[group].each do |qname|
+          sweeped[qname] = @pool.delete(qname) if @pool[qname] && @pool[qname].size > 0
+        end
         sweeped
       end
       sweep_pool.keys.each do |k|

data/lib/norikra/query.rb

@@ -9,10 +9,11 @@ require 'norikra/query/ast'
 
 module Norikra
   class Query
-    attr_accessor :name, :expression, :statement_name, :fieldsets
+    attr_accessor :name, :group, :expression, :statement_name, :fieldsets
 
     def initialize(param={})
       @name = param[:name]
+      @group = param[:group] # default nil
       @expression = param[:expression]
       @statement_name = nil
       @fieldsets = {} # { target => fieldset }
@@ -23,7 +24,7 @@ module Norikra
     end
 
     def dup
-      self.class.new(:name => @name, :expression => @expression.dup)
+      self.class.new(:name => @name, :group => @group, :expression => @expression.dup)
     end
 
     def dup_with_stream_name(actual_name)
@@ -37,7 +38,7 @@ module Norikra
     end
 
     def to_hash
-      {'name' => @name, 'expression' => @expression, 'targets' => self.targets}
+      {'name' => @name, 'group' => @group, 'expression' => @expression, 'targets' => self.targets}
     end
 
     def targets

data/lib/norikra/rpc/handler.rb

@@ -56,9 +56,9 @@ class Norikra::RPC::Handler
     }
   end
 
-  def register(query_name, expression)
-    logging(:manage, :register, query_name, expression){
-      r = @engine.register(Norikra::Query.new(:name => query_name, :expression => expression))
+  def register(query_name, query_group, expression)
+    logging(:manage, :register, query_name, query_group, expression){
+      r = @engine.register(Norikra::Query.new(:name => query_name, :group => query_group, :expression => expression))
       !!r
     }
   end
@@ -96,9 +96,9 @@ class Norikra::RPC::Handler
     }
   end
 
-  def sweep
+  def sweep(query_group=nil)
     logging(:show, :sweep){
-      @engine.output_pool.sweep
+      @engine.output_pool.sweep(query_group)
     }
   end
 

data/lib/norikra/rpc/http.rb

@@ -6,14 +6,20 @@ require_relative 'handler'
 
 module Norikra::RPC
   class HTTP
-    #TODO Xmx of mizuno/jetty
+    DEFAULT_LISTEN_HOST = '0.0.0.0'
+    DEFAULT_LISTEN_PORT = 26571
+    # 26571 = 3026 + 3014 + 2968 + 2950 + 2891 + 2896 + 2975 + 2979 + 2872
+
+    DEFAULT_THREADS = 2
+
     attr_accessor :host, :port, :threads
     attr_accessor :engine, :mizuno, :thread
 
     def initialize(opts={})
       @engine = opts[:engine]
-      @host = opts[:host]
-      @port = opts[:port]
+      @host = opts[:host] || DEFAULT_LISTEN_HOST
+      @port = opts[:port] || DEFAULT_LISTEN_PORT
+      @threads = opts[:threads] || DEFAULT_THREADS
       handler = Norikra::RPC::Handler.new(@engine)
       @app = Rack::Builder.new {
         run MessagePack::RPCOverHTTP::Server.app(handler)
@@ -23,7 +29,7 @@ module Norikra::RPC
     def start
       @thread = Thread.new do
         @mizuno = Mizuno::Server.new
-        @mizuno.run(@app, :embedded => true, :threads => 5, :port => @port, :host => @host)
+        @mizuno.run(@app, :embedded => true, :threads => @threads, :port => @port, :host => @host)
       end
     end
 

data/lib/norikra/rubyudf.rb

@@ -1,49 +1,49 @@
 # this is note for future update
 
-module Norikra
-  module UDF
-    class FailedUDFImplementationPureRuby
-      # require 'jruby/core_ext'
-      class WootheeIsCrawler < Norikra::UDF::Base # Norikra::UDF::WootheeIsCrawler < Norikra::UDF::Base
-        def self.init
-          require 'woothee'
-        end
-
-        def self.function_name
-          "isCrawler"
-        end
-
-        def self.isCrawler(agent)
-          Woothee.is_crawler(agent)
-        end
-        class << self
-          add_method_signature( "isCrawler", [java.lang.Boolean, java.lang.String] )
-        end
-      end
-
-      # for engine.rb
-      def load_udf_actually(udf_klass)
-        require 'jruby/core_ext'
-        udf_klass.init
-
-        jclass = udf_klass.become_java!(".")
-        className = jclass.get_name.to_java(:string)
-
-        #### try for NullPointerException, but doesn't work well
-        # field = jclass.getDeclaredField("ruby");
-        # field.setAccessible(java.lang.Boolean::TRUE)
-        # field.set(nil, org.jruby.Ruby.getGlobalRuntime)
-
-        functionName = udf_klass.function_name.to_java(:string)
-        methodName = udf_klass.method_name.to_java(:string)
-
-        valueCache = udf_klass.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
-        filterOptimizable = udf_klass.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
-        rethrowExceptions = udf_klass.rethrow_exceptions
-
-        debug "adding SingleRowFunction", :class => udf_klass.to_s, :javaClass => jclass.get_name
-        @config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
-      end
-    end
-  end
-end
+# module Norikra
+#   module UDF
+#     class FailedUDFImplementationPureRuby
+#       # require 'jruby/core_ext'
+#       class WootheeIsCrawler < Norikra::UDF::Base # Norikra::UDF::WootheeIsCrawler < Norikra::UDF::Base
+#         def self.init
+#           require 'woothee'
+#         end
+
+#         def self.function_name
+#           "isCrawler"
+#         end
+
+#         def self.isCrawler(agent)
+#           Woothee.is_crawler(agent)
+#         end
+#         class << self
+#           add_method_signature( "isCrawler", [java.lang.Boolean, java.lang.String] )
+#         end
+#       end
+
+#       # for engine.rb
+#       def load_udf_actually(udf_klass)
+#         require 'jruby/core_ext'
+#         udf_klass.init
+
+#         jclass = udf_klass.become_java!(".")
+#         className = jclass.get_name.to_java(:string)
+
+#         #### try for NullPointerException, but doesn't work well
+#         # field = jclass.getDeclaredField("ruby");
+#         # field.setAccessible(java.lang.Boolean::TRUE)
+#         # field.set(nil, org.jruby.Ruby.getGlobalRuntime)
+
+#         functionName = udf_klass.function_name.to_java(:string)
+#         methodName = udf_klass.method_name.to_java(:string)
+
+#         valueCache = udf_klass.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
+#         filterOptimizable = udf_klass.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
+#         rethrowExceptions = udf_klass.rethrow_exceptions
+
+#         debug "adding SingleRowFunction", :class => udf_klass.to_s, :javaClass => jclass.get_name
+#         @config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
+#       end
+#     end
+#   end
+# end

data/lib/norikra/server.rb

@@ -1,5 +1,6 @@
 require 'norikra/engine'
 
+require 'norikra/stats'
 require 'norikra/logger'
 include Norikra::Log
 
@@ -13,29 +14,102 @@ require 'norikra/udf'
 
 module Norikra
   class Server
-    RPC_DEFAULT_HOST = '0.0.0.0'
-    RPC_DEFAULT_PORT = 26571
-    # 26571 = 3026 + 3014 + 2968 + 2950 + 2891 + 2896 + 2975 + 2979 + 2872
-
     attr_accessor :running
 
-    def initialize(host=RPC_DEFAULT_HOST, port=RPC_DEFAULT_PORT, conf={})
-      #TODO: initial configuration (targets/queries)
+    MICRO_PREDEFINED = {
+      :engine => { inbound:    { threads: 0, capacity: 0 }, outbound:   { threads: 0, capacity: 0 },
+                   route_exec: { threads: 0, capacity: 0 }, timer_exec: { threads: 0, capacity: 0 }, },
+      :rpc => { threads: 2 },
+    }
+    SMALL_PREDEFINED = {
+      :engine => { inbound:    { threads: 1, capacity: 0 }, outbound:   { threads: 1, capacity: 0 },
+                   route_exec: { threads: 1, capacity: 0 }, timer_exec: { threads: 1, capacity: 0 }, },
+      :rpc => { threads: 2 },
+    }
+    MIDDLE_PREDEFINED = {
+      :engine => { inbound:    { threads: 4, capacity: 0 }, outbound:   { threads: 2, capacity: 0 },
+                   route_exec: { threads: 2, capacity: 0 }, timer_exec: { threads: 2, capacity: 0 }, },
+      :rpc => { threads: 4 },
+    }
+    LARGE_PREDEFINED = {
+      :engine => { inbound:    { threads: 6, capacity: 0 }, outbound:   { threads: 6, capacity: 0 },
+                   route_exec: { threads: 4, capacity: 0 }, timer_exec: { threads: 4, capacity: 0 }, },
+      :rpc => { threads: 8 },
+    }
+
+    def self.threading_configuration(conf, stats)
+      threads = case conf[:predefined]
+                when :micro then MICRO_PREDEFINED
+                when :small then SMALL_PREDEFINED
+                when :middle then MIDDLE_PREDEFINED
+                when :large then LARGE_PREDEFINED
+                else (stats ? stats.threads : MICRO_PREDEFINED)
+                end
+      [:inbound, :outbound, :route_exec, :timer_exec].each do |type|
+        [:threads, :capacity].each do |item|
+          threads[:engine][type][item] = conf[:engine][type][item] if conf[:engine][type][item]
+        end
+      end
+      threads[:rpc][:threads] = conf[:rpc][:threads] if conf[:rpc][:threads]
+      threads
+    end
+
+    def self.log_configuration(conf, stats)
+      logconf = stats ? stats.log : { level: nil, dir: nil, filesize: nil, backups: nil }
+      [:level, :dir, :filesize, :backups].each do |sym|
+        logconf[sym] = conf[sym] if conf[sym]
+      end
+      logconf
+    end
+
+    def initialize(host, port, conf={})
+      @stats_path = conf[:stats][:path]
+      @stats_suppress_dump = conf[:stats][:suppress]
+      @stats = if @stats_path && test(?r, @stats_path)
+                 Norikra::Stats.load(@stats_path)
+               else
+                 nil
+               end
+
+      @host = host || (@stats ? @stats.host : nil)
+      @port = port || (@stats ? @stats.port : nil)
+
+      @thread_conf = self.class.threading_configuration(conf[:thread], @stats)
+      @log_conf = self.class.log_configuration(conf[:log], @stats)
+
+      Norikra::Log.init(@log_conf[:level], @log_conf[:dir], {:filesize => @log_conf[:filesize], :backups => @log_conf[:backups]})
+
+      info "thread configurations", @thread_conf
+      info "logging configurations", @log_conf
+
       @typedef_manager = Norikra::TypedefManager.new
       @output_pool = Norikra::OutputPool.new
 
-      Norikra::Log.init(conf[:loglevel], conf[:logdir], {:filesize => conf[:logfilesize], :backups => conf[:logbackups]})
-
-      @engine = Norikra::Engine.new(@output_pool, @typedef_manager, {thread: conf[:thread]})
-      @rpcserver = Norikra::RPC::HTTP.new(:engine => @engine, :port => port)
+      @engine = Norikra::Engine.new(@output_pool, @typedef_manager, {thread: @thread_conf[:engine]})
+      @rpcserver = Norikra::RPC::HTTP.new(:engine => @engine, :host => @host, :port => @port, :threads => @thread_conf[:rpc][:threads])
     end
 
     def run
       @engine.start
-      @rpcserver.start
 
       load_plugins
 
+      if @stats
+        info "loading from stats file"
+        if @stats.targets && @stats.targets.size > 0
+          @stats.targets.each do |target|
+            @engine.open(target[:name], target[:fields])
+          end
+        end
+        if @stats.queries && @stats.queries.size > 0
+          @stats.queries.each do |query|
+            @engine.register(Norikra::Query.new(:name => query[:name], :expression => query[:expression]))
+          end
+        end
+      end
+
+      @rpcserver.start
+
       @running = true
       info "Norikra server started."
 
@@ -71,6 +145,21 @@ module Norikra
       info "Norikra server shutting down."
       @rpcserver.stop
       @engine.stop
+      info "Norikra server stopped."
+
+      if @stats_path && !@stats_suppress_dump
+        stats = Norikra::Stats.new(
+          host: @host,
+          port: @port,
+          threads: @thread_conf,
+          log: @log_conf,
+          targets: @engine.typedef_manager.dump,
+          queries: @engine.queries.map{|q| {:name => q.name, :expression => q.expression}}
+        )
+        stats.dump(@stats_path)
+        info "Current status saved", :path => @stats_path
+      end
+
       info "Norikra server shutdown complete."
     end
   end

data/lib/norikra/stats.rb

@@ -0,0 +1,33 @@
+require 'json'
+
+module Norikra
+  class Stats
+    attr_accessor :host, :port, :threads, :log
+    attr_accessor :targets, :queries
+
+    def initialize(opts={})
+      @host = opts[:host]
+      @port = opts[:port]
+      @threads = opts[:threads]
+      @log = opts[:log]
+      @targets = opts[:targets] || []
+      @queries = opts[:queries] || []
+    end
+
+    def to_hash
+      {host: @host, port: @port, threads: @threads, log: @log, targets: @targets, queries: @queries}
+    end
+
+    def dump(path)
+      File.open(path, 'w') do |file|
+        file.write(JSON.pretty_generate(self.to_hash))
+      end
+    end
+
+    def self.load(path)
+      File.open(path, 'r') do |file|
+        self.new(JSON.parse(file.read, symbolize_names: true))
+      end
+    end
+  end
+end

data/lib/norikra/target.rb

@@ -0,0 +1,7 @@
+module Norikra
+  class Target
+    def self.valid?(target_name)
+      target_name =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/
+    end
+  end
+end

data/lib/norikra/typedef.rb

@@ -15,8 +15,12 @@ module Norikra
       @optional = optional
     end
 
-    def to_hash
-      {'name' => @name, 'type' => @type, 'optional' => @optional}
+    def to_hash(sym=false)
+      if sym
+        {name: @name, type: @type, optional: @optional}
+      else
+        {'name' => @name, 'type' => @type, 'optional' => @optional}
+      end
     end
 
     def dup(optional=nil)
@@ -55,7 +59,7 @@ module Norikra
       when 'float' then 'float'
       when 'double' then 'double'
       else
-        raise Norikra::ArgumentError, "invalid field type #{type}"
+        raise Norikra::ArgumentError, "invalid field type '#{type}'"
       end
     end
 
@@ -80,13 +84,13 @@ module Norikra
       fields.keys.each do |key|
         data = fields[key]
         type,optional = if data.is_a?(Hash)
-                          [data[:type], (data.has_key?(:optional) ? data[:optional] : default_optional)]
-                        elsif data.is_a?(String)
-                          [data, default_optional]
+                          [data[:type].to_s, (data.has_key?(:optional) ? data[:optional] : default_optional)]
+                        elsif data.is_a?(String) || data.is_a?(Symbol)
+                          [data.to_s, default_optional]
                         else
                           raise ArgumentError, "FieldSet.new argument class unknown: #{fields.class}"
                         end
-        @fields[key.to_s] = Field.new(key, type, optional)
+        @fields[key.to_s] = Field.new(key.to_s, type, optional)
       end
       self.update_summary
 
@@ -366,5 +370,13 @@ module Norikra
       end
       ret
     end
+
+    def dump
+      fields = {}
+      @fields.map{|key,field|
+        fields[key.to_sym] = field.to_hash(true)
+      }
+      fields
+    end
   end
 end

data/lib/norikra/typedef_manager.rb

@@ -140,5 +140,11 @@ module Norikra
     def format(target, event)
       @typedefs[target].format(event)
     end
+
+    def dump
+      @typedefs.keys.map{|target|
+        {:name => target, :fields => @typedefs[target].dump}
+      }
+    end
   end
 end

data/lib/norikra/version.rb

@@ -1,3 +1,3 @@
 module Norikra
-  VERSION = "0.0.11"
+  VERSION = "0.0.12"
 end

data/norikra.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "rack"
   spec.add_runtime_dependency "msgpack-rpc-over-http-jruby", ">= 0.0.5"
   spec.add_runtime_dependency "thor"
-  spec.add_runtime_dependency "norikra-client-jruby", ">= 0.0.4"
+  spec.add_runtime_dependency "norikra-client-jruby", ">= 0.0.6"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"

data/spec/output_pool_spec.rb

@@ -12,15 +12,18 @@ describe Norikra::OutputPool do
 
     describe '#push' do
       context 'with empty array' do
-        subject { p = Norikra::OutputPool.new; p.push('TestTable query1', []); p }
-        its(:pool){ should == {'TestTable query1' => []} }
+        it 'will be ignored' do
+          p = Norikra::OutputPool.new
+          p.push('TestTable query1', nil, [])
+          expect(p.pool).to eql({})
+        end
       end
 
       context 'with event array' do
         it 'has pool with event' do
           pool = Norikra::OutputPool.new
           t = Time.now.to_i
-          pool.push('TestTable query1', [{'count'=>1},{'count'=>2}])
+          pool.push('TestTable query1',nil,[[t, {'count'=>1}],[t ,{'count'=>2}]])
 
           pool.pool.keys.should eql(['TestTable query1'])
           events = pool.pool['TestTable query1']
@@ -46,8 +49,8 @@ describe Norikra::OutputPool do
       it 'returns all events of specified query in pool' do
         pool = Norikra::OutputPool.new
         t = Time.now.to_i
-        pool.push('TestTable query1', [{'count'=>1},{'count'=>2}])
-        pool.push('TestTable query2', [{'count'=>3},{'count'=>4}])
+        pool.push('TestTable query1', nil, [[t,{'count'=>1}],[t,{'count'=>2}]])
+        pool.push('TestTable query2', nil, [[t,{'count'=>3}],[t,{'count'=>4}]])
 
         expect(pool.pop('TestTable query0').size).to eql(0)
         expect(pool.pop('TestTable query1').size).to eql(2)
@@ -57,18 +60,42 @@ describe Norikra::OutputPool do
     end
 
     describe '#sweep' do
-      it 'returns all events for all queries in pool' do
-        pool = Norikra::OutputPool.new
-        t = Time.now.to_i
-        pool.push('TestTable query1', [{'count'=>1},{'count'=>2}])
-        pool.push('TestTable query2', [{'count'=>3},{'count'=>4},{'count'=>5}])
+      context 'with default query group' do
+        it 'returns all events for all queries in pool' do
+          pool = Norikra::OutputPool.new
+          t = Time.now.to_i
+          pool.push('TestTable query1', nil, [[t,{'count'=>1}],[t,{'count'=>2}]])
+          pool.push('TestTable query2', nil, [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
+          pool.push('TestTable query3', 'x', [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
 
-        chunk = pool.sweep
-        expect(chunk.keys.size).to eql(2)
+          chunk = pool.sweep
+          expect(chunk.keys.size).to eql(2)
 
-        expect(chunk['TestTable query1'].size).to eql(2)
-        expect(chunk['TestTable query2'].size).to eql(3)
-        expect(chunk['TestTable query2'].last.last['count']).to eql(5)
+          expect(chunk['TestTable query1'].size).to eql(2)
+          expect(chunk['TestTable query2'].size).to eql(3)
+          expect(chunk['TestTable query2'].last.last['count']).to eql(5)
+        end
+      end
+
+      context 'with group specified' do
+        it 'returns all events for all queries in pool' do
+          pool = Norikra::OutputPool.new
+          t = Time.now.to_i
+          pool.push('TestTable query1', nil, [[t,{'count'=>1}],[t,{'count'=>2}]])
+          pool.push('TestTable query2', nil, [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
+          pool.push('TestTable query3', 'x', [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
+          pool.push('TestTable query4', 'y', [[t,{'count'=>1}],[t,{'count'=>2}]])
+          pool.push('TestTable query5', 'x', [[t,{'count'=>9}]])
+          pool.push('TestTable query6', 'x', [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
+          pool.push('TestTable query6', 'x', [[t,{'count'=>6}],[t,{'count'=>7}]])
+
+          chunk = pool.sweep('x')
+          expect(chunk.keys.size).to eql(3)
+
+          expect(chunk['TestTable query3'].size).to eql(3)
+          expect(chunk['TestTable query5'].size).to eql(1)
+          expect(chunk['TestTable query6'].size).to eql(5)
+        end
       end
     end
   end

data/spec/query_spec.rb

@@ -12,6 +12,7 @@ describe Norikra::Query do
             :name => 'TestTable query1', :expression => expression
           )
           expect(q.name).to eql('TestTable query1')
+          expect(q.group).to be_nil
           expect(q.expression).to eql(expression)
           expect(q.targets).to eql(['TestTable'])
 
@@ -25,9 +26,10 @@ describe Norikra::Query do
         it 'returns query instances collectly parsed' do
           expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) AS source WHERE source.path="/" AND Math.abs(-1 * source.size) > 3'
           q = Norikra::Query.new(
-            :name => 'TestTable query2', :expression => expression
+            :name => 'TestTable query2', :group => 'label1', :expression => expression
           )
           expect(q.name).to eql('TestTable query2')
+          expect(q.group).to eql('label1')
           expect(q.expression).to eql(expression)
           expect(q.targets).to eql(['TestTable'])
 
@@ -106,7 +108,31 @@ describe Norikra::Query do
       end
     end
 
-   describe '#dup_with_stream_name' do
+    describe '#dup' do
+      context 'for queries without group (default group)' do
+        it 'returns query object with default group' do
+          e1 = 'SELECT max(num) AS max FROM TestTable1.win:time(5 sec)'
+          query = Norikra::Query.new(:name => 'q1', :group => nil, :expression => e1)
+          q = query.dup
+          expect(q.name).to eql('q1')
+          expect(q.group).to be_nil
+          expect(q.expression).to eql(e1)
+        end
+      end
+
+      context 'for queries with group' do
+        it 'returns query object with specified group' do
+          e2 = 'SELECT max(num) AS max FROM TestTable2.win:time(5 sec)'
+          query = Norikra::Query.new(:name => 'q2', :group => 'g2', :expression => e2)
+          q = query.dup
+          expect(q.name).to eql('q2')
+          expect(q.group).to eql('g2')
+          expect(q.expression).to eql(e2)
+        end
+      end
+    end
+
+    describe '#dup_with_stream_name' do
       context 'with simple query' do
         expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) WHERE path="/" AND size > 100 and param.length() > 0'
         it 'returns duplicated object, with replaced ' do

data/spec/stats_spec.rb

@@ -0,0 +1,53 @@
+require_relative './spec_helper'
+
+require 'norikra/stats'
+require 'norikra/server'
+
+require 'tmpdir'
+
+describe Norikra::Stats do
+  describe '#to_hash' do
+    it 'returns internal stats as hash with symbolized keys' do
+      args = {
+        host: nil,
+        port: nil,
+        threads: Norikra::Server::SMALL_PREDEFINED,
+        log: {level: nil, dir: nil, filesize: nil, backups: nil},
+        targets: [],
+        queries: [],
+      }
+      s = Norikra::Stats.new(args)
+      expect(s.to_hash).to eql(args)
+    end
+  end
+
+  describe '.load' do
+    it 'can restore stats data from #dump -ed json' do
+      Dir.mktmpdir do |dir|
+        File.open("#{dir}/stats.json", 'w') do |file|
+          args = {
+            host: nil,
+            port: nil,
+            threads: Norikra::Server::LARGE_PREDEFINED,
+            log: {level: 'WARN', dir: '/var/log/norikra', filesize: '50MB', backups: 300},
+            targets: [
+              { name: 'test1', fields: { id: { name: 'id', type: 'int', optional: false}, data: { name: 'data', type: 'string', optional: true } } },
+            ],
+            queries: [
+              { name: 'testq2', expression: 'select count(*) from test1.win:time(5 sec)' },
+              { name: 'testq1', expression: 'select count(*) from test1.win:time(10 sec)' },
+            ],
+          }
+          s1 = Norikra::Stats.new(args)
+          expect(s1.to_hash).to eql(args)
+
+          s1.dump(file.path)
+
+          s2 = Norikra::Stats.load(file.path)
+          expect(s2.to_hash).to eql(s1.to_hash)
+          expect(s2.to_hash).to eql(args)
+        end
+      end
+    end
+  end
+end

data/spec/target_spec.rb

@@ -0,0 +1,24 @@
+require_relative './spec_helper'
+
+require 'norikra/target'
+# require 'norikra/error'
+
+describe Norikra::Target do
+  describe '.valid?' do
+    it 'raises Norikra::ArgumentError for invalid name' do
+      expect(Norikra::Target.valid?('foobar')).to be_true
+      expect(Norikra::Target.valid?('FooBar')).to be_true
+      expect(Norikra::Target.valid?('foo_bar')).to be_true
+      expect(Norikra::Target.valid?('foo_bar_baz')).to be_true
+
+      expect(Norikra::Target.valid?('')).to be_false
+      expect(Norikra::Target.valid?('.')).to be_false
+      expect(Norikra::Target.valid?('_')).to be_false
+      expect(Norikra::Target.valid?('_a_')).to be_false
+      expect(Norikra::Target.valid?('foo_')).to be_false
+      expect(Norikra::Target.valid?('_Foo')).to be_false
+      expect(Norikra::Target.valid?('foo bar')).to be_false
+      expect(Norikra::Target.valid?('_Foo')).to be_false
+    end
+  end
+end

data/spec/typedef_manager_spec.rb

@@ -200,5 +200,31 @@ describe Norikra::TypedefManager do
         expect(r.summary).to eql('a:string,b:string,c:double,f:boolean')
       end
     end
+
+    describe '#dump' do
+      it 'returns array of target typedef dump' do
+        m = Norikra::TypedefManager.new
+        m.add_target('sample', {'a'=>'string','b'=>'string','c'=>'double'})
+        m.reserve('sample', 'z', 'boolean')
+        m.add_target('sample_next', {'a'=>'string','b'=>'string','c'=>'double','d'=>'double'})
+
+        r = m.dump
+        expect(r.size).to eql(2)
+        expect(r[0][:name]).to eql('sample')
+        expect(r[0][:fields]).to eql({
+            a: {name: 'a', type: 'string', optional: false},
+            b: {name: 'b', type: 'string', optional: false},
+            c: {name: 'c', type: 'double', optional: false},
+            z: {name: 'z', type: 'boolean', optional: true},
+          })
+        expect(r[1][:name]).to eql('sample_next')
+        expect(r[1][:fields]).to eql({
+            a: {name: 'a', type: 'string', optional: false},
+            b: {name: 'b', type: 'string', optional: false},
+            c: {name: 'c', type: 'double', optional: false},
+            d: {name: 'd', type: 'double', optional: false},
+          })
+      end
+    end
   end
 end

data/spec/typedef_spec.rb

@@ -296,5 +296,27 @@ describe Norikra::Typedef do
         expect(d['d']).to eql(3.14)
       end
     end
+
+    describe '#dump' do
+      it 'returns hash instance to show fields and its types/optionals' do
+        t = Norikra::Typedef.new({'a'=>'string','b'=>'long'})
+        t.push(:query, Norikra::FieldSet.new({'a'=>'string','b'=>'long'}))
+        t.push(:data, Norikra::FieldSet.new({'a'=>'string','b'=>'long'}))
+        t.push(:data, Norikra::FieldSet.new({'a'=>'string','b'=>'long','c'=>'double'}))
+        t.push(:query, Norikra::FieldSet.new({'a'=>'string','b'=>'long','d'=>'string'}))
+        fields = t.fields
+
+        r = t.dump
+        expect(r.keys.sort).to eql([:a, :b, :c, :d])
+        expect(r[:a]).to eql({name: 'a', type: 'string', optional: false})
+        expect(r[:b]).to eql({name: 'b', type: 'long', optional: false})
+        expect(r[:c]).to eql({name: 'c', type: 'double', optional: true})
+        expect(r[:d]).to eql({name: 'd', type: 'string', optional: true})
+
+        t2 = Norikra::Typedef.new(r)
+        expect(t2.fields.keys.sort).to eql(fields.keys.sort)
+        expect(t2.fields.values.map(&:to_hash)).to eql(fields.values.map(&:to_hash))
+      end
+    end
   end
 end

metadata

@@ -2,14 +2,14 @@
 name: norikra
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.0.11
+  version: 0.0.12
 platform: java
 authors:
 - TAGOMORI Satoshi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-08-27 00:00:00.000000000 Z
+date: 2013-09-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mizuno
@@ -81,13 +81,13 @@ dependencies:
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: 0.0.6
     none: false
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.0.4
+        version: 0.0.6
     none: false
   prerelease: false
   type: :runtime
@@ -183,6 +183,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .ruby-version
+- .travis.yml
 - Gemfile
 - LICENSE
 - README.md
@@ -274,6 +275,8 @@ files:
 - lib/norikra/rpc/http.rb
 - lib/norikra/rubyudf.rb
 - lib/norikra/server.rb
+- lib/norikra/stats.rb
+- lib/norikra/target.rb
 - lib/norikra/typedef.rb
 - lib/norikra/typedef_manager.rb
 - lib/norikra/udf.rb
@@ -286,6 +289,8 @@ files:
 - spec/output_pool_spec.rb
 - spec/query_spec.rb
 - spec/spec_helper.rb
+- spec/stats_spec.rb
+- spec/target_spec.rb
 - spec/typedef_manager_spec.rb
 - spec/typedef_spec.rb
 homepage: https://github.com/tagomoris/norikra
@@ -300,12 +305,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: 2
       version: '0'
   none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - '>='
     - !ruby/object:Gem::Version
+      segments:
+      - 0
+      hash: 2
       version: '0'
   none: false
 requirements: []
@@ -320,5 +331,7 @@ test_files:
 - spec/output_pool_spec.rb
 - spec/query_spec.rb
 - spec/spec_helper.rb
+- spec/stats_spec.rb
+- spec/target_spec.rb
 - spec/typedef_manager_spec.rb
 - spec/typedef_spec.rb