norikra 0.0.11-java → 0.0.12-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.travis.yml ADDED
@@ -0,0 +1,8 @@
1
+ language: ruby
2
+ rvm:
3
+ - jruby-19mode
4
+ jdk:
5
+ - oraclejdk7
6
+ - oraclejdk6
7
+ - openjdk7
8
+ - openjdk6
data/lib/norikra/cli.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  require 'norikra/server'
2
+ require 'norikra/error'
2
3
  require 'thor'
3
4
 
4
5
  module Norikra
@@ -6,9 +7,13 @@ module Norikra
6
7
  desc "start", "start Norikra server process"
7
8
 
8
9
  ### Server options
9
- option :host, :type => :string, :default => '0.0.0.0', :aliases => "-H", :desc => 'host address that server listen [0.0.0.0]'
10
- option :port, :type => :numeric, :default => 26571, :aliases => "-P", :desc => 'port that server uses [26571]'
11
- # option :config, :type => :string, :default => nil, :aliases => "-c", :desc => 'configuration file to define target/query [none]'
10
+ option :host, :type => :string, :default => nil, :aliases => "-H", :desc => 'host address that server listen [0.0.0.0]'
11
+ option :port, :type => :numeric, :default => nil, :aliases => "-P", :desc => 'port that server uses [26571]'
12
+
13
+ option :stats, :type => :string, :default => nil, :aliases => "-s", \
14
+ :desc => 'status file path to load/dump targets, queries and server configurations [none]'
15
+ option :'suppress-dump-stat', :type => :boolean, :default => false, \
16
+ :desc => 'specify not to update stat file with updated targets/queries/configurations on runtime [false]'
12
17
 
13
18
  ### Execution options
14
19
  option :daemonize, :type => :boolean, :default => false, :aliases => "-d", \
@@ -17,20 +22,32 @@ module Norikra
17
22
  :desc => "pidfile path when daemonized [/var/run/norikra.pid]"
18
23
 
19
24
  ### Performance options
20
- option :'inbound-threads', :type => :numeric, :default => 0, :desc => 'number of threads for inbound data'
21
- option :'outbound-threads', :type => :numeric, :default => 0, :desc => 'number of threads for outbound data'
22
- option :'route-threads', :type => :numeric, :default => 0, :desc => 'number of threads for events routing for query execution'
23
- option :'timer-threads', :type => :numeric, :default => 0, :desc => 'number of threads for internal timers for query execution'
24
- option :'inbound-thread-capacity', :type => :numeric, :default => 0
25
- option :'outbound-thread-capacity', :type => :numeric, :default => 0
26
- option :'route-thread-capacity', :type => :numeric, :default => 0
27
- option :'timer-thread-capacity', :type => :numeric, :default => 0
25
+ # performance predefined configuration sets
26
+ option :micro, :type => :boolean, :default => false, \
27
+ :desc => 'development or testing (inbound:0, outbound:0, route:0, timer:0, rpc:2)'
28
+ option :small, :type => :boolean, :default => false, \
29
+ :desc => 'virtual or small scale servers (inbound:1, outbount:1, route:1, timer:1, rpc:2)'
30
+ option :middle, :type => :boolean, :default => false, \
31
+ :desc => 'rackmount servers (inbound:4, outbound:2, route:2, timer:2, rpc:4)'
32
+ option :large, :type => :boolean, :default => false, \
33
+ :desc => 'high performance servers (inbound: 6, outbound: 6, route:4, timer:4, rpc: 8)'
34
+ # Esper
35
+ option :'inbound-threads', :type => :numeric, :default => nil, :desc => 'number of threads for inbound data'
36
+ option :'outbound-threads', :type => :numeric, :default => nil, :desc => 'number of threads for outbound data'
37
+ option :'route-threads', :type => :numeric, :default => nil, :desc => 'number of threads for events routing for query execution'
38
+ option :'timer-threads', :type => :numeric, :default => nil, :desc => 'number of threads for internal timers for query execution'
39
+ option :'inbound-thread-capacity', :type => :numeric, :default => nil
40
+ option :'outbound-thread-capacity', :type => :numeric, :default => nil
41
+ option :'route-thread-capacity', :type => :numeric, :default => nil
42
+ option :'timer-thread-capacity', :type => :numeric, :default => nil
43
+ # Jetty
44
+ option :'rpc-threads', :type => :numeric, :default => nil, :desc => 'number of threads for rpc handlers'
28
45
 
29
46
  ### Logging options
30
47
  option :logdir, :type => :string, :default => nil, :aliases => "-l", \
31
48
  :desc => "directory path of logfiles when daemonized [nil (console)]"
32
- option :'log-filesize', :type => :string, :default => '10MB'
33
- option :'log-backups' , :type => :numeric, :default => 10
49
+ option :'log-filesize', :type => :string, :default => nil, :desc => 'log rotation size [10MB]'
50
+ option :'log-backups' , :type => :numeric, :default => nil, :desc => 'log rotation backups [10]'
34
51
 
35
52
  ### Loglevel options
36
53
  option :'more-quiet', :type => :boolean, :default => false, :desc => 'set loglevel as ERROR'
@@ -38,7 +55,6 @@ module Norikra
38
55
  option :verbose, :type => :boolean, :default => false, :aliases => "-v", :desc => 'set loglevel as DEBUG'
39
56
  option :'more-verbose', :type => :boolean, :default => false, :desc => 'set loglevel as TRACE'
40
57
 
41
- #TODO: configuration file to init
42
58
  def start
43
59
  conf = {}
44
60
 
@@ -46,23 +62,39 @@ module Norikra
46
62
  raise NotImplementedError if options[:daemonize]
47
63
  #TODO: pidcheck if daemonize
48
64
 
65
+ ### stat file
66
+ conf[:stats] = {
67
+ path: options[:stats], suppress: options[:'suppress-dump-stat'],
68
+ }
69
+
70
+ ### threads
71
+ predefined_selecteds = [:micro, :small, :middle, :larage].select{|sym| options[sym]}
72
+ if predefined_selecteds.size > 1
73
+ raise Norikra::ConfigurationError, "one of micro/small/middle/large should be specified"
74
+ end
49
75
  conf[:thread] = {
50
- inbound: {threads: options[:'inbound-threads'], capacity: options[:'inbound-thread-capacity']},
51
- outbound: {threads: options[:'outbound-threads'], capacity: options[:'outbound-thread-capacity']},
52
- route_exec: {threads: options[:'route-threads'], capacity: options[:'route-thread-capacity']},
53
- timer_exec: {threads: options[:'timer-threads'], capacity: options[:'timer-thread-capacity']},
76
+ predefined: predefined_selecteds.first,
77
+ micro: options[:micro], small: options[:small], middle: options[:middle], large: options[:large],
78
+ engine: {inbound:{}, outbound:{}, route_exec:{}, timer_exec:{}},
79
+ rpc: {},
54
80
  }
81
+ [:inbound, :outbound, :route_exec, :timer_exec].each do |sym|
82
+ conf[:thread][:engine][sym][:threads] = options[:"#{sym}-threads"] if options[:"#{sym}-threads"]
83
+ conf[:thread][:engine][sym][:capacity] = options[:"#{sym}-thread-capacity"] if options[:"#{sym}-thread-capacity"]
84
+ end
85
+ conf[:thread][:rpc][:threads] = options[:'rpc-threads'] if options[:'rpc-threads']
55
86
 
56
- conf[:loglevel] = case
57
- when options[:'more-verbose'] then 'TRACE'
58
- when options[:verbose] then 'DEBUG'
59
- when options[:quiet] then 'WARN'
60
- when options[:'more-quiet'] then 'ERROR'
61
- else nil # for default (assumed as 'INFO')
62
- end
63
- conf[:logdir] = options[:logdir]
64
- conf[:logfilesize] = options[:'log-filesize']
65
- conf[:logbackups] = options[:'log-backups']
87
+ ### logs
88
+ loglevel = case
89
+ when options[:'more-verbose'] then 'TRACE'
90
+ when options[:verbose] then 'DEBUG'
91
+ when options[:quiet] then 'WARN'
92
+ when options[:'more-quiet'] then 'ERROR'
93
+ else nil # for default (assumed as 'INFO')
94
+ end
95
+ conf[:log] = {
96
+ level: loglevel, dir: options[:logdir], filesize: options[:'log-filesize'], backups: options[:'log-backups'],
97
+ }
66
98
 
67
99
  server = Norikra::Server.new( options[:host], options[:port], conf )
68
100
  server.run
@@ -1,6 +1,7 @@
1
1
  require 'java'
2
2
 
3
3
  require 'norikra/error'
4
+ require 'norikra/target'
4
5
 
5
6
  require 'norikra/logger'
6
7
  include Norikra::Log
@@ -78,8 +79,11 @@ module Norikra
78
79
  end
79
80
 
80
81
  def open(target, fields=nil)
82
+ # fields nil || [] => lazy
83
+ # fields {'fieldname' => 'type'} : type 'string', 'boolean', 'int', 'long', 'float', 'double'
81
84
  info "opening target", :target => target, :fields => fields
82
85
  return false if @targets.include?(target)
86
+ raise Norikra::ArgumentError, "invalid target name" unless Norikra::Target.valid?(target)
83
87
  open_target(target, fields)
84
88
  end
85
89
 
@@ -161,8 +165,9 @@ module Norikra
161
165
  class Listener
162
166
  include com.espertech.esper.client.UpdateListener
163
167
 
164
- def initialize(query_name, output_pool)
168
+ def initialize(query_name, query_group, output_pool)
165
169
  @query_name = query_name
170
+ @query_group = query_group
166
171
  @output_pool = output_pool
167
172
  end
168
173
 
@@ -182,8 +187,8 @@ module Norikra
182
187
  def update(new_events, old_events)
183
188
  t = Time.now.to_i
184
189
  events = new_events.map{|e| [t, type_convert(e)]}
185
- trace "updated event", :query => @query_name, :event => events
186
- @output_pool.push(@query_name, events)
190
+ trace "updated event", :query => @query_name, :group => @query_group, :event => events
191
+ @output_pool.push(@query_name, @query_group, events)
187
192
  end
188
193
  end
189
194
  ##### Unmatched events are simply ignored
@@ -357,7 +362,7 @@ module Norikra
357
362
  Norikra::Query.rewrite_event_type_name(statement_model, event_type_name_map)
358
363
 
359
364
  epl = administrator.create(statement_model)
360
- epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], Listener.new(query.name, @output_pool)
365
+ epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], Listener.new(query.name, query.group, @output_pool)
361
366
  query.statement_name = epl.getName
362
367
  # epl is automatically started.
363
368
  # epl.isStarted #=> true
data/lib/norikra/error.rb CHANGED
@@ -1,4 +1,5 @@
1
1
  module Norikra
2
+ class ConfigurationError < StandardError; end
2
3
  class ClientError < StandardError; end
3
4
  class ArgumentError < ClientError; end
4
5
  class QueryError < ClientError; end
@@ -1,9 +1,12 @@
1
+ require 'set'
2
+
1
3
  module Norikra
2
4
  class OutputPool
3
5
  attr_accessor :pool
4
6
 
5
7
  def initialize
6
- @pool = {}
8
+ @pool = {} # { query_name => [events] }
9
+ @groups = {} # { group_name => Set(query_names) }
7
10
  @mutex = Mutex.new
8
11
  end
9
12
 
@@ -11,10 +14,19 @@ module Norikra
11
14
  @pool.keys
12
15
  end
13
16
 
14
- def push(query_name, events)
17
+ def push(query_name, query_group, events) # events must be [time(int), event_record]
18
+ # called with blank events for window leavings (and/or other situations)
19
+ return if events.size < 1
20
+
15
21
  @mutex.synchronize do
22
+ if @groups[query_group]
23
+ @groups[query_group].add(query_name) # Set is unique set of elements
24
+ else
25
+ @groups[query_group] ||= Set.new([query_name])
26
+ end
27
+
16
28
  @pool[query_name] ||= []
17
- @pool[query_name].push(events) if events.size > 0
29
+ @pool[query_name].push(events)
18
30
  end
19
31
  end
20
32
 
@@ -27,11 +39,15 @@ module Norikra
27
39
  end
28
40
 
29
41
  # returns {query_name => [[time, event], ...]}
30
- def sweep
42
+ def sweep(group=nil)
43
+ return {} if @groups[group].nil?
44
+
31
45
  ret = {}
32
46
  sweep_pool = @mutex.synchronize do
33
- sweeped = @pool
34
- @pool = {}
47
+ sweeped = {}
48
+ @groups[group].each do |qname|
49
+ sweeped[qname] = @pool.delete(qname) if @pool[qname] && @pool[qname].size > 0
50
+ end
35
51
  sweeped
36
52
  end
37
53
  sweep_pool.keys.each do |k|
data/lib/norikra/query.rb CHANGED
@@ -9,10 +9,11 @@ require 'norikra/query/ast'
9
9
 
10
10
  module Norikra
11
11
  class Query
12
- attr_accessor :name, :expression, :statement_name, :fieldsets
12
+ attr_accessor :name, :group, :expression, :statement_name, :fieldsets
13
13
 
14
14
  def initialize(param={})
15
15
  @name = param[:name]
16
+ @group = param[:group] # default nil
16
17
  @expression = param[:expression]
17
18
  @statement_name = nil
18
19
  @fieldsets = {} # { target => fieldset }
@@ -23,7 +24,7 @@ module Norikra
23
24
  end
24
25
 
25
26
  def dup
26
- self.class.new(:name => @name, :expression => @expression.dup)
27
+ self.class.new(:name => @name, :group => @group, :expression => @expression.dup)
27
28
  end
28
29
 
29
30
  def dup_with_stream_name(actual_name)
@@ -37,7 +38,7 @@ module Norikra
37
38
  end
38
39
 
39
40
  def to_hash
40
- {'name' => @name, 'expression' => @expression, 'targets' => self.targets}
41
+ {'name' => @name, 'group' => @group, 'expression' => @expression, 'targets' => self.targets}
41
42
  end
42
43
 
43
44
  def targets
@@ -56,9 +56,9 @@ class Norikra::RPC::Handler
56
56
  }
57
57
  end
58
58
 
59
- def register(query_name, expression)
60
- logging(:manage, :register, query_name, expression){
61
- r = @engine.register(Norikra::Query.new(:name => query_name, :expression => expression))
59
+ def register(query_name, query_group, expression)
60
+ logging(:manage, :register, query_name, query_group, expression){
61
+ r = @engine.register(Norikra::Query.new(:name => query_name, :group => query_group, :expression => expression))
62
62
  !!r
63
63
  }
64
64
  end
@@ -96,9 +96,9 @@ class Norikra::RPC::Handler
96
96
  }
97
97
  end
98
98
 
99
- def sweep
99
+ def sweep(query_group=nil)
100
100
  logging(:show, :sweep){
101
- @engine.output_pool.sweep
101
+ @engine.output_pool.sweep(query_group)
102
102
  }
103
103
  end
104
104
 
@@ -6,14 +6,20 @@ require_relative 'handler'
6
6
 
7
7
  module Norikra::RPC
8
8
  class HTTP
9
- #TODO Xmx of mizuno/jetty
9
+ DEFAULT_LISTEN_HOST = '0.0.0.0'
10
+ DEFAULT_LISTEN_PORT = 26571
11
+ # 26571 = 3026 + 3014 + 2968 + 2950 + 2891 + 2896 + 2975 + 2979 + 2872
12
+
13
+ DEFAULT_THREADS = 2
14
+
10
15
  attr_accessor :host, :port, :threads
11
16
  attr_accessor :engine, :mizuno, :thread
12
17
 
13
18
  def initialize(opts={})
14
19
  @engine = opts[:engine]
15
- @host = opts[:host]
16
- @port = opts[:port]
20
+ @host = opts[:host] || DEFAULT_LISTEN_HOST
21
+ @port = opts[:port] || DEFAULT_LISTEN_PORT
22
+ @threads = opts[:threads] || DEFAULT_THREADS
17
23
  handler = Norikra::RPC::Handler.new(@engine)
18
24
  @app = Rack::Builder.new {
19
25
  run MessagePack::RPCOverHTTP::Server.app(handler)
@@ -23,7 +29,7 @@ module Norikra::RPC
23
29
  def start
24
30
  @thread = Thread.new do
25
31
  @mizuno = Mizuno::Server.new
26
- @mizuno.run(@app, :embedded => true, :threads => 5, :port => @port, :host => @host)
32
+ @mizuno.run(@app, :embedded => true, :threads => @threads, :port => @port, :host => @host)
27
33
  end
28
34
  end
29
35
 
@@ -1,49 +1,49 @@
1
1
  # this is note for future update
2
2
 
3
- module Norikra
4
- module UDF
5
- class FailedUDFImplementationPureRuby
6
- # require 'jruby/core_ext'
7
- class WootheeIsCrawler < Norikra::UDF::Base # Norikra::UDF::WootheeIsCrawler < Norikra::UDF::Base
8
- def self.init
9
- require 'woothee'
10
- end
11
-
12
- def self.function_name
13
- "isCrawler"
14
- end
15
-
16
- def self.isCrawler(agent)
17
- Woothee.is_crawler(agent)
18
- end
19
- class << self
20
- add_method_signature( "isCrawler", [java.lang.Boolean, java.lang.String] )
21
- end
22
- end
23
-
24
- # for engine.rb
25
- def load_udf_actually(udf_klass)
26
- require 'jruby/core_ext'
27
- udf_klass.init
28
-
29
- jclass = udf_klass.become_java!(".")
30
- className = jclass.get_name.to_java(:string)
31
-
32
- #### try for NullPointerException, but doesn't work well
33
- # field = jclass.getDeclaredField("ruby");
34
- # field.setAccessible(java.lang.Boolean::TRUE)
35
- # field.set(nil, org.jruby.Ruby.getGlobalRuntime)
36
-
37
- functionName = udf_klass.function_name.to_java(:string)
38
- methodName = udf_klass.method_name.to_java(:string)
39
-
40
- valueCache = udf_klass.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
41
- filterOptimizable = udf_klass.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
42
- rethrowExceptions = udf_klass.rethrow_exceptions
43
-
44
- debug "adding SingleRowFunction", :class => udf_klass.to_s, :javaClass => jclass.get_name
45
- @config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
46
- end
47
- end
48
- end
49
- end
3
+ # module Norikra
4
+ # module UDF
5
+ # class FailedUDFImplementationPureRuby
6
+ # # require 'jruby/core_ext'
7
+ # class WootheeIsCrawler < Norikra::UDF::Base # Norikra::UDF::WootheeIsCrawler < Norikra::UDF::Base
8
+ # def self.init
9
+ # require 'woothee'
10
+ # end
11
+
12
+ # def self.function_name
13
+ # "isCrawler"
14
+ # end
15
+
16
+ # def self.isCrawler(agent)
17
+ # Woothee.is_crawler(agent)
18
+ # end
19
+ # class << self
20
+ # add_method_signature( "isCrawler", [java.lang.Boolean, java.lang.String] )
21
+ # end
22
+ # end
23
+
24
+ # # for engine.rb
25
+ # def load_udf_actually(udf_klass)
26
+ # require 'jruby/core_ext'
27
+ # udf_klass.init
28
+
29
+ # jclass = udf_klass.become_java!(".")
30
+ # className = jclass.get_name.to_java(:string)
31
+
32
+ # #### try for NullPointerException, but doesn't work well
33
+ # # field = jclass.getDeclaredField("ruby");
34
+ # # field.setAccessible(java.lang.Boolean::TRUE)
35
+ # # field.set(nil, org.jruby.Ruby.getGlobalRuntime)
36
+
37
+ # functionName = udf_klass.function_name.to_java(:string)
38
+ # methodName = udf_klass.method_name.to_java(:string)
39
+
40
+ # valueCache = udf_klass.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
41
+ # filterOptimizable = udf_klass.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
42
+ # rethrowExceptions = udf_klass.rethrow_exceptions
43
+
44
+ # debug "adding SingleRowFunction", :class => udf_klass.to_s, :javaClass => jclass.get_name
45
+ # @config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
46
+ # end
47
+ # end
48
+ # end
49
+ # end
@@ -1,5 +1,6 @@
1
1
  require 'norikra/engine'
2
2
 
3
+ require 'norikra/stats'
3
4
  require 'norikra/logger'
4
5
  include Norikra::Log
5
6
 
@@ -13,29 +14,102 @@ require 'norikra/udf'
13
14
 
14
15
  module Norikra
15
16
  class Server
16
- RPC_DEFAULT_HOST = '0.0.0.0'
17
- RPC_DEFAULT_PORT = 26571
18
- # 26571 = 3026 + 3014 + 2968 + 2950 + 2891 + 2896 + 2975 + 2979 + 2872
19
-
20
17
  attr_accessor :running
21
18
 
22
- def initialize(host=RPC_DEFAULT_HOST, port=RPC_DEFAULT_PORT, conf={})
23
- #TODO: initial configuration (targets/queries)
19
+ MICRO_PREDEFINED = {
20
+ :engine => { inbound: { threads: 0, capacity: 0 }, outbound: { threads: 0, capacity: 0 },
21
+ route_exec: { threads: 0, capacity: 0 }, timer_exec: { threads: 0, capacity: 0 }, },
22
+ :rpc => { threads: 2 },
23
+ }
24
+ SMALL_PREDEFINED = {
25
+ :engine => { inbound: { threads: 1, capacity: 0 }, outbound: { threads: 1, capacity: 0 },
26
+ route_exec: { threads: 1, capacity: 0 }, timer_exec: { threads: 1, capacity: 0 }, },
27
+ :rpc => { threads: 2 },
28
+ }
29
+ MIDDLE_PREDEFINED = {
30
+ :engine => { inbound: { threads: 4, capacity: 0 }, outbound: { threads: 2, capacity: 0 },
31
+ route_exec: { threads: 2, capacity: 0 }, timer_exec: { threads: 2, capacity: 0 }, },
32
+ :rpc => { threads: 4 },
33
+ }
34
+ LARGE_PREDEFINED = {
35
+ :engine => { inbound: { threads: 6, capacity: 0 }, outbound: { threads: 6, capacity: 0 },
36
+ route_exec: { threads: 4, capacity: 0 }, timer_exec: { threads: 4, capacity: 0 }, },
37
+ :rpc => { threads: 8 },
38
+ }
39
+
40
+ def self.threading_configuration(conf, stats)
41
+ threads = case conf[:predefined]
42
+ when :micro then MICRO_PREDEFINED
43
+ when :small then SMALL_PREDEFINED
44
+ when :middle then MIDDLE_PREDEFINED
45
+ when :large then LARGE_PREDEFINED
46
+ else (stats ? stats.threads : MICRO_PREDEFINED)
47
+ end
48
+ [:inbound, :outbound, :route_exec, :timer_exec].each do |type|
49
+ [:threads, :capacity].each do |item|
50
+ threads[:engine][type][item] = conf[:engine][type][item] if conf[:engine][type][item]
51
+ end
52
+ end
53
+ threads[:rpc][:threads] = conf[:rpc][:threads] if conf[:rpc][:threads]
54
+ threads
55
+ end
56
+
57
+ def self.log_configuration(conf, stats)
58
+ logconf = stats ? stats.log : { level: nil, dir: nil, filesize: nil, backups: nil }
59
+ [:level, :dir, :filesize, :backups].each do |sym|
60
+ logconf[sym] = conf[sym] if conf[sym]
61
+ end
62
+ logconf
63
+ end
64
+
65
+ def initialize(host, port, conf={})
66
+ @stats_path = conf[:stats][:path]
67
+ @stats_suppress_dump = conf[:stats][:suppress]
68
+ @stats = if @stats_path && test(?r, @stats_path)
69
+ Norikra::Stats.load(@stats_path)
70
+ else
71
+ nil
72
+ end
73
+
74
+ @host = host || (@stats ? @stats.host : nil)
75
+ @port = port || (@stats ? @stats.port : nil)
76
+
77
+ @thread_conf = self.class.threading_configuration(conf[:thread], @stats)
78
+ @log_conf = self.class.log_configuration(conf[:log], @stats)
79
+
80
+ Norikra::Log.init(@log_conf[:level], @log_conf[:dir], {:filesize => @log_conf[:filesize], :backups => @log_conf[:backups]})
81
+
82
+ info "thread configurations", @thread_conf
83
+ info "logging configurations", @log_conf
84
+
24
85
  @typedef_manager = Norikra::TypedefManager.new
25
86
  @output_pool = Norikra::OutputPool.new
26
87
 
27
- Norikra::Log.init(conf[:loglevel], conf[:logdir], {:filesize => conf[:logfilesize], :backups => conf[:logbackups]})
28
-
29
- @engine = Norikra::Engine.new(@output_pool, @typedef_manager, {thread: conf[:thread]})
30
- @rpcserver = Norikra::RPC::HTTP.new(:engine => @engine, :port => port)
88
+ @engine = Norikra::Engine.new(@output_pool, @typedef_manager, {thread: @thread_conf[:engine]})
89
+ @rpcserver = Norikra::RPC::HTTP.new(:engine => @engine, :host => @host, :port => @port, :threads => @thread_conf[:rpc][:threads])
31
90
  end
32
91
 
33
92
  def run
34
93
  @engine.start
35
- @rpcserver.start
36
94
 
37
95
  load_plugins
38
96
 
97
+ if @stats
98
+ info "loading from stats file"
99
+ if @stats.targets && @stats.targets.size > 0
100
+ @stats.targets.each do |target|
101
+ @engine.open(target[:name], target[:fields])
102
+ end
103
+ end
104
+ if @stats.queries && @stats.queries.size > 0
105
+ @stats.queries.each do |query|
106
+ @engine.register(Norikra::Query.new(:name => query[:name], :expression => query[:expression]))
107
+ end
108
+ end
109
+ end
110
+
111
+ @rpcserver.start
112
+
39
113
  @running = true
40
114
  info "Norikra server started."
41
115
 
@@ -71,6 +145,21 @@ module Norikra
71
145
  info "Norikra server shutting down."
72
146
  @rpcserver.stop
73
147
  @engine.stop
148
+ info "Norikra server stopped."
149
+
150
+ if @stats_path && !@stats_suppress_dump
151
+ stats = Norikra::Stats.new(
152
+ host: @host,
153
+ port: @port,
154
+ threads: @thread_conf,
155
+ log: @log_conf,
156
+ targets: @engine.typedef_manager.dump,
157
+ queries: @engine.queries.map{|q| {:name => q.name, :expression => q.expression}}
158
+ )
159
+ stats.dump(@stats_path)
160
+ info "Current status saved", :path => @stats_path
161
+ end
162
+
74
163
  info "Norikra server shutdown complete."
75
164
  end
76
165
  end
@@ -0,0 +1,33 @@
1
+ require 'json'
2
+
3
+ module Norikra
4
+ class Stats
5
+ attr_accessor :host, :port, :threads, :log
6
+ attr_accessor :targets, :queries
7
+
8
+ def initialize(opts={})
9
+ @host = opts[:host]
10
+ @port = opts[:port]
11
+ @threads = opts[:threads]
12
+ @log = opts[:log]
13
+ @targets = opts[:targets] || []
14
+ @queries = opts[:queries] || []
15
+ end
16
+
17
+ def to_hash
18
+ {host: @host, port: @port, threads: @threads, log: @log, targets: @targets, queries: @queries}
19
+ end
20
+
21
+ def dump(path)
22
+ File.open(path, 'w') do |file|
23
+ file.write(JSON.pretty_generate(self.to_hash))
24
+ end
25
+ end
26
+
27
+ def self.load(path)
28
+ File.open(path, 'r') do |file|
29
+ self.new(JSON.parse(file.read, symbolize_names: true))
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,7 @@
1
+ module Norikra
2
+ class Target
3
+ def self.valid?(target_name)
4
+ target_name =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/
5
+ end
6
+ end
7
+ end
@@ -15,8 +15,12 @@ module Norikra
15
15
  @optional = optional
16
16
  end
17
17
 
18
- def to_hash
19
- {'name' => @name, 'type' => @type, 'optional' => @optional}
18
+ def to_hash(sym=false)
19
+ if sym
20
+ {name: @name, type: @type, optional: @optional}
21
+ else
22
+ {'name' => @name, 'type' => @type, 'optional' => @optional}
23
+ end
20
24
  end
21
25
 
22
26
  def dup(optional=nil)
@@ -55,7 +59,7 @@ module Norikra
55
59
  when 'float' then 'float'
56
60
  when 'double' then 'double'
57
61
  else
58
- raise Norikra::ArgumentError, "invalid field type #{type}"
62
+ raise Norikra::ArgumentError, "invalid field type '#{type}'"
59
63
  end
60
64
  end
61
65
 
@@ -80,13 +84,13 @@ module Norikra
80
84
  fields.keys.each do |key|
81
85
  data = fields[key]
82
86
  type,optional = if data.is_a?(Hash)
83
- [data[:type], (data.has_key?(:optional) ? data[:optional] : default_optional)]
84
- elsif data.is_a?(String)
85
- [data, default_optional]
87
+ [data[:type].to_s, (data.has_key?(:optional) ? data[:optional] : default_optional)]
88
+ elsif data.is_a?(String) || data.is_a?(Symbol)
89
+ [data.to_s, default_optional]
86
90
  else
87
91
  raise ArgumentError, "FieldSet.new argument class unknown: #{fields.class}"
88
92
  end
89
- @fields[key.to_s] = Field.new(key, type, optional)
93
+ @fields[key.to_s] = Field.new(key.to_s, type, optional)
90
94
  end
91
95
  self.update_summary
92
96
 
@@ -366,5 +370,13 @@ module Norikra
366
370
  end
367
371
  ret
368
372
  end
373
+
374
+ def dump
375
+ fields = {}
376
+ @fields.map{|key,field|
377
+ fields[key.to_sym] = field.to_hash(true)
378
+ }
379
+ fields
380
+ end
369
381
  end
370
382
  end
@@ -140,5 +140,11 @@ module Norikra
140
140
  def format(target, event)
141
141
  @typedefs[target].format(event)
142
142
  end
143
+
144
+ def dump
145
+ @typedefs.keys.map{|target|
146
+ {:name => target, :fields => @typedefs[target].dump}
147
+ }
148
+ end
143
149
  end
144
150
  end
@@ -1,3 +1,3 @@
1
1
  module Norikra
2
- VERSION = "0.0.11"
2
+ VERSION = "0.0.12"
3
3
  end
data/norikra.gemspec CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |spec|
23
23
  spec.add_runtime_dependency "rack"
24
24
  spec.add_runtime_dependency "msgpack-rpc-over-http-jruby", ">= 0.0.5"
25
25
  spec.add_runtime_dependency "thor"
26
- spec.add_runtime_dependency "norikra-client-jruby", ">= 0.0.4"
26
+ spec.add_runtime_dependency "norikra-client-jruby", ">= 0.0.6"
27
27
 
28
28
  spec.add_development_dependency "bundler", "~> 1.3"
29
29
  spec.add_development_dependency "rake"
@@ -12,15 +12,18 @@ describe Norikra::OutputPool do
12
12
 
13
13
  describe '#push' do
14
14
  context 'with empty array' do
15
- subject { p = Norikra::OutputPool.new; p.push('TestTable query1', []); p }
16
- its(:pool){ should == {'TestTable query1' => []} }
15
+ it 'will be ignored' do
16
+ p = Norikra::OutputPool.new
17
+ p.push('TestTable query1', nil, [])
18
+ expect(p.pool).to eql({})
19
+ end
17
20
  end
18
21
 
19
22
  context 'with event array' do
20
23
  it 'has pool with event' do
21
24
  pool = Norikra::OutputPool.new
22
25
  t = Time.now.to_i
23
- pool.push('TestTable query1', [{'count'=>1},{'count'=>2}])
26
+ pool.push('TestTable query1',nil,[[t, {'count'=>1}],[t ,{'count'=>2}]])
24
27
 
25
28
  pool.pool.keys.should eql(['TestTable query1'])
26
29
  events = pool.pool['TestTable query1']
@@ -46,8 +49,8 @@ describe Norikra::OutputPool do
46
49
  it 'returns all events of specified query in pool' do
47
50
  pool = Norikra::OutputPool.new
48
51
  t = Time.now.to_i
49
- pool.push('TestTable query1', [{'count'=>1},{'count'=>2}])
50
- pool.push('TestTable query2', [{'count'=>3},{'count'=>4}])
52
+ pool.push('TestTable query1', nil, [[t,{'count'=>1}],[t,{'count'=>2}]])
53
+ pool.push('TestTable query2', nil, [[t,{'count'=>3}],[t,{'count'=>4}]])
51
54
 
52
55
  expect(pool.pop('TestTable query0').size).to eql(0)
53
56
  expect(pool.pop('TestTable query1').size).to eql(2)
@@ -57,18 +60,42 @@ describe Norikra::OutputPool do
57
60
  end
58
61
 
59
62
  describe '#sweep' do
60
- it 'returns all events for all queries in pool' do
61
- pool = Norikra::OutputPool.new
62
- t = Time.now.to_i
63
- pool.push('TestTable query1', [{'count'=>1},{'count'=>2}])
64
- pool.push('TestTable query2', [{'count'=>3},{'count'=>4},{'count'=>5}])
63
+ context 'with default query group' do
64
+ it 'returns all events for all queries in pool' do
65
+ pool = Norikra::OutputPool.new
66
+ t = Time.now.to_i
67
+ pool.push('TestTable query1', nil, [[t,{'count'=>1}],[t,{'count'=>2}]])
68
+ pool.push('TestTable query2', nil, [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
69
+ pool.push('TestTable query3', 'x', [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
65
70
 
66
- chunk = pool.sweep
67
- expect(chunk.keys.size).to eql(2)
71
+ chunk = pool.sweep
72
+ expect(chunk.keys.size).to eql(2)
68
73
 
69
- expect(chunk['TestTable query1'].size).to eql(2)
70
- expect(chunk['TestTable query2'].size).to eql(3)
71
- expect(chunk['TestTable query2'].last.last['count']).to eql(5)
74
+ expect(chunk['TestTable query1'].size).to eql(2)
75
+ expect(chunk['TestTable query2'].size).to eql(3)
76
+ expect(chunk['TestTable query2'].last.last['count']).to eql(5)
77
+ end
78
+ end
79
+
80
+ context 'with group specified' do
81
+ it 'returns all events for all queries in pool' do
82
+ pool = Norikra::OutputPool.new
83
+ t = Time.now.to_i
84
+ pool.push('TestTable query1', nil, [[t,{'count'=>1}],[t,{'count'=>2}]])
85
+ pool.push('TestTable query2', nil, [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
86
+ pool.push('TestTable query3', 'x', [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
87
+ pool.push('TestTable query4', 'y', [[t,{'count'=>1}],[t,{'count'=>2}]])
88
+ pool.push('TestTable query5', 'x', [[t,{'count'=>9}]])
89
+ pool.push('TestTable query6', 'x', [[t,{'count'=>3}],[t,{'count'=>4}],[t,{'count'=>5}]])
90
+ pool.push('TestTable query6', 'x', [[t,{'count'=>6}],[t,{'count'=>7}]])
91
+
92
+ chunk = pool.sweep('x')
93
+ expect(chunk.keys.size).to eql(3)
94
+
95
+ expect(chunk['TestTable query3'].size).to eql(3)
96
+ expect(chunk['TestTable query5'].size).to eql(1)
97
+ expect(chunk['TestTable query6'].size).to eql(5)
98
+ end
72
99
  end
73
100
  end
74
101
  end
data/spec/query_spec.rb CHANGED
@@ -12,6 +12,7 @@ describe Norikra::Query do
12
12
  :name => 'TestTable query1', :expression => expression
13
13
  )
14
14
  expect(q.name).to eql('TestTable query1')
15
+ expect(q.group).to be_nil
15
16
  expect(q.expression).to eql(expression)
16
17
  expect(q.targets).to eql(['TestTable'])
17
18
 
@@ -25,9 +26,10 @@ describe Norikra::Query do
25
26
  it 'returns query instances collectly parsed' do
26
27
  expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) AS source WHERE source.path="/" AND Math.abs(-1 * source.size) > 3'
27
28
  q = Norikra::Query.new(
28
- :name => 'TestTable query2', :expression => expression
29
+ :name => 'TestTable query2', :group => 'label1', :expression => expression
29
30
  )
30
31
  expect(q.name).to eql('TestTable query2')
32
+ expect(q.group).to eql('label1')
31
33
  expect(q.expression).to eql(expression)
32
34
  expect(q.targets).to eql(['TestTable'])
33
35
 
@@ -106,7 +108,31 @@ describe Norikra::Query do
106
108
  end
107
109
  end
108
110
 
109
- describe '#dup_with_stream_name' do
111
+ describe '#dup' do
112
+ context 'for queries without group (default group)' do
113
+ it 'returns query object with default group' do
114
+ e1 = 'SELECT max(num) AS max FROM TestTable1.win:time(5 sec)'
115
+ query = Norikra::Query.new(:name => 'q1', :group => nil, :expression => e1)
116
+ q = query.dup
117
+ expect(q.name).to eql('q1')
118
+ expect(q.group).to be_nil
119
+ expect(q.expression).to eql(e1)
120
+ end
121
+ end
122
+
123
+ context 'for queries with group' do
124
+ it 'returns query object with specified group' do
125
+ e2 = 'SELECT max(num) AS max FROM TestTable2.win:time(5 sec)'
126
+ query = Norikra::Query.new(:name => 'q2', :group => 'g2', :expression => e2)
127
+ q = query.dup
128
+ expect(q.name).to eql('q2')
129
+ expect(q.group).to eql('g2')
130
+ expect(q.expression).to eql(e2)
131
+ end
132
+ end
133
+ end
134
+
135
+ describe '#dup_with_stream_name' do
110
136
  context 'with simple query' do
111
137
  expression = 'SELECT count(*) AS cnt FROM TestTable.win:time_batch(10 sec) WHERE path="/" AND size > 100 and param.length() > 0'
112
138
  it 'returns duplicated object, with replaced ' do
@@ -0,0 +1,53 @@
1
+ require_relative './spec_helper'
2
+
3
+ require 'norikra/stats'
4
+ require 'norikra/server'
5
+
6
+ require 'tmpdir'
7
+
8
+ describe Norikra::Stats do
9
+ describe '#to_hash' do
10
+ it 'returns internal stats as hash with symbolized keys' do
11
+ args = {
12
+ host: nil,
13
+ port: nil,
14
+ threads: Norikra::Server::SMALL_PREDEFINED,
15
+ log: {level: nil, dir: nil, filesize: nil, backups: nil},
16
+ targets: [],
17
+ queries: [],
18
+ }
19
+ s = Norikra::Stats.new(args)
20
+ expect(s.to_hash).to eql(args)
21
+ end
22
+ end
23
+
24
+ describe '.load' do
25
+ it 'can restore stats data from #dump -ed json' do
26
+ Dir.mktmpdir do |dir|
27
+ File.open("#{dir}/stats.json", 'w') do |file|
28
+ args = {
29
+ host: nil,
30
+ port: nil,
31
+ threads: Norikra::Server::LARGE_PREDEFINED,
32
+ log: {level: 'WARN', dir: '/var/log/norikra', filesize: '50MB', backups: 300},
33
+ targets: [
34
+ { name: 'test1', fields: { id: { name: 'id', type: 'int', optional: false}, data: { name: 'data', type: 'string', optional: true } } },
35
+ ],
36
+ queries: [
37
+ { name: 'testq2', expression: 'select count(*) from test1.win:time(5 sec)' },
38
+ { name: 'testq1', expression: 'select count(*) from test1.win:time(10 sec)' },
39
+ ],
40
+ }
41
+ s1 = Norikra::Stats.new(args)
42
+ expect(s1.to_hash).to eql(args)
43
+
44
+ s1.dump(file.path)
45
+
46
+ s2 = Norikra::Stats.load(file.path)
47
+ expect(s2.to_hash).to eql(s1.to_hash)
48
+ expect(s2.to_hash).to eql(args)
49
+ end
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,24 @@
1
+ require_relative './spec_helper'
2
+
3
+ require 'norikra/target'
4
+ # require 'norikra/error'
5
+
6
+ describe Norikra::Target do
7
+ describe '.valid?' do
8
+ it 'raises Norikra::ArgumentError for invalid name' do
9
+ expect(Norikra::Target.valid?('foobar')).to be_true
10
+ expect(Norikra::Target.valid?('FooBar')).to be_true
11
+ expect(Norikra::Target.valid?('foo_bar')).to be_true
12
+ expect(Norikra::Target.valid?('foo_bar_baz')).to be_true
13
+
14
+ expect(Norikra::Target.valid?('')).to be_false
15
+ expect(Norikra::Target.valid?('.')).to be_false
16
+ expect(Norikra::Target.valid?('_')).to be_false
17
+ expect(Norikra::Target.valid?('_a_')).to be_false
18
+ expect(Norikra::Target.valid?('foo_')).to be_false
19
+ expect(Norikra::Target.valid?('_Foo')).to be_false
20
+ expect(Norikra::Target.valid?('foo bar')).to be_false
21
+ expect(Norikra::Target.valid?('_Foo')).to be_false
22
+ end
23
+ end
24
+ end
@@ -200,5 +200,31 @@ describe Norikra::TypedefManager do
200
200
  expect(r.summary).to eql('a:string,b:string,c:double,f:boolean')
201
201
  end
202
202
  end
203
+
204
+ describe '#dump' do
205
+ it 'returns array of target typedef dump' do
206
+ m = Norikra::TypedefManager.new
207
+ m.add_target('sample', {'a'=>'string','b'=>'string','c'=>'double'})
208
+ m.reserve('sample', 'z', 'boolean')
209
+ m.add_target('sample_next', {'a'=>'string','b'=>'string','c'=>'double','d'=>'double'})
210
+
211
+ r = m.dump
212
+ expect(r.size).to eql(2)
213
+ expect(r[0][:name]).to eql('sample')
214
+ expect(r[0][:fields]).to eql({
215
+ a: {name: 'a', type: 'string', optional: false},
216
+ b: {name: 'b', type: 'string', optional: false},
217
+ c: {name: 'c', type: 'double', optional: false},
218
+ z: {name: 'z', type: 'boolean', optional: true},
219
+ })
220
+ expect(r[1][:name]).to eql('sample_next')
221
+ expect(r[1][:fields]).to eql({
222
+ a: {name: 'a', type: 'string', optional: false},
223
+ b: {name: 'b', type: 'string', optional: false},
224
+ c: {name: 'c', type: 'double', optional: false},
225
+ d: {name: 'd', type: 'double', optional: false},
226
+ })
227
+ end
228
+ end
203
229
  end
204
230
  end
data/spec/typedef_spec.rb CHANGED
@@ -296,5 +296,27 @@ describe Norikra::Typedef do
296
296
  expect(d['d']).to eql(3.14)
297
297
  end
298
298
  end
299
+
300
+ describe '#dump' do
301
+ it 'returns hash instance to show fields and its types/optionals' do
302
+ t = Norikra::Typedef.new({'a'=>'string','b'=>'long'})
303
+ t.push(:query, Norikra::FieldSet.new({'a'=>'string','b'=>'long'}))
304
+ t.push(:data, Norikra::FieldSet.new({'a'=>'string','b'=>'long'}))
305
+ t.push(:data, Norikra::FieldSet.new({'a'=>'string','b'=>'long','c'=>'double'}))
306
+ t.push(:query, Norikra::FieldSet.new({'a'=>'string','b'=>'long','d'=>'string'}))
307
+ fields = t.fields
308
+
309
+ r = t.dump
310
+ expect(r.keys.sort).to eql([:a, :b, :c, :d])
311
+ expect(r[:a]).to eql({name: 'a', type: 'string', optional: false})
312
+ expect(r[:b]).to eql({name: 'b', type: 'long', optional: false})
313
+ expect(r[:c]).to eql({name: 'c', type: 'double', optional: true})
314
+ expect(r[:d]).to eql({name: 'd', type: 'string', optional: true})
315
+
316
+ t2 = Norikra::Typedef.new(r)
317
+ expect(t2.fields.keys.sort).to eql(fields.keys.sort)
318
+ expect(t2.fields.values.map(&:to_hash)).to eql(fields.values.map(&:to_hash))
319
+ end
320
+ end
299
321
  end
300
322
  end
metadata CHANGED
@@ -2,14 +2,14 @@
2
2
  name: norikra
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.0.11
5
+ version: 0.0.12
6
6
  platform: java
7
7
  authors:
8
8
  - TAGOMORI Satoshi
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-08-27 00:00:00.000000000 Z
12
+ date: 2013-09-03 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: mizuno
@@ -81,13 +81,13 @@ dependencies:
81
81
  requirements:
82
82
  - - '>='
83
83
  - !ruby/object:Gem::Version
84
- version: 0.0.4
84
+ version: 0.0.6
85
85
  none: false
86
86
  requirement: !ruby/object:Gem::Requirement
87
87
  requirements:
88
88
  - - '>='
89
89
  - !ruby/object:Gem::Version
90
- version: 0.0.4
90
+ version: 0.0.6
91
91
  none: false
92
92
  prerelease: false
93
93
  type: :runtime
@@ -183,6 +183,7 @@ extra_rdoc_files: []
183
183
  files:
184
184
  - .gitignore
185
185
  - .ruby-version
186
+ - .travis.yml
186
187
  - Gemfile
187
188
  - LICENSE
188
189
  - README.md
@@ -274,6 +275,8 @@ files:
274
275
  - lib/norikra/rpc/http.rb
275
276
  - lib/norikra/rubyudf.rb
276
277
  - lib/norikra/server.rb
278
+ - lib/norikra/stats.rb
279
+ - lib/norikra/target.rb
277
280
  - lib/norikra/typedef.rb
278
281
  - lib/norikra/typedef_manager.rb
279
282
  - lib/norikra/udf.rb
@@ -286,6 +289,8 @@ files:
286
289
  - spec/output_pool_spec.rb
287
290
  - spec/query_spec.rb
288
291
  - spec/spec_helper.rb
292
+ - spec/stats_spec.rb
293
+ - spec/target_spec.rb
289
294
  - spec/typedef_manager_spec.rb
290
295
  - spec/typedef_spec.rb
291
296
  homepage: https://github.com/tagomoris/norikra
@@ -300,12 +305,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
300
305
  requirements:
301
306
  - - '>='
302
307
  - !ruby/object:Gem::Version
308
+ segments:
309
+ - 0
310
+ hash: 2
303
311
  version: '0'
304
312
  none: false
305
313
  required_rubygems_version: !ruby/object:Gem::Requirement
306
314
  requirements:
307
315
  - - '>='
308
316
  - !ruby/object:Gem::Version
317
+ segments:
318
+ - 0
319
+ hash: 2
309
320
  version: '0'
310
321
  none: false
311
322
  requirements: []
@@ -320,5 +331,7 @@ test_files:
320
331
  - spec/output_pool_spec.rb
321
332
  - spec/query_spec.rb
322
333
  - spec/spec_helper.rb
334
+ - spec/stats_spec.rb
335
+ - spec/target_spec.rb
323
336
  - spec/typedef_manager_spec.rb
324
337
  - spec/typedef_spec.rb