norikra 1.1.2-java → 1.2.0-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.ruby-version +1 -1
- data/Changes.md +4 -0
- data/Gemfile +4 -0
- data/README.md +3 -6
- data/lib/norikra/cli.rb +54 -54
- data/lib/norikra/engine.rb +91 -44
- data/lib/norikra/field.rb +12 -7
- data/lib/norikra/fieldset.rb +26 -13
- data/lib/norikra/listener.rb +192 -61
- data/lib/norikra/logger.rb +34 -10
- data/lib/norikra/output_pool.rb +1 -1
- data/lib/norikra/query/ast.rb +240 -20
- data/lib/norikra/query.rb +103 -46
- data/lib/norikra/rpc/handler.rb +3 -3
- data/lib/norikra/rpc/http.rb +1 -1
- data/lib/norikra/server.rb +34 -28
- data/lib/norikra/stats.rb +3 -3
- data/lib/norikra/target.rb +1 -1
- data/lib/norikra/typedef.rb +22 -17
- data/lib/norikra/typedef_manager.rb +9 -6
- data/lib/norikra/udf.rb +6 -2
- data/lib/norikra/version.rb +1 -1
- data/lib/norikra/webui/api.rb +3 -3
- data/lib/norikra/webui/handler.rb +6 -6
- data/norikra.gemspec +0 -1
- data/script/{spec_server_pry → pry} +0 -24
- data/spec/field_spec.rb +14 -0
- data/spec/fieldset_spec.rb +111 -0
- data/spec/listener_spec.rb +64 -26
- data/spec/query_spec.rb +171 -25
- data/spec/spec_helper.rb +0 -46
- data/spec/typedef_manager_spec.rb +15 -9
- data/spec/typedef_spec.rb +11 -11
- metadata +3 -19
- data/lib/norikra/rpc/error.rb +0 -0
- data/lib/norikra/rubyudf.rb +0 -49
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e7e969ece3deb5788c7e7e76c9d6cc4961b4307c
|
4
|
+
data.tar.gz: bbfdc5a4056935c355082877e6ef347b7ded54ec
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4ea0f4f04ba875e2432db51a179d75e3ae18e3a05fc03e945b14fdce187b61da969cd334e11d67aaf8a49b0a032b389f2f57a6c385c98f284b4ecd5bd3156156
|
7
|
+
data.tar.gz: 0add6a6f2539d188f2de82856f1d8cc222f9e6dd60b31b96acbc10167f047d9395c71273dde65a97d23b9b0838d30e0c7661b899d226077def93301e272515da
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
jruby-1.7.
|
1
|
+
jruby-1.7.18
|
data/Changes.md
CHANGED
@@ -3,6 +3,10 @@
|
|
3
3
|
Changes of norikra.
|
4
4
|
|
5
5
|
## v1
|
6
|
+
* v1.2.0
|
7
|
+
* Add `NULLABLE(...)` field to query NULL explicitly
|
8
|
+
* Pluggable listener and user defined listener gems
|
9
|
+
* Fix bug to ignore fields in Esper built-in function
|
6
10
|
* v1.1.2
|
7
11
|
* Enable `-javaagent` option
|
8
12
|
* Fix default/pre-set threads
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -38,12 +38,6 @@ Fix code and tests:
|
|
38
38
|
1. fix code in `lib`
|
39
39
|
1. run `bundle exec rake`
|
40
40
|
|
41
|
-
Run tests faster than 2 or more times:
|
42
|
-
|
43
|
-
1. execute `spork`
|
44
|
-
1. execute `script/spec_server_pry` in another terminal
|
45
|
-
1. run `rspec` in pry console (executed fastly after second times)
|
46
|
-
|
47
41
|
Execute norikra server with target/query continuation:
|
48
42
|
|
49
43
|
1. `bundle exec rake devserver`
|
@@ -70,6 +64,9 @@ See: http://norikra.github.io/
|
|
70
64
|
|
71
65
|
## Changes
|
72
66
|
|
67
|
+
* v1.2
|
68
|
+
* `NULLABLE()` fields
|
69
|
+
* Pluggable listeners
|
73
70
|
* v1.1
|
74
71
|
* Suspend/Resume queries
|
75
72
|
* v1.0
|
data/lib/norikra/cli.rb
CHANGED
@@ -11,70 +11,70 @@ module Norikra
|
|
11
11
|
def self.register_common_start_options(klass)
|
12
12
|
klass.module_exec {
|
13
13
|
### Server options
|
14
|
-
option :host, :
|
15
|
-
option :port, :
|
16
|
-
option :'ui-port', :
|
14
|
+
option :host, type: :string, default: nil, aliases: "-H", desc: 'host address that server listen [0.0.0.0]'
|
15
|
+
option :port, type: :numeric, default: nil, aliases: "-P", desc: 'port that server uses [26571]'
|
16
|
+
option :'ui-port', type: :numeric, default: nil, desc: 'port that Web UI server uses [26578]'
|
17
17
|
|
18
|
-
option :stats, :
|
19
|
-
:
|
20
|
-
option :'stats-secondary', :
|
21
|
-
:
|
22
|
-
option :'suppress-dump-stat', :
|
23
|
-
:
|
24
|
-
option :'dump-stat-interval', :
|
25
|
-
:
|
18
|
+
option :stats, type: :string, default: nil, aliases: "-s", \
|
19
|
+
desc: 'status file path to load/dump targets and queries [none]'
|
20
|
+
option :'stats-secondary', type: :string, default: nil, \
|
21
|
+
desc: 'status file secondary path, to dump stats with date/time, like "stats.%Y%m%d.json" [none]'
|
22
|
+
option :'suppress-dump-stat', type: :boolean, default: false, \
|
23
|
+
desc: 'specify not to update stat file with updated targets/queries on runtime [false]'
|
24
|
+
option :'dump-stat-interval', type: :numeric, default: nil, \
|
25
|
+
desc: 'interval(seconds) of status file dumps on runtime [none (on shutdown only)]'
|
26
26
|
|
27
27
|
### Daemonize options
|
28
|
-
option :daemonize, :
|
29
|
-
:
|
30
|
-
option :pidfile, :
|
31
|
-
:
|
32
|
-
option :outfile, :
|
33
|
-
:
|
28
|
+
option :daemonize, type: :boolean, default: false, aliases: "-d", \
|
29
|
+
desc: 'daemonize Norikra server [false (foreground)]'
|
30
|
+
option :pidfile, type: :string, default: DEFAULT_PID_PATH, aliases: "-p", \
|
31
|
+
desc: "pidfile path when daemonized [#{DEFAULT_PID_PATH}]"
|
32
|
+
option :outfile, type: :string, default: nil, \
|
33
|
+
desc: "stdout redirect file when daemonized [${logdir}/norikra.out]"
|
34
34
|
|
35
35
|
### JVM options
|
36
|
-
option :'bare-jvm', :
|
37
|
-
option :'gc-log', :
|
36
|
+
option :'bare-jvm', type: :boolean, default: false, desc: "use JVM without any recommended options"
|
37
|
+
option :'gc-log', type: :string, default: nil, desc: "output gc logs on specified file path"
|
38
38
|
|
39
39
|
### Performance options
|
40
40
|
# performance predefined configuration sets
|
41
|
-
option :micro, :
|
42
|
-
:
|
43
|
-
option :small, :
|
44
|
-
:
|
45
|
-
option :middle, :
|
46
|
-
:
|
47
|
-
option :large, :
|
48
|
-
:
|
41
|
+
option :micro, type: :boolean, default: false, \
|
42
|
+
desc: 'development or testing (inbound:0, outbound:0, route:0, timer:0, rpc:2)'
|
43
|
+
option :small, type: :boolean, default: false, \
|
44
|
+
desc: 'virtual or small scale servers (inbound:1, outbount:1, route:1, timer:1, rpc:2)'
|
45
|
+
option :middle, type: :boolean, default: false, \
|
46
|
+
desc: 'rackmount servers (inbound:4, outbound:2, route:2, timer:2, rpc:4)'
|
47
|
+
option :large, type: :boolean, default: false, \
|
48
|
+
desc: 'high performance servers (inbound: 6, outbound: 6, route:4, timer:4, rpc: 8)'
|
49
49
|
# Esper
|
50
|
-
option :'inbound-threads', :
|
51
|
-
option :'outbound-threads', :
|
52
|
-
option :'route-threads', :
|
53
|
-
option :'timer-threads', :
|
50
|
+
option :'inbound-threads', type: :numeric, default: nil, desc: 'number of threads for inbound data'
|
51
|
+
option :'outbound-threads', type: :numeric, default: nil, desc: 'number of threads for outbound data'
|
52
|
+
option :'route-threads', type: :numeric, default: nil, desc: 'number of threads for events routing for query execution'
|
53
|
+
option :'timer-threads', type: :numeric, default: nil, desc: 'number of threads for internal timers for query execution'
|
54
54
|
### about capacity options of esper's capacity-bound queue processing, see Esper's thread reference.
|
55
55
|
# http://esper.codehaus.org/esper-4.10.0/doc/reference/en-US/html/configuration.html#config-engine-threading-advanced
|
56
56
|
# default nil: unbound queueing
|
57
|
-
option :'inbound-thread-capacity', :
|
58
|
-
option :'outbound-thread-capacity', :
|
59
|
-
option :'route-thread-capacity', :
|
60
|
-
option :'timer-thread-capacity', :
|
57
|
+
option :'inbound-thread-capacity', type: :numeric, default: nil
|
58
|
+
option :'outbound-thread-capacity', type: :numeric, default: nil
|
59
|
+
option :'route-thread-capacity', type: :numeric, default: nil
|
60
|
+
option :'timer-thread-capacity', type: :numeric, default: nil
|
61
61
|
# Jetty
|
62
|
-
option :'rpc-threads', :
|
63
|
-
option :'web-threads', :
|
62
|
+
option :'rpc-threads', type: :numeric, default: nil, desc: 'number of threads for rpc handlers'
|
63
|
+
option :'web-threads', type: :numeric, default: nil, desc: 'number of threads for WebUI handlers'
|
64
64
|
|
65
65
|
### Logging options
|
66
|
-
option :logdir, :
|
67
|
-
:
|
68
|
-
option :'log-filesize', :
|
69
|
-
option :'log-backups' , :
|
70
|
-
option :'log-buffer-lines', :
|
71
|
-
option :'log4j-properties-path', :
|
66
|
+
option :logdir, type: :string, default: nil, aliases: "-l", \
|
67
|
+
desc: "directory path of logfiles when daemonized [nil (console for foreground)]"
|
68
|
+
option :'log-filesize', type: :string, default: nil, desc: 'log rotation size [10MB]'
|
69
|
+
option :'log-backups' , type: :numeric, default: nil, desc: 'log rotation backups [10]'
|
70
|
+
option :'log-buffer-lines', type: :numeric, default: nil, desc: 'log lines to fetch from API [1000]'
|
71
|
+
option :'log4j-properties-path', type: :string, default: nil, desc: 'path to log4j.properties. ignore other log* options when this option is present'
|
72
72
|
|
73
73
|
### Loglevel options
|
74
|
-
option :'more-quiet', :
|
75
|
-
option :quiet, :
|
76
|
-
option :verbose, :
|
77
|
-
option :'more-verbose', :
|
74
|
+
option :'more-quiet', type: :boolean, default: false, desc: 'set loglevel as ERROR'
|
75
|
+
option :quiet, type: :boolean, default: false, aliases: "-q", desc: 'set loglevel as WARN'
|
76
|
+
option :verbose, type: :boolean, default: false, aliases: "-v", desc: 'set loglevel as DEBUG'
|
77
|
+
option :'more-verbose', type: :boolean, default: false, desc: 'set loglevel as TRACE'
|
78
78
|
}
|
79
79
|
end
|
80
80
|
end
|
@@ -117,7 +117,7 @@ module Norikra
|
|
117
117
|
### 'start' and 'serverprocess' have almost same option set (for parse/help)
|
118
118
|
### DIFF: jvm options (-X)
|
119
119
|
Norikra::CLIUtil.register_common_start_options(self)
|
120
|
-
option :help, :
|
120
|
+
option :help, type: :boolean, default: false, aliases: "-h", desc: "show this message"
|
121
121
|
desc "start [-Xxxx] [other options]", "Start Norikra server process"
|
122
122
|
def start(*optargs)
|
123
123
|
if options[:help]
|
@@ -168,7 +168,7 @@ module Norikra
|
|
168
168
|
File.open(outfile, 'w'){|file| file.write 'write test on parent process'}
|
169
169
|
|
170
170
|
pidfile = File.open(options[:pidfile], 'w')
|
171
|
-
pid = spawn(jruby_path, *args, :
|
171
|
+
pid = spawn(jruby_path, *args, pgroup: 0)
|
172
172
|
pidfile.write(pid.to_s)
|
173
173
|
pidfile.close
|
174
174
|
waiting_child = true
|
@@ -183,12 +183,12 @@ module Norikra
|
|
183
183
|
end
|
184
184
|
|
185
185
|
Norikra::CLIUtil.register_common_start_options(self)
|
186
|
-
desc "serverproc", "execute server process actually (don't execute this subcommand directly)", :
|
186
|
+
desc "serverproc", "execute server process actually (don't execute this subcommand directly)", hide: true
|
187
187
|
def serverproc
|
188
188
|
conf = {}
|
189
189
|
|
190
190
|
if options[:daemonize]
|
191
|
-
conf[:daemonize] = {:
|
191
|
+
conf[:daemonize] = {outfile: options[:outfile]}
|
192
192
|
end
|
193
193
|
|
194
194
|
### stat file
|
@@ -247,9 +247,9 @@ module Norikra
|
|
247
247
|
server.shutdown
|
248
248
|
end
|
249
249
|
|
250
|
-
option :pidfile, :
|
251
|
-
:
|
252
|
-
option :timeout, :
|
250
|
+
option :pidfile, type: :string, default: DEFAULT_PID_PATH, aliases: "-p", \
|
251
|
+
desc: "pidfile path when daemonized [#{DEFAULT_PID_PATH}]"
|
252
|
+
option :timeout, type: :numeric, default: 5, desc: "timeout seconds to wait process exit [5]"
|
253
253
|
desc "stop [options]", "stop daemonized Norikra server"
|
254
254
|
def stop
|
255
255
|
unless test(?r,options[:pidfile])
|
data/lib/norikra/engine.rb
CHANGED
@@ -44,6 +44,9 @@ module Norikra
|
|
44
44
|
@suspended_queries = []
|
45
45
|
|
46
46
|
@waiting_queries = []
|
47
|
+
|
48
|
+
@listeners = []
|
49
|
+
@running_listeners = {} # query_name => listener
|
47
50
|
end
|
48
51
|
|
49
52
|
def statistics
|
@@ -109,7 +112,7 @@ module Norikra
|
|
109
112
|
|
110
113
|
threads = t[sym][:threads].to_i
|
111
114
|
capacity = t[sym][:capacity].to_i
|
112
|
-
info "Engine #{sym} thread pool enabling", :
|
115
|
+
info "Engine #{sym} thread pool enabling", threads: threads, capacity: (capacity == 0 ? 'default' : capacity)
|
113
116
|
|
114
117
|
cam = camelize(sym)
|
115
118
|
threading.send("setThreadPool#{cam}".to_sym, true)
|
@@ -138,7 +141,7 @@ module Norikra
|
|
138
141
|
def open(target_name, fields=nil, auto_field=true)
|
139
142
|
# fields nil || [] => lazy
|
140
143
|
# fields {'fieldname' => 'type'} : type 'string', 'boolean', 'int', 'long', 'float', 'double'
|
141
|
-
info "opening target", :
|
144
|
+
info "opening target", target: target_name, fields: fields, auto_field: auto_field
|
142
145
|
raise Norikra::ArgumentError, "invalid target name" unless Norikra::Target.valid?(target_name)
|
143
146
|
target = Norikra::Target.new(target_name, fields, auto_field)
|
144
147
|
return false if @targets.include?(target)
|
@@ -146,7 +149,7 @@ module Norikra
|
|
146
149
|
end
|
147
150
|
|
148
151
|
def close(target_name)
|
149
|
-
info "closing target", :
|
152
|
+
info "closing target", target: target_name
|
150
153
|
targets = @targets.select{|t| t.name == target_name}
|
151
154
|
return false if targets.size != 1
|
152
155
|
target = targets.first
|
@@ -157,7 +160,7 @@ module Norikra
|
|
157
160
|
end
|
158
161
|
|
159
162
|
def modify(target_name, auto_field)
|
160
|
-
info "modify target", :
|
163
|
+
info "modify target", target: target_name, auto_field: auto_field
|
161
164
|
targets = @targets.select{|t| t.name == target_name}
|
162
165
|
if targets.size != 1
|
163
166
|
raise Norikra::ArgumentError, "target name '#{target_name}' not found"
|
@@ -171,10 +174,12 @@ module Norikra
|
|
171
174
|
end
|
172
175
|
|
173
176
|
def register(query)
|
174
|
-
info "registering query", :
|
177
|
+
info "registering query", name: query.name, targets: query.targets, expression: query.expression
|
175
178
|
raise Norikra::ClientError, "query name '#{query.name}' already exists" if @queries.select{|q| q.name == query.name }.size > 0
|
176
179
|
raise Norikra::ClientError, "query name '#{query.name}' already exists in suspended" if @suspended_queries.select{|q| q.name == query.name }.size > 0
|
177
|
-
|
180
|
+
if reason = query.invalid?
|
181
|
+
raise Norikra::ClientError, "invalid query '#{query.name}': #{reason}"
|
182
|
+
end
|
178
183
|
|
179
184
|
query.targets.each do |target_name|
|
180
185
|
open(target_name) unless @targets.any?{|t| t.name == target_name}
|
@@ -183,7 +188,7 @@ module Norikra
|
|
183
188
|
end
|
184
189
|
|
185
190
|
def deregister(query_name)
|
186
|
-
info "de-registering query", :
|
191
|
+
info "de-registering query", name: query_name
|
187
192
|
queries = @queries.select{|q| q.name == query_name }
|
188
193
|
s_queries = @suspended_queries.select{|q| q.name == query_name }
|
189
194
|
|
@@ -242,12 +247,12 @@ module Norikra
|
|
242
247
|
end
|
243
248
|
|
244
249
|
def send(target_name, events)
|
245
|
-
trace "send messages", :
|
250
|
+
trace "send messages", target: target_name, events: events
|
246
251
|
|
247
252
|
@statistics[:events][:input] += events.size
|
248
253
|
|
249
254
|
unless @targets.any?{|t| t.name == target_name} # discard events for target not registered
|
250
|
-
trace "messages skipped for non-opened target", :
|
255
|
+
trace "messages skipped for non-opened target", target: target_name
|
251
256
|
return
|
252
257
|
end
|
253
258
|
return if events.size < 1
|
@@ -255,19 +260,19 @@ module Norikra
|
|
255
260
|
target = @targets.select{|t| t.name == target_name}.first
|
256
261
|
|
257
262
|
if @typedef_manager.lazy?(target.name)
|
258
|
-
info "opening lazy target", :
|
263
|
+
info "opening lazy target", target: target
|
259
264
|
|
260
265
|
first_event = event_filter(events.first)
|
261
266
|
if first_event.nil? # non-hash object
|
262
267
|
raise Norikra::ClientError, "Input data must be JSON object"
|
263
268
|
end
|
264
|
-
debug "generating base fieldset from event", :
|
269
|
+
debug "generating base fieldset from event", target: target.name, event: first_event
|
265
270
|
base_fieldset = @typedef_manager.generate_base_fieldset(target.name, first_event)
|
266
271
|
|
267
|
-
debug "registering base fieldset", :
|
272
|
+
debug "registering base fieldset", target: target.name, base: base_fieldset
|
268
273
|
register_base_fieldset(target.name, base_fieldset)
|
269
274
|
|
270
|
-
info "target successfully opened with fieldset", :
|
275
|
+
info "target successfully opened with fieldset", target: target, base: base_fieldset
|
271
276
|
end
|
272
277
|
|
273
278
|
registered_data_fieldset = @registered_fieldsets[target_name][:data]
|
@@ -282,18 +287,19 @@ module Norikra
|
|
282
287
|
|
283
288
|
unless registered_data_fieldset[fieldset.summary]
|
284
289
|
# register waiting queries including this fieldset, and this fieldset itself
|
285
|
-
debug "registering unknown fieldset", :
|
290
|
+
debug "registering unknown fieldset", target: target_name, fieldset: fieldset
|
286
291
|
register_fieldset(target_name, fieldset)
|
287
292
|
debug "successfully registered"
|
288
293
|
|
289
294
|
# fieldset should be refined, when waiting_queries rewrite inheritance structure and data fieldset be renewed.
|
290
295
|
fieldset = @typedef_manager.refer(target_name, event, strict_refer)
|
296
|
+
debug "re-referred data fieldset", target: target_name, fieldset: fieldset
|
291
297
|
end
|
292
298
|
|
293
|
-
trace "calling sendEvent with bound fieldset (w/ valid event_type_name)", :
|
294
|
-
trace
|
299
|
+
trace "calling sendEvent with bound fieldset (w/ valid event_type_name)", target: target_name, event: event
|
300
|
+
trace("This is assert for valid event_type_name"){ { event_type_name: fieldset.event_type_name } }
|
295
301
|
formed = fieldset.format(event)
|
296
|
-
trace "sendEvent", :
|
302
|
+
trace "sendEvent", data: formed
|
297
303
|
@runtime.sendEvent(formed.to_java, fieldset.event_type_name)
|
298
304
|
end
|
299
305
|
target.update!
|
@@ -301,8 +307,13 @@ module Norikra
|
|
301
307
|
nil
|
302
308
|
end
|
303
309
|
|
304
|
-
def load(plugin_klass)
|
305
|
-
|
310
|
+
def load(type, plugin_klass)
|
311
|
+
case type
|
312
|
+
when :udf then load_udf(plugin_klass)
|
313
|
+
when :listener then load_listener(plugin_klass)
|
314
|
+
else
|
315
|
+
raise "BUG: unknown plugin type: #{type}"
|
316
|
+
end
|
306
317
|
end
|
307
318
|
|
308
319
|
private
|
@@ -312,7 +323,7 @@ module Norikra
|
|
312
323
|
return false if @targets.include?(target)
|
313
324
|
|
314
325
|
@typedef_manager.add_target(target.name, target.fields)
|
315
|
-
@registered_fieldsets[target.name] = {:
|
326
|
+
@registered_fieldsets[target.name] = {base: {}, query: {}, data: {}}
|
316
327
|
|
317
328
|
unless @typedef_manager.lazy?(target.name)
|
318
329
|
base_fieldset = @typedef_manager.base_fieldset(target.name)
|
@@ -352,7 +363,7 @@ module Norikra
|
|
352
363
|
def update_inherits_graph(target_name, query_fieldset)
|
353
364
|
# replace registered data fieldsets with new fieldset inherits this query fieldset
|
354
365
|
@typedef_manager.supersets(target_name, query_fieldset).each do |set|
|
355
|
-
rebound = set.rebind(true) # update event_type_name with new inheritations
|
366
|
+
rebound = set.rebind(true, query_fieldset) # update event_type_name with new inheritations & nullable fields
|
356
367
|
|
357
368
|
register_fieldset_actually(target_name, rebound, :data, true) # replacing on esper engine
|
358
369
|
@typedef_manager.replace_fieldset(target_name, set, rebound)
|
@@ -362,26 +373,28 @@ module Norikra
|
|
362
373
|
|
363
374
|
def register_query(query)
|
364
375
|
|
365
|
-
if lo_target_name = Norikra::
|
376
|
+
if lo_target_name = Norikra::Listener::Loopback.check(query.group)
|
366
377
|
raise "Invalid loopback target name should be checked before. THIS IS BUG." unless Norikra::Target.valid?(lo_target_name)
|
367
378
|
|
368
379
|
target = Norikra::Target.new(lo_target_name)
|
369
380
|
unless @targets.include?(target)
|
370
|
-
info "opening loopback target", :
|
381
|
+
info "opening loopback target", target: lo_target_name
|
371
382
|
open_target(target)
|
372
383
|
end
|
373
384
|
end
|
374
385
|
|
375
386
|
@mutex.synchronize do
|
376
387
|
raise Norikra::ClientError, "query '#{query.name}' already exists" unless @queries.select{|q| q.name == query.name }.empty?
|
377
|
-
|
378
|
-
|
388
|
+
if reason = query.invalid?
|
389
|
+
raise Norikra::ClientError, "invalid query '#{query.name}': #{reason}"
|
390
|
+
end
|
391
|
+
if lo_target_name = Norikra::Listener::Loopback.check(query.group)
|
379
392
|
raise Norikra::ClientError, "loopback target '#{lo_target_name}'" unless Norikra::Target.valid?(lo_target_name)
|
380
393
|
end
|
381
394
|
|
382
395
|
unless @typedef_manager.ready?(query)
|
383
396
|
@waiting_queries.push(query)
|
384
|
-
trace
|
397
|
+
trace("waiting query fields"){ { targets: query.targets, fields: query.targets.map{|t| query.fields(t)} } }
|
385
398
|
@typedef_manager.register_waiting_fields(query)
|
386
399
|
@queries.push(query)
|
387
400
|
return
|
@@ -389,7 +402,9 @@ module Norikra
|
|
389
402
|
|
390
403
|
mapping = @typedef_manager.generate_fieldset_mapping(query)
|
391
404
|
mapping.each do |target_name, query_fieldset|
|
405
|
+
trace "binding query fieldset", fieldset: query_fieldset
|
392
406
|
@typedef_manager.bind_fieldset(target_name, :query, query_fieldset)
|
407
|
+
trace "registering query fieldset", fieldset: query_fieldset
|
393
408
|
register_fieldset_actually(target_name, query_fieldset, :query)
|
394
409
|
update_inherits_graph(target_name, query_fieldset)
|
395
410
|
query.fieldsets[target_name] = query_fieldset
|
@@ -453,7 +468,9 @@ module Norikra
|
|
453
468
|
ready.each do |query|
|
454
469
|
mapping = @typedef_manager.generate_fieldset_mapping(query)
|
455
470
|
mapping.each do |target_name, query_fieldset|
|
471
|
+
trace "binding query fieldset for waiting query", query: query, target: target_name, fieldset: query_fieldset
|
456
472
|
@typedef_manager.bind_fieldset(target_name, :query, query_fieldset)
|
473
|
+
trace "registering query fieldset", target: target_name, fieldset: query_fieldset
|
457
474
|
register_fieldset_actually(target_name, query_fieldset, :query)
|
458
475
|
update_inherits_graph(target_name, query_fieldset)
|
459
476
|
query.fieldsets[target_name] = query_fieldset
|
@@ -464,12 +481,30 @@ module Norikra
|
|
464
481
|
|
465
482
|
def register_fieldset(target_name, fieldset)
|
466
483
|
@mutex.synchronize do
|
484
|
+
trace "binding data fieldset", fieldset: fieldset # to prepare pickup waiting queries by newly comming fields
|
467
485
|
@typedef_manager.bind_fieldset(target_name, :data, fieldset)
|
468
486
|
|
469
487
|
if @waiting_queries.size > 0
|
470
488
|
register_waiting_queries
|
471
489
|
end
|
472
|
-
|
490
|
+
|
491
|
+
diff_nullable_fields = []
|
492
|
+
|
493
|
+
@typedef_manager.subsets(target_name, fieldset).each do |query_fieldset|
|
494
|
+
next unless query_fieldset.level == :query
|
495
|
+
# fill nullable fields of all required query fieldsets
|
496
|
+
diff_nullable_fields += fieldset.nullable_diff(query_fieldset)
|
497
|
+
end
|
498
|
+
|
499
|
+
unless diff_nullable_fields.empty?
|
500
|
+
trace "query fieldset has nullable diff", diff: diff_nullable_fields
|
501
|
+
fieldset.update(diff_nullable_fields, true) # nullable fields are always optional
|
502
|
+
trace "rebinding data fieldset w/ nullable fields", fieldset: fieldset
|
503
|
+
rebound = fieldset.rebind(false) # type_name is not required to be updated because it is not registered yet
|
504
|
+
@typedef_manager.replace_fieldset(target_name, fieldset, rebound)
|
505
|
+
fieldset = rebound
|
506
|
+
end
|
507
|
+
debug "registering data fieldset", target: target_name, fieldset: fieldset
|
473
508
|
register_fieldset_actually(target_name, fieldset, :data)
|
474
509
|
end
|
475
510
|
end
|
@@ -485,6 +520,10 @@ module Norikra
|
|
485
520
|
end
|
486
521
|
end
|
487
522
|
|
523
|
+
def load_listener(listener_klass)
|
524
|
+
@listeners.push(listener_klass)
|
525
|
+
end
|
526
|
+
|
488
527
|
# this method should be protected with @mutex lock
|
489
528
|
def register_query_actually(query, mapping)
|
490
529
|
# 'mapping' argument is {target => fieldset}
|
@@ -498,13 +537,19 @@ module Norikra
|
|
498
537
|
statement_model = administrator.compileEPL(query.expression)
|
499
538
|
Norikra::Query.rewrite_query(statement_model, event_type_name_map)
|
500
539
|
|
501
|
-
listener =
|
502
|
-
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
540
|
+
listener = nil
|
541
|
+
@listeners.each do |klass|
|
542
|
+
trace("checking listeners"){ {target: klass, result: klass.check(query.group)} }
|
543
|
+
if klass.check(query.group)
|
544
|
+
listener = klass.new(query.name, query.group, @statistics[:events])
|
545
|
+
break
|
546
|
+
end
|
547
|
+
end
|
548
|
+
raise "BUG: no listener is selected" unless listener
|
549
|
+
listener.engine = self if listener.respond_to?(:engine=)
|
550
|
+
listener.output_pool = @output_pool if listener.respond_to?(:output_pool=)
|
551
|
+
listener.start
|
552
|
+
@running_listeners[query.name] = listener
|
508
553
|
|
509
554
|
epl = administrator.create(statement_model)
|
510
555
|
epl.java_send :addListener, [com.espertech.esper.client.UpdateListener.java_class], listener
|
@@ -522,9 +567,11 @@ module Norikra
|
|
522
567
|
epl = administrator.getStatement(query.statement_name)
|
523
568
|
return unless epl
|
524
569
|
|
525
|
-
@output_pool.remove(query.name, query.group)
|
526
570
|
epl.stop unless epl.isStopped
|
527
571
|
epl.destroy unless epl.isDestroyed
|
572
|
+
listener = @running_listeners.delete(query.name)
|
573
|
+
listener.shutdown
|
574
|
+
@output_pool.remove(query.name, query.group)
|
528
575
|
end
|
529
576
|
|
530
577
|
# this method should be protected with @mutex lock
|
@@ -537,15 +584,15 @@ module Norikra
|
|
537
584
|
# .addEventType("AccountUpdate", accountUpdateDef, new String[] {"BaseUpdate"});
|
538
585
|
case level
|
539
586
|
when :base
|
540
|
-
debug
|
587
|
+
debug("add event type"){ { target: target_name, level: 'base', event_type: fieldset.event_type_name } }
|
541
588
|
@config.addEventType(fieldset.event_type_name, fieldset.definition)
|
542
589
|
when :query
|
543
590
|
base_name = @typedef_manager.base_fieldset(target_name).event_type_name
|
544
|
-
debug
|
591
|
+
debug("add event type"){ { target: target_name, level: 'query', event_type: fieldset.event_type_name, base: base_name } }
|
545
592
|
@config.addEventType(fieldset.event_type_name, fieldset.definition, [base_name].to_java(:string))
|
546
|
-
else
|
593
|
+
else # :data
|
547
594
|
subset_names = @typedef_manager.subsets(target_name, fieldset).map(&:event_type_name)
|
548
|
-
debug
|
595
|
+
debug("add event type"){ { target: target_name, level: 'data', event_type: fieldset.event_type_name, inherit: subset_names } }
|
549
596
|
@config.addEventType(fieldset.event_type_name, fieldset.definition, subset_names.to_java(:string))
|
550
597
|
|
551
598
|
@registered_fieldsets[target_name][level][fieldset.summary] = fieldset
|
@@ -559,7 +606,7 @@ module Norikra
|
|
559
606
|
|
560
607
|
# DON'T check @registered_fieldsets[target_name][level][fieldset.summary]
|
561
608
|
# removed fieldset should be already replaced with register_fieldset_actually w/ replace flag
|
562
|
-
debug "remove event type", :
|
609
|
+
debug "remove event type", target: target_name, event_type: event_type_name
|
563
610
|
@config.removeEventType(event_type_name, true)
|
564
611
|
end
|
565
612
|
|
@@ -567,7 +614,7 @@ module Norikra
|
|
567
614
|
FILTER_OPTIMIZABLE_ENUM = com.espertech.esper.client.ConfigurationPlugInSingleRowFunction::FilterOptimizable
|
568
615
|
|
569
616
|
def load_udf_actually(udf)
|
570
|
-
debug "importing class into config object", :
|
617
|
+
debug "importing class into config object", name: udf.class.to_s
|
571
618
|
|
572
619
|
functionName, className, methodName = udf.definition
|
573
620
|
|
@@ -575,17 +622,17 @@ module Norikra
|
|
575
622
|
filterOptimizable = udf.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
|
576
623
|
rethrowExceptions = udf.rethrow_exceptions
|
577
624
|
|
578
|
-
debug "adding SingleRowFunction", :
|
625
|
+
debug "adding SingleRowFunction", function: functionName, javaClass: className, javaMethod: methodName
|
579
626
|
@config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
|
580
627
|
functionName
|
581
628
|
end
|
582
629
|
|
583
630
|
def load_udf_aggregation_actually(udf)
|
584
|
-
debug "importing class into config object", :
|
631
|
+
debug "importing class into config object", name: udf.class.to_s
|
585
632
|
|
586
633
|
functionName, factoryClassName = udf.definition
|
587
634
|
|
588
|
-
debug "adding AggregationSingleFactory", :
|
635
|
+
debug "adding AggregationSingleFactory", function: functionName, javaClass: factoryClassName
|
589
636
|
@config.addPlugInAggregationFunctionFactory(functionName, factoryClassName)
|
590
637
|
functionName
|
591
638
|
end
|