norikra 0.0.7-java → 0.0.8-java
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +2 -0
- data/README.md +6 -1
- data/lib/norikra/engine.rb +42 -0
- data/lib/norikra/rubyudf.rb +49 -0
- data/lib/norikra/server.rb +21 -1
- data/lib/norikra/udf.rb +124 -0
- data/lib/norikra/udf_spec_helper.rb +113 -0
- data/lib/norikra/version.rb +1 -1
- metadata +5 -2
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -86,6 +86,12 @@ And send more events:
|
|
86
86
|
|
87
87
|
Query 'www.search' matches the last event automatically.
|
88
88
|
|
89
|
+
## User Defined Functions
|
90
|
+
|
91
|
+
UDF as plugins over rubygems are available. For example, see 'norikra-udf-mock' repository.
|
92
|
+
|
93
|
+
TBD
|
94
|
+
|
89
95
|
## Versions
|
90
96
|
|
91
97
|
TBD
|
@@ -94,7 +100,6 @@ TBD
|
|
94
100
|
|
95
101
|
* daemonize
|
96
102
|
* performance parameters
|
97
|
-
* query unregister
|
98
103
|
|
99
104
|
## Copyright
|
100
105
|
|
data/lib/norikra/engine.rb
CHANGED
@@ -123,6 +123,10 @@ module Norikra
|
|
123
123
|
nil
|
124
124
|
end
|
125
125
|
|
126
|
+
def load(plugin_klass)
|
127
|
+
load_udf(plugin_klass)
|
128
|
+
end
|
129
|
+
|
126
130
|
class Listener
|
127
131
|
include com.espertech.esper.client.UpdateListener
|
128
132
|
|
@@ -282,6 +286,17 @@ module Norikra
|
|
282
286
|
end
|
283
287
|
end
|
284
288
|
|
289
|
+
def load_udf(udf_klass)
|
290
|
+
udf_klass.init if udf_klass.respond_to?(:init)
|
291
|
+
|
292
|
+
udf = udf_klass.new
|
293
|
+
if udf.is_a? Norikra::UDF::SingleRow
|
294
|
+
load_udf_actually(udf)
|
295
|
+
elsif udf.is_a? Norikra::UDF::AggregationSingle
|
296
|
+
load_udf_aggregation_actually(udf)
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
285
300
|
# this method should be protected with @mutex lock
|
286
301
|
def register_query_actually(query, mapping)
|
287
302
|
# 'mapping' argument is {target => fieldset}
|
@@ -347,5 +362,32 @@ module Norikra
|
|
347
362
|
debug "remove event type", :target => target, :event_type => event_type_name
|
348
363
|
@config.removeEventType(event_type_name, true)
|
349
364
|
end
|
365
|
+
|
366
|
+
VALUE_CACHE_ENUM = com.espertech.esper.client.ConfigurationPlugInSingleRowFunction::ValueCache
|
367
|
+
FILTER_OPTIMIZABLE_ENUM = com.espertech.esper.client.ConfigurationPlugInSingleRowFunction::FilterOptimizable
|
368
|
+
|
369
|
+
def load_udf_actually(udf)
|
370
|
+
debug "importing class into config object", :name => udf.class.to_s
|
371
|
+
|
372
|
+
functionName, className, methodName = udf.definition
|
373
|
+
|
374
|
+
valueCache = udf.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
|
375
|
+
filterOptimizable = udf.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
|
376
|
+
rethrowExceptions = udf.rethrow_exceptions
|
377
|
+
|
378
|
+
debug "adding SingleRowFunction", :function => functionName, :javaClass => className, :javaMethod => methodName
|
379
|
+
@config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
|
380
|
+
functionName
|
381
|
+
end
|
382
|
+
|
383
|
+
def load_udf_aggregation_actually(udf)
|
384
|
+
debug "importing class into config object", :name => udf.class.to_s
|
385
|
+
|
386
|
+
functionName, factoryClassName = udf.definition
|
387
|
+
|
388
|
+
debug "adding AggregationSingleFactory", :function => functionName, :javaClass => factoryClassName
|
389
|
+
@config.addPlugInAggregationFunctionFactory(functionName, factoryClassName)
|
390
|
+
functionName
|
391
|
+
end
|
350
392
|
end
|
351
393
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# this is note for future update
|
2
|
+
|
3
|
+
module Norikra
|
4
|
+
module UDF
|
5
|
+
class FailedUDFImplementationPureRuby
|
6
|
+
# require 'jruby/core_ext'
|
7
|
+
class WootheeIsCrawler < Norikra::UDF::Base # Norikra::UDF::WootheeIsCrawler < Norikra::UDF::Base
|
8
|
+
def self.init
|
9
|
+
require 'woothee'
|
10
|
+
end
|
11
|
+
|
12
|
+
def self.function_name
|
13
|
+
"isCrawler"
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.isCrawler(agent)
|
17
|
+
Woothee.is_crawler(agent)
|
18
|
+
end
|
19
|
+
class << self
|
20
|
+
add_method_signature( "isCrawler", [java.lang.Boolean, java.lang.String] )
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# for engine.rb
|
25
|
+
def load_udf_actually(udf_klass)
|
26
|
+
require 'jruby/core_ext'
|
27
|
+
udf_klass.init
|
28
|
+
|
29
|
+
jclass = udf_klass.become_java!(".")
|
30
|
+
className = jclass.get_name.to_java(:string)
|
31
|
+
|
32
|
+
#### try for NullPointerException, but doesn't work well
|
33
|
+
# field = jclass.getDeclaredField("ruby");
|
34
|
+
# field.setAccessible(java.lang.Boolean::TRUE)
|
35
|
+
# field.set(nil, org.jruby.Ruby.getGlobalRuntime)
|
36
|
+
|
37
|
+
functionName = udf_klass.function_name.to_java(:string)
|
38
|
+
methodName = udf_klass.method_name.to_java(:string)
|
39
|
+
|
40
|
+
valueCache = udf_klass.value_cache ? VALUE_CACHE_ENUM::ENABLED : VALUE_CACHE_ENUM::DISABLED
|
41
|
+
filterOptimizable = udf_klass.filter_optimizable ? FILTER_OPTIMIZABLE_ENUM::ENABLED : FILTER_OPTIMIZABLE_ENUM::DISABLED
|
42
|
+
rethrowExceptions = udf_klass.rethrow_exceptions
|
43
|
+
|
44
|
+
debug "adding SingleRowFunction", :class => udf_klass.to_s, :javaClass => jclass.get_name
|
45
|
+
@config.addPlugInSingleRowFunction(functionName, className, methodName, valueCache, filterOptimizable, rethrowExceptions)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/norikra/server.rb
CHANGED
@@ -9,6 +9,7 @@ require 'norikra/typedef'
|
|
9
9
|
require 'norikra/query'
|
10
10
|
|
11
11
|
require 'norikra/rpc'
|
12
|
+
require 'norikra/udf'
|
12
13
|
|
13
14
|
module Norikra
|
14
15
|
class Server
|
@@ -32,8 +33,11 @@ module Norikra
|
|
32
33
|
def run
|
33
34
|
@engine.start
|
34
35
|
@rpcserver.start
|
35
|
-
|
36
|
+
|
37
|
+
load_plugins
|
38
|
+
|
36
39
|
@running = true
|
40
|
+
info "Norikra server started."
|
37
41
|
|
38
42
|
shutdown_proc = ->{ @running = false }
|
39
43
|
# JVM uses SIGQUIT for thread/heap state dumping
|
@@ -47,6 +51,22 @@ module Norikra
|
|
47
51
|
end
|
48
52
|
end
|
49
53
|
|
54
|
+
def load_plugins
|
55
|
+
info "Loading UDF plugins"
|
56
|
+
Norikra::UDF.listup.each do |mojule|
|
57
|
+
if mojule.is_a?(Class)
|
58
|
+
name = @engine.load(mojule)
|
59
|
+
info "UDF loaded", :name => name
|
60
|
+
elsif mojule.is_a?(Module) && mojule.respond_to?(:plugins)
|
61
|
+
mojule.init if mojule.respond_to?(:init)
|
62
|
+
mojule.plugins.each do |klass|
|
63
|
+
name = @engine.load(klass)
|
64
|
+
info "UDF loaded", :name => name
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
50
70
|
def shutdown
|
51
71
|
info "Norikra server shutting down."
|
52
72
|
@rpcserver.stop
|
data/lib/norikra/udf.rb
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'norikra/error'
|
2
|
+
require 'rubygems'
|
3
|
+
|
4
|
+
module Norikra
|
5
|
+
module UDF
|
6
|
+
#### esper-4.9.0/esper/doc/reference/html/extension.html#custom-singlerow-function
|
7
|
+
# <esper-configuration
|
8
|
+
# <plugin-singlerow-function name="getDate"
|
9
|
+
# function-class="mycompany.DateUtil" function-method="parseDate"
|
10
|
+
# value-cache="enabled"
|
11
|
+
# filter-optimizable="disabled"
|
12
|
+
# rethrow-exceptions="disabled" />
|
13
|
+
# </esper-configuration>
|
14
|
+
class SingleRow
|
15
|
+
def self.init
|
16
|
+
true
|
17
|
+
end
|
18
|
+
|
19
|
+
def definition
|
20
|
+
[self.function_name, self.class_name, self.method_name]
|
21
|
+
end
|
22
|
+
|
23
|
+
# UDF function name in queries
|
24
|
+
def function_name
|
25
|
+
raise NotImplementedError
|
26
|
+
end
|
27
|
+
|
28
|
+
def class_name
|
29
|
+
raise NotImplementedError
|
30
|
+
end
|
31
|
+
|
32
|
+
def method_name
|
33
|
+
function_name
|
34
|
+
end
|
35
|
+
|
36
|
+
# 17.3.3. Value Cache
|
37
|
+
# When a single-row function receives parameters that are all constant values or expressions
|
38
|
+
# that themselves receive only constant values, Esper can pre-evaluate the result of
|
39
|
+
# the single-row function at time of statement creation.
|
40
|
+
# By default, Esper does not pre-evaluate the single-row function unless you configure
|
41
|
+
# the value cache as enabled.
|
42
|
+
def value_cache
|
43
|
+
false
|
44
|
+
end
|
45
|
+
|
46
|
+
# 17.3.4. Single-Row Functions in Filter Predicate Expressions
|
47
|
+
# Your EPL may use plug-in single row functions among the predicate expressions
|
48
|
+
# as part of the filters in a stream or pattern.
|
49
|
+
# For example, the EPL below uses the function computeHash as part of a predicate expression:
|
50
|
+
#
|
51
|
+
# select * from MyEvent(computeHash(field) = 100)
|
52
|
+
#
|
53
|
+
# When you have many EPL statements or many context partitions that refer to the same function,
|
54
|
+
# event type and parameters in a predicate expression, the engine may optimize evaluation:
|
55
|
+
# The function gets evaluated only once per event.
|
56
|
+
#
|
57
|
+
# While the optimization is enabled by default for all plug-in single row functions,
|
58
|
+
# you can also disable the optimization for a specific single-row function.
|
59
|
+
# By disabling the optimization for a single-row function the engine may use less memory
|
60
|
+
# to identify reusable function footprints but may cause the engine to evaluate each function
|
61
|
+
# more frequently then necessary.
|
62
|
+
def filter_optimizable
|
63
|
+
true
|
64
|
+
end
|
65
|
+
|
66
|
+
# 17.3.7. Exception Handling
|
67
|
+
# By default the engine logs any exceptions thrown by the single row function and returns
|
68
|
+
# a null value. To have exceptions be re-thrown instead, which makes exceptions visible
|
69
|
+
# to any registered exception handler, please configure as discussed herein.
|
70
|
+
def rethrow_exceptions
|
71
|
+
false
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
class AggregationSingle
|
76
|
+
def self.init
|
77
|
+
true
|
78
|
+
end
|
79
|
+
|
80
|
+
def definition
|
81
|
+
[self.function_name, self.factory_class_name]
|
82
|
+
end
|
83
|
+
|
84
|
+
# UDF aggregation function name in queries
|
85
|
+
def function_name
|
86
|
+
raise NotImplementedError
|
87
|
+
end
|
88
|
+
|
89
|
+
def factory_class_name
|
90
|
+
raise NotImplementedError
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def self.listup
|
95
|
+
return unless defined? Gem
|
96
|
+
|
97
|
+
#TODO: CHECK: same name files of different versions?
|
98
|
+
plugins = Gem.find_files('norikra/udf/*.rb')
|
99
|
+
plugins.each do |plugin|
|
100
|
+
begin
|
101
|
+
debug "plugin file found!", :file => plugin
|
102
|
+
load plugin
|
103
|
+
rescue => e
|
104
|
+
warn "Failed to load norikra UDF plugin", :plugin => plugin.to_s, :error_class => e.class, :error => e.message
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
known_consts = [:SingleRow, :AggregateSingle]
|
109
|
+
udf_bases = [Norikra::UDF::SingleRow, Norikra::UDF::AggregationSingle]
|
110
|
+
udfs = []
|
111
|
+
self.constants.each do |c|
|
112
|
+
next if known_consts.include?(c)
|
113
|
+
|
114
|
+
klass = Norikra::UDF.const_get(c)
|
115
|
+
if klass.is_a?(Class) && udf_bases.include?(klass.superclass)
|
116
|
+
udfs.push(klass)
|
117
|
+
elsif klass.is_a?(Module) && klass.respond_to?(:plugins)
|
118
|
+
udfs.push(klass)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
udfs
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
@@ -0,0 +1,113 @@
|
|
1
|
+
module Norikra
|
2
|
+
module UDFSpecHelper
|
3
|
+
@@plugins = {}
|
4
|
+
|
5
|
+
def function(name)
|
6
|
+
@@plugins[name.to_s]
|
7
|
+
end
|
8
|
+
|
9
|
+
def fcall(name, *args)
|
10
|
+
@@plugins[name.to_s]._call(*args)
|
11
|
+
end
|
12
|
+
|
13
|
+
# params: for AggregationSingle only
|
14
|
+
# required keys: :valueType, :windowed, :distinct, :parameters
|
15
|
+
# :parameters => [[parameterType, constant?, contantValue], ... ]
|
16
|
+
def udf_function(mojule, params={})
|
17
|
+
require 'esper-4.9.0.jar'
|
18
|
+
|
19
|
+
unless mojule.is_a?(Class)
|
20
|
+
mojule.init if mojule.respond_to?(:init)
|
21
|
+
ps = []
|
22
|
+
mojule.plugins.each{|p| ps.push(udf_function(p))} if mojule.respond_to?(:plugins)
|
23
|
+
return ps
|
24
|
+
end
|
25
|
+
|
26
|
+
klass = mojule
|
27
|
+
klass.init if klass.respond_to?(:init)
|
28
|
+
|
29
|
+
if klass.superclass == Norikra::UDF::SingleRow
|
30
|
+
name, classname, methodname = klass.new.definition
|
31
|
+
@@plugins[name] = UDFInstance.new(classname, methodname)
|
32
|
+
|
33
|
+
elsif klass.superclass == Norikra::UDF::AggregationSingle
|
34
|
+
name, factoryclassname = klass.new.definition
|
35
|
+
factory = UDFAggregationFactoryInstance.new(factoryclassname)
|
36
|
+
factory.check(name, params[:valueType], params[:parameters], params.fetch(:distinct, false), params.fetch(:windowed, true))
|
37
|
+
@@plugins[name] = factory.create
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
module UDFHelper
|
42
|
+
def classObject(classname)
|
43
|
+
parts = classname.split('.')
|
44
|
+
clazzname = parts.pop
|
45
|
+
eval("Java::" + parts.map(&:capitalize).join + "::" + clazzname)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
class UDFInstance
|
50
|
+
include UDFHelper
|
51
|
+
|
52
|
+
def initialize(classname, methodname)
|
53
|
+
@methodname = methodname
|
54
|
+
@clazz = classObject(classname)
|
55
|
+
end
|
56
|
+
|
57
|
+
def _call(*args)
|
58
|
+
@clazz.send(@methodname.to_sym, *args)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class UDFAggregationInstance
|
63
|
+
def initialize(instance)
|
64
|
+
@func = instance
|
65
|
+
end
|
66
|
+
|
67
|
+
def _call(type, *args)
|
68
|
+
self.send(type, *args)
|
69
|
+
end
|
70
|
+
|
71
|
+
def getValueType; @func.getValueType; end # public Class getValueType()
|
72
|
+
def enter(*args); @func.enter(*args); end # public void enter(Object value)
|
73
|
+
def leave(*args); @func.leave(*args); end # public void leave(Object value)
|
74
|
+
def getValue; @func.getValue; end # public Object getValue()
|
75
|
+
def clear; @func.clear; end # public void clear()
|
76
|
+
end
|
77
|
+
|
78
|
+
class UDFAggregationFactoryInstance
|
79
|
+
include UDFHelper
|
80
|
+
|
81
|
+
def initialize(classname)
|
82
|
+
@factory = classObject(classname).new
|
83
|
+
end
|
84
|
+
|
85
|
+
# parameters => [[parameterType, constant?, contantValue], ... ]
|
86
|
+
def check(func_name, value_type, parameters, distinct, windowed)
|
87
|
+
@factory.setFunctionName(func_name)
|
88
|
+
unless @factory.getValueType == value_type.java_class
|
89
|
+
raise "Aggregation UDF value type mismatch, expected '#{value_type}', actually '#{@factory.getValueType}'"
|
90
|
+
end
|
91
|
+
|
92
|
+
parameterTypes = parameters.map{|t,bc,c| t.java_class }
|
93
|
+
constantValue = parameters.map{|t,bc,c| bc.nil? ? false : bc }
|
94
|
+
constantValues = parameters.map{|t,bc,c| c }
|
95
|
+
|
96
|
+
# public AggregationValidationContext(java.lang.Class[] parameterTypes,
|
97
|
+
# boolean[] constantValue,
|
98
|
+
# java.lang.Object[] constantValues,
|
99
|
+
# boolean distinct,
|
100
|
+
# boolean windowed,
|
101
|
+
# ExprNode[] expressions)
|
102
|
+
context_class = com.espertech.esper.epl.agg.service.AggregationValidationContext
|
103
|
+
context = context_class.new(parameterTypes, constantValue, constantValues, distinct, windowed, [])
|
104
|
+
@factory.validate(context)
|
105
|
+
true
|
106
|
+
end
|
107
|
+
|
108
|
+
def create
|
109
|
+
UDFAggregationInstance.new(@factory.newAggregator)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
data/lib/norikra/version.rb
CHANGED
metadata
CHANGED
@@ -2,14 +2,14 @@
|
|
2
2
|
name: norikra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.8
|
6
6
|
platform: java
|
7
7
|
authors:
|
8
8
|
- TAGOMORI Satoshi
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-08-
|
12
|
+
date: 2013-08-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: mizuno
|
@@ -272,9 +272,12 @@ files:
|
|
272
272
|
- lib/norikra/rpc/error.rb
|
273
273
|
- lib/norikra/rpc/handler.rb
|
274
274
|
- lib/norikra/rpc/http.rb
|
275
|
+
- lib/norikra/rubyudf.rb
|
275
276
|
- lib/norikra/server.rb
|
276
277
|
- lib/norikra/typedef.rb
|
277
278
|
- lib/norikra/typedef_manager.rb
|
279
|
+
- lib/norikra/udf.rb
|
280
|
+
- lib/norikra/udf_spec_helper.rb
|
278
281
|
- lib/norikra/version.rb
|
279
282
|
- norikra.gemspec
|
280
283
|
- script/spec_server_pry
|