fluent-plugin-norikra 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-norikra.gemspec +1 -1
- data/lib/fluent/plugin/norikra_target.rb +27 -7
- data/lib/fluent/plugin/out_norikra.rb +19 -16
- data/test/plugin/test_out_norikra.rb +11 -0
- data/test/test_query_generator.rb +1 -1
- data/test/test_target.rb +64 -34
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e15eebb32dfadbfe46e9c5d52c7c3eae438cb772
|
4
|
+
data.tar.gz: 25e60695315a63fd5e3e774a63e03500bb4b5bef
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab124b24f446d522b688ace53ea7f1a65ffc095c82e2fef64f44cf78327214908ed9e9fc4e0deecee30aade019973f2482be83bae33eba7b356c6741bb076dff
|
7
|
+
data.tar.gz: f03549e79600346c01af7806274c7d004bbcd374fdfeea589b0f955f2da1efccc6e348c46b0169aa1997291708a23bc000e4c538d6c3a75c314e6d62cb93f603
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |spec|
|
4
4
|
spec.name = "fluent-plugin-norikra"
|
5
|
-
spec.version = "0.0.
|
5
|
+
spec.version = "0.0.4"
|
6
6
|
spec.authors = ["TAGOMORI Satoshi"]
|
7
7
|
spec.email = ["tagomoris@gmail.com"]
|
8
8
|
spec.description = %q{process events on fluentd with SQL like query, with built-in Norikra server if needed.}
|
@@ -31,17 +31,17 @@ class Fluent::NorikraOutput
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
|
-
def generate(
|
34
|
+
def generate(name, escaped)
|
35
35
|
Fluent::NorikraOutput::Query.new(
|
36
|
-
self.class.replace_target(
|
37
|
-
self.class.replace_target(
|
38
|
-
self.class.replace_target(
|
36
|
+
self.class.replace_target(name, @name_template),
|
37
|
+
self.class.replace_target(escaped, @expression_template),
|
38
|
+
self.class.replace_target(name, @tag_template),
|
39
39
|
@fetch_interval
|
40
40
|
)
|
41
41
|
end
|
42
42
|
|
43
|
-
def self.replace_target(
|
44
|
-
str.gsub('${target}',
|
43
|
+
def self.replace_target(t, str)
|
44
|
+
str.gsub('${target}', t)
|
45
45
|
end
|
46
46
|
|
47
47
|
def self.parse_time_period(string)
|
@@ -192,12 +192,32 @@ class Fluent::NorikraOutput
|
|
192
192
|
|
193
193
|
class Target
|
194
194
|
attr_accessor :name, :fields, :queries
|
195
|
+
attr_reader :escaped_name
|
196
|
+
|
197
|
+
def self.escape(src)
|
198
|
+
if src.nil? || src.empty?
|
199
|
+
return 'FluentdGenerated'
|
200
|
+
end
|
201
|
+
|
202
|
+
dst = src.gsub(/[^_a-zA-Z0-9]/, '_')
|
203
|
+
unless dst =~ /^[a-zA-Z]([_a-zA-Z0-9]*[a-zA-Z0-9])?$/
|
204
|
+
unless dst =~ /^[a-zA-Z]/
|
205
|
+
dst = 'Fluentd' + dst
|
206
|
+
end
|
207
|
+
unless dst =~ /[a-zA-Z0-9]$/
|
208
|
+
dst = dst + 'Generated'
|
209
|
+
end
|
210
|
+
end
|
211
|
+
dst
|
212
|
+
end
|
195
213
|
|
196
214
|
def initialize(target, config)
|
197
215
|
@name = target
|
216
|
+
@escaped_name = self.class.escape(@name)
|
217
|
+
|
198
218
|
@filter = RecordFilter.new(*([:include, :include_regexp, :exclude, :exclude_regexp].map{|s| config.filter_params[s]}))
|
199
219
|
@fields = config.field_definitions
|
200
|
-
@queries = config.query_generators.map{|g| g.generate(
|
220
|
+
@queries = config.query_generators.map{|g| g.generate(@name, @escaped_name)}
|
201
221
|
end
|
202
222
|
|
203
223
|
def filter(record)
|
@@ -157,14 +157,17 @@ module Fluent
|
|
157
157
|
|
158
158
|
@register_thread.join
|
159
159
|
@fetch_thread.join
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
160
|
+
|
161
|
+
if @execute_server
|
162
|
+
begin
|
163
|
+
counter = 0
|
164
|
+
while !Process.waitpid(@norikra_pid, Process::WNOHANG)
|
165
|
+
sleep 1
|
166
|
+
break if counter > 3
|
167
|
+
end
|
168
|
+
rescue Errno::ECHILD
|
169
|
+
# norikra server process exited.
|
165
170
|
end
|
166
|
-
rescue Errno::ECHILD
|
167
|
-
# norikra server process exited.
|
168
171
|
end
|
169
172
|
end
|
170
173
|
|
@@ -268,14 +271,14 @@ module Fluent
|
|
268
271
|
|
269
272
|
event = t.filter(record)
|
270
273
|
|
271
|
-
out << [
|
274
|
+
out << [t.escaped_name,event].to_msgpack
|
272
275
|
end
|
273
276
|
|
274
277
|
out
|
275
278
|
end
|
276
279
|
|
277
280
|
def prepared?(target_names)
|
278
|
-
@norikra_started && target_names.reduce(true){|r,t| r && @target_map
|
281
|
+
@norikra_started && target_names.reduce(true){|r,t| r && @target_map.values.any?{|target| target.escaped_name == t}}
|
279
282
|
end
|
280
283
|
|
281
284
|
def write(chunk)
|
@@ -300,15 +303,15 @@ module Fluent
|
|
300
303
|
# target open and reserve fields
|
301
304
|
$log.debug "Going to prepare about target"
|
302
305
|
begin
|
303
|
-
unless client.targets.include?(target.
|
304
|
-
$log.debug "opening target #{target.
|
305
|
-
client.open(target.
|
306
|
-
$log.debug "opening target #{target.
|
306
|
+
unless client.targets.include?(target.escaped_name)
|
307
|
+
$log.debug "opening target #{target.escaped_name}"
|
308
|
+
client.open(target.escaped_name, target.reserve_fields)
|
309
|
+
$log.debug "opening target #{target.escaped_name}, done."
|
307
310
|
end
|
308
311
|
|
309
312
|
reserving = target.reserve_fields
|
310
313
|
reserved = []
|
311
|
-
client.fields(target.
|
314
|
+
client.fields(target.escaped_name).each do |field|
|
312
315
|
if reserving[field['name']]
|
313
316
|
reserved.push(field['name'])
|
314
317
|
if reserving[field['name']] != field['type']
|
@@ -318,10 +321,10 @@ module Fluent
|
|
318
321
|
end
|
319
322
|
|
320
323
|
reserving.each do |fieldname,type|
|
321
|
-
client.reserve(target, fieldname, type) unless reserved.include?(fieldname)
|
324
|
+
client.reserve(target.escaped_name, fieldname, type) unless reserved.include?(fieldname)
|
322
325
|
end
|
323
326
|
rescue => e
|
324
|
-
$log.error "failed to prepare target:#{target.
|
327
|
+
$log.error "failed to prepare target:#{target.escaped_name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
|
325
328
|
return false
|
326
329
|
end
|
327
330
|
|
@@ -36,7 +36,7 @@ class QueryGeneratorTest < Test::Unit::TestCase
|
|
36
36
|
|
37
37
|
def test_generate
|
38
38
|
g = @this.new('query_${target}', 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}')
|
39
|
-
q = g.generate('test')
|
39
|
+
q = g.generate('test', 'test')
|
40
40
|
assert_equal 'query_test', q.name
|
41
41
|
assert_equal 'SELECT * FROM test.win:time_batch( 10 min ) WHERE x=1', q.expression
|
42
42
|
assert_equal 'tag.test', q.tag
|
data/test/test_target.rb
CHANGED
@@ -6,42 +6,54 @@ class TargetTest < Test::Unit::TestCase
|
|
6
6
|
@this = Fluent::NorikraOutput::Target
|
7
7
|
end
|
8
8
|
|
9
|
-
def
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
c1 = Fluent::Config::Element.new('default', nil, {
|
21
|
-
'include' => '*',
|
22
|
-
'exclude' => 'flag',
|
23
|
-
'exclude_regexp' => 'f_.*',
|
24
|
-
'field_string' => 's1,s2,s3',
|
25
|
-
'field_boolean' => 'bool1,bool2',
|
26
|
-
'field_int' => 'i1,i2,i3,i4',
|
27
|
-
'field_long' => 'num1,num2',
|
28
|
-
'field_float' => 'f1,f2',
|
29
|
-
'field_double' => 'd'
|
30
|
-
}, [q1,q2])
|
31
|
-
s1 = Fluent::NorikraOutput::ConfigSection.new(c1)
|
9
|
+
def test_target_name_escape
|
10
|
+
assert_equal 'target1', @this.escape('target1')
|
11
|
+
assert_equal 'target1_subtarget1', @this.escape('target1.subtarget1')
|
12
|
+
assert_equal 'test_tag_foo', @this.escape('test.tag.foo')
|
13
|
+
|
14
|
+
assert_equal 'FluentdGenerated', @this.escape('')
|
15
|
+
assert_equal 'Fluentd_Generated', @this.escape(':')
|
16
|
+
assert_equal 'a', @this.escape('a')
|
17
|
+
assert_equal 'Fluentd_a', @this.escape('_a')
|
18
|
+
assert_equal 'a_Generated', @this.escape('a_')
|
19
|
+
end
|
32
20
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
}
|
42
|
-
|
21
|
+
Q1 = Fluent::Config::Element.new('query', nil, {
|
22
|
+
'name' => 'q1_${target}',
|
23
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
|
24
|
+
'tag' => 'q1.${target}'
|
25
|
+
}, [])
|
26
|
+
Q2 = Fluent::Config::Element.new('query', nil, {
|
27
|
+
'name' => 'q2_${target}',
|
28
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
|
29
|
+
'tag' => 'q2.${target}'
|
30
|
+
}, [])
|
31
|
+
C1 = Fluent::Config::Element.new('default', nil, {
|
32
|
+
'include' => '*',
|
33
|
+
'exclude' => 'flag',
|
34
|
+
'exclude_regexp' => 'f_.*',
|
35
|
+
'field_string' => 's1,s2,s3',
|
36
|
+
'field_boolean' => 'bool1,bool2',
|
37
|
+
'field_int' => 'i1,i2,i3,i4',
|
38
|
+
'field_long' => 'num1,num2',
|
39
|
+
'field_float' => 'f1,f2',
|
40
|
+
'field_double' => 'd'
|
41
|
+
}, [Q1,Q2])
|
42
|
+
S1 = Fluent::NorikraOutput::ConfigSection.new(C1)
|
43
43
|
|
44
|
-
|
44
|
+
Q3 = Fluent::Config::Element.new('query', nil, {
|
45
|
+
'name' => 'q3_test',
|
46
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"',
|
47
|
+
'tag' => 'q3.test'
|
48
|
+
}, [])
|
49
|
+
C2 = Fluent::Config::Element.new('target', 'test', {
|
50
|
+
'exclude_regexp' => '(f|g)_.*',
|
51
|
+
'field_double' => 'd1,d2,d3,d4'
|
52
|
+
}, [Q3])
|
53
|
+
S2 = Fluent::NorikraOutput::ConfigSection.new(C2)
|
54
|
+
|
55
|
+
def test_instanciate
|
56
|
+
t = @this.new('test', S1 + S2)
|
45
57
|
|
46
58
|
assert_equal 'test', t.name
|
47
59
|
assert_equal({
|
@@ -63,4 +75,22 @@ class TargetTest < Test::Unit::TestCase
|
|
63
75
|
'd' => 'double', 'd1' => 'double', 'd2' => 'double', 'd3' => 'double', 'd4' => 'double'
|
64
76
|
}, t.reserve_fields)
|
65
77
|
end
|
78
|
+
|
79
|
+
def test_queries
|
80
|
+
t = @this.new('test.service', S1 + S2)
|
81
|
+
|
82
|
+
assert_equal 3, t.queries.size
|
83
|
+
|
84
|
+
assert_equal 'q1_test.service', t.queries[0].name
|
85
|
+
assert_equal 'SELECT * FROM test_service.win:time_batch(10 min) WHERE q1', t.queries[0].expression
|
86
|
+
assert_equal 'q1.test.service', t.queries[0].tag
|
87
|
+
|
88
|
+
assert_equal 'q2_test.service', t.queries[1].name
|
89
|
+
assert_equal 'SELECT * FROM test_service.win:time_batch(50 min) WHERE q2.length() > 0', t.queries[1].expression
|
90
|
+
assert_equal 'q2.test.service', t.queries[1].tag
|
91
|
+
|
92
|
+
assert_equal 'q3_test', t.queries[2].name
|
93
|
+
assert_equal 'SELECT * FROM test_service.win:time_batch(30 min) WHERE q3="/"', t.queries[2].expression
|
94
|
+
assert_equal 'q3.test', t.queries[2].tag
|
95
|
+
end
|
66
96
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-norikra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-
|
11
|
+
date: 2013-08-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: norikra-client
|
@@ -86,6 +86,7 @@ files:
|
|
86
86
|
- lib/fluent/plugin/norikra_target.rb
|
87
87
|
- lib/fluent/plugin/out_norikra.rb
|
88
88
|
- test/helper.rb
|
89
|
+
- test/plugin/test_out_norikra.rb
|
89
90
|
- test/test_config_section.rb
|
90
91
|
- test/test_query.rb
|
91
92
|
- test/test_query_generator.rb
|
@@ -111,14 +112,16 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
111
112
|
version: '0'
|
112
113
|
requirements: []
|
113
114
|
rubyforge_project:
|
114
|
-
rubygems_version: 2.0.
|
115
|
+
rubygems_version: 2.0.3
|
115
116
|
signing_key:
|
116
117
|
specification_version: 4
|
117
118
|
summary: Fluentd plugin to do CEP with norikra
|
118
119
|
test_files:
|
119
120
|
- test/helper.rb
|
121
|
+
- test/plugin/test_out_norikra.rb
|
120
122
|
- test/test_config_section.rb
|
121
123
|
- test/test_query.rb
|
122
124
|
- test/test_query_generator.rb
|
123
125
|
- test/test_record_filter.rb
|
124
126
|
- test/test_target.rb
|
127
|
+
has_rdoc:
|