fluent-plugin-norikra 0.0.9 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +174 -125
- data/example/blank.conf +2 -3
- data/example/blank.rb +15 -0
- data/example/blank2.conf +1 -1
- data/example/example1.conf +6 -7
- data/example/test1.conf +4 -5
- data/example/test1.rb +40 -0
- data/example/test2.conf +5 -6
- data/example/test_in_out.rb +48 -0
- data/fluent-plugin-norikra.gemspec +1 -1
- data/lib/fluent/plugin/in_norikra.rb +75 -0
- data/lib/fluent/plugin/norikra/config_section.rb +67 -0
- data/lib/fluent/plugin/norikra/fetch_request.rb +72 -0
- data/lib/fluent/plugin/norikra/input.rb +109 -0
- data/lib/fluent/plugin/norikra/output.rb +213 -0
- data/lib/fluent/plugin/norikra/query.rb +13 -0
- data/lib/fluent/plugin/norikra/query_generator.rb +61 -0
- data/lib/fluent/plugin/norikra/record_filter.rb +62 -0
- data/lib/fluent/plugin/norikra/target.rb +47 -0
- data/lib/fluent/plugin/norikra_target.rb +0 -246
- data/lib/fluent/plugin/out_norikra.rb +15 -364
- data/lib/fluent/plugin/out_norikra_filter.rb +172 -0
- data/test/helper.rb +2 -0
- data/test/plugin/test_in_norikra.rb +14 -0
- data/test/plugin/test_out_norikra.rb +5 -1
- data/test/plugin/test_out_norikra_filter.rb +15 -0
- data/test/test_config_section.rb +80 -25
- data/test/test_query.rb +6 -2
- data/test/test_query_generator.rb +9 -4
- data/test/test_record_filter.rb +2 -2
- data/test/test_target.rb +11 -13
- metadata +19 -2
@@ -0,0 +1,172 @@
|
|
1
|
+
require_relative 'norikra/input'
|
2
|
+
require_relative 'norikra/output'
|
3
|
+
|
4
|
+
require 'norikra-client'
|
5
|
+
|
6
|
+
module Fluent
|
7
|
+
class NorikraFilterOutput < Fluent::BufferedOutput
|
8
|
+
include Fluent::NorikraPlugin::InputMixin
|
9
|
+
include Fluent::NorikraPlugin::OutputMixin
|
10
|
+
|
11
|
+
Fluent::Plugin.register_output('norikra_filter', self)
|
12
|
+
|
13
|
+
config_set_default :flush_interval, 1 # 1sec
|
14
|
+
|
15
|
+
config_param :norikra, :string, :default => 'localhost:26571'
|
16
|
+
|
17
|
+
config_param :connect_timeout, :integer, :default => nil
|
18
|
+
config_param :send_timeout, :integer, :default => nil
|
19
|
+
config_param :receive_timeout, :integer, :default => nil
|
20
|
+
|
21
|
+
#<server>
|
22
|
+
attr_reader :execute_server, :execute_server_path
|
23
|
+
|
24
|
+
#for OutputMixin
|
25
|
+
config_param :remove_tag_prefix, :string, :default => nil
|
26
|
+
config_param :target_map_tag, :bool, :default => false
|
27
|
+
config_param :target_map_key, :string, :default => nil
|
28
|
+
config_param :target_string, :string, :default => nil
|
29
|
+
|
30
|
+
# <default>
|
31
|
+
# <target TARGET>
|
32
|
+
|
33
|
+
# <fetch> tags
|
34
|
+
# <fetch>
|
35
|
+
# method event
|
36
|
+
# target QUERY_NAME
|
37
|
+
# interval 5s
|
38
|
+
# tag query_name
|
39
|
+
# # tag field FIELDNAME
|
40
|
+
# # tag string FIXED_STRING
|
41
|
+
# tag_prefix norikra.event # actual tag: norikra.event.QUERYNAME
|
42
|
+
# </fetch>
|
43
|
+
# <fetch>
|
44
|
+
# method sweep
|
45
|
+
# target QUERY_GROUP # or unspecified => default
|
46
|
+
# interval 60s
|
47
|
+
# tag field group_by_key
|
48
|
+
# tag_prefix norikra.query
|
49
|
+
# </fetch>
|
50
|
+
|
51
|
+
def configure(conf)
|
52
|
+
super
|
53
|
+
|
54
|
+
@host,@port = @norikra.split(':', 2)
|
55
|
+
@port = @port.to_i
|
56
|
+
|
57
|
+
if !@target_map_tag && @target_map_key.nil? && @target_string.nil?
|
58
|
+
raise Fluent::ConfigError, 'target naming not specified (target_map_tag/target_map_key/target_string)'
|
59
|
+
end
|
60
|
+
|
61
|
+
@execute_server = false
|
62
|
+
|
63
|
+
conf.elements.each do |element|
|
64
|
+
case element.name
|
65
|
+
when 'default', 'target'
|
66
|
+
# ignore: processed in OutputMixin
|
67
|
+
when 'fetch'
|
68
|
+
# ignore: processed in InputMixin, and set @fetch_queue
|
69
|
+
when 'server'
|
70
|
+
@execute_server = true
|
71
|
+
@execute_jruby_path = element['jruby']
|
72
|
+
@execute_server_path = element['path']
|
73
|
+
@execute_server_opts = element['opts']
|
74
|
+
else
|
75
|
+
raise Fluent::ConfigError, "unknown configuration section name for this plugin: #{element.name}"
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
setup_output(conf, true) # <query> enabled in <default> and <target TARGET>
|
80
|
+
setup_input(conf)
|
81
|
+
end
|
82
|
+
|
83
|
+
def client(opts={})
|
84
|
+
Norikra::Client.new(@host, @port, {
|
85
|
+
:connect_timeout => opts[:connect_timeout] || @connect_timeout,
|
86
|
+
:send_timeout => opts[:send_timeout] || @send_timeout,
|
87
|
+
:receive_timeout => opts[:receive_timeout] || @receive_timeout,
|
88
|
+
})
|
89
|
+
end
|
90
|
+
|
91
|
+
def start
|
92
|
+
super
|
93
|
+
|
94
|
+
@norikra_started = false
|
95
|
+
|
96
|
+
if @execute_server
|
97
|
+
@norikra_pid = nil
|
98
|
+
@norikra_thread = Thread.new(&method(:server_starter))
|
99
|
+
# @norikra_started will be set in server_starter
|
100
|
+
else
|
101
|
+
@norikra_pid = nil
|
102
|
+
@norikra_thread = nil
|
103
|
+
@norikra_started = true
|
104
|
+
end
|
105
|
+
|
106
|
+
start_output
|
107
|
+
start_input
|
108
|
+
end
|
109
|
+
|
110
|
+
def shutdown
|
111
|
+
stop_output
|
112
|
+
stop_input
|
113
|
+
Process.kill(:TERM, @norikra_pid) if @execute_server
|
114
|
+
|
115
|
+
shutdown_output
|
116
|
+
shutdown_input
|
117
|
+
|
118
|
+
if @execute_server
|
119
|
+
begin
|
120
|
+
counter = 0
|
121
|
+
while !Process.waitpid(@norikra_pid, Process::WNOHANG)
|
122
|
+
sleep 1
|
123
|
+
break if counter > 3
|
124
|
+
end
|
125
|
+
rescue Errno::ECHILD
|
126
|
+
# norikra server process exited.
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def server_starter
|
132
|
+
$log.info "starting Norikra server process #{@host}:#{@port}"
|
133
|
+
base_options = [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]
|
134
|
+
cmd,options = if @execute_jruby_path
|
135
|
+
[@execute_jruby_path, [@execute_server_path, 'start', '-H', @host, '-P', @port.to_s]]
|
136
|
+
else
|
137
|
+
[@execute_server_path, ['start', '-H', @host, '-P', @port.to_s]]
|
138
|
+
end
|
139
|
+
if @execute_server_opts
|
140
|
+
options += @execute_server_opts.split(/ +/)
|
141
|
+
end
|
142
|
+
@norikra_pid = fork do
|
143
|
+
ENV.keys.select{|k| k =~ /^(RUBY|GEM|BUNDLE|RBENV|RVM|rvm)/}.each {|k| ENV.delete(k)}
|
144
|
+
exec([cmd, 'norikra(fluentd)'], *options)
|
145
|
+
end
|
146
|
+
connecting = true
|
147
|
+
$log.info "trying to confirm norikra server status..."
|
148
|
+
while connecting
|
149
|
+
begin
|
150
|
+
$log.debug "start to connect norikra server #{@host}:#{@port}"
|
151
|
+
client(:connect_timeout => 1, :send_timeout => 1, :receive_timeout => 1).targets
|
152
|
+
# discard result: no exceptions is success
|
153
|
+
connecting = false
|
154
|
+
next
|
155
|
+
rescue HTTPClient::TimeoutError
|
156
|
+
$log.debug "Norikra server test connection timeout. retrying..."
|
157
|
+
rescue Errno::ECONNREFUSED
|
158
|
+
$log.debug "Norikra server test connection refused. retrying..."
|
159
|
+
rescue => e
|
160
|
+
$log.error "unknown error in confirming norikra server, #{e.class}:#{e.message}"
|
161
|
+
end
|
162
|
+
sleep 3
|
163
|
+
end
|
164
|
+
$log.info "confirmed that norikra server #{@host}:#{@port} started."
|
165
|
+
@norikra_started = true
|
166
|
+
end
|
167
|
+
|
168
|
+
def fetchable?
|
169
|
+
@norikra_started
|
170
|
+
end
|
171
|
+
end
|
172
|
+
end
|
data/test/helper.rb
CHANGED
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class NorikraInputTest < Test::Unit::TestCase
|
4
|
+
CONFIG = %[
|
5
|
+
]
|
6
|
+
|
7
|
+
def create_driver(conf=CONFIG, tag='test')
|
8
|
+
Fluent::Test::InputTestDriver.new(Fluent::NorikraInput).configure(conf)
|
9
|
+
end
|
10
|
+
|
11
|
+
def test_init
|
12
|
+
create_driver
|
13
|
+
end
|
14
|
+
end
|
@@ -1,11 +1,15 @@
|
|
1
1
|
require 'helper'
|
2
2
|
|
3
3
|
class NorikraOutputTest < Test::Unit::TestCase
|
4
|
-
|
4
|
+
CONFIG = %[
|
5
5
|
target_map_tag yes
|
6
6
|
]
|
7
7
|
|
8
8
|
def create_driver(conf=CONFIG, tag='test')
|
9
9
|
Fluent::Test::OutputTestDriver.new(Fluent::NorikraOutput, tag).configure(conf)
|
10
10
|
end
|
11
|
+
|
12
|
+
def test_init
|
13
|
+
create_driver
|
14
|
+
end
|
11
15
|
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class NorikraFilterOutputTest < Test::Unit::TestCase
|
4
|
+
CONFIG = %[
|
5
|
+
target_map_tag yes
|
6
|
+
]
|
7
|
+
|
8
|
+
def create_driver(conf=CONFIG, tag='test')
|
9
|
+
Fluent::Test::OutputTestDriver.new(Fluent::NorikraFilterOutput, tag).configure(conf)
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_init
|
13
|
+
create_driver
|
14
|
+
end
|
15
|
+
end
|
data/test/test_config_section.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'helper'
|
2
|
-
require 'fluent/plugin/
|
2
|
+
require 'fluent/plugin/norikra/config_section'
|
3
3
|
|
4
4
|
class ConfigSectionTest < Test::Unit::TestCase
|
5
5
|
def setup
|
6
|
-
@this = Fluent::
|
6
|
+
@this = Fluent::NorikraPlugin::ConfigSection
|
7
7
|
end
|
8
8
|
|
9
9
|
def test_init_default
|
@@ -23,23 +23,45 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
23
23
|
'exclude_regexp' => 'f_.*',
|
24
24
|
'field_string' => 's1,s2,s3',
|
25
25
|
'field_boolean' => 'bool1,bool2',
|
26
|
-
'
|
27
|
-
'
|
28
|
-
'field_float' => 'f1,f2',
|
29
|
-
'field_double' => 'd'
|
26
|
+
'field_integer' => 'i1,i2,i3,i4,num1,num2',
|
27
|
+
'field_float' => 'f1,f2,d',
|
30
28
|
}, [q1,q2])
|
31
29
|
s1 = @this.new(c1)
|
32
30
|
|
33
31
|
assert_nil s1.target
|
34
32
|
assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => 'f_.*'}, s1.filter_params)
|
35
33
|
assert_equal({
|
36
|
-
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :
|
37
|
-
:float => %w(f1 f2
|
34
|
+
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2),
|
35
|
+
:float => %w(f1 f2 d),
|
38
36
|
}, s1.field_definitions)
|
39
37
|
assert_equal 2, s1.query_generators.size
|
40
38
|
assert_equal (10 * 60 / 5), s1.query_generators.map(&:fetch_interval).sort.first
|
41
39
|
end
|
42
40
|
|
41
|
+
def test_init_default_without_query
|
42
|
+
q1 = Fluent::Config::Element.new('query', nil, {
|
43
|
+
'name' => 'q1_${target}',
|
44
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
|
45
|
+
'tag' => 'q1.${target}'
|
46
|
+
}, [])
|
47
|
+
q2 = Fluent::Config::Element.new('query', nil, {
|
48
|
+
'name' => 'q2_${target}',
|
49
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
|
50
|
+
'tag' => 'q2.${target}'
|
51
|
+
}, [])
|
52
|
+
c1 = Fluent::Config::Element.new('default', nil, {
|
53
|
+
'include' => '*',
|
54
|
+
'exclude' => 'flag',
|
55
|
+
'exclude_regexp' => 'f_.*',
|
56
|
+
'field_string' => 's1,s2,s3',
|
57
|
+
'field_boolean' => 'bool1,bool2',
|
58
|
+
'field_integer' => 'i1,i2,i3,i4,num1,num2',
|
59
|
+
'field_float' => 'f1,f2,d',
|
60
|
+
}, [q1,q2])
|
61
|
+
s1 = @this.new(c1, false)
|
62
|
+
assert_equal 0, s1.query_generators.size
|
63
|
+
end
|
64
|
+
|
43
65
|
def test_init_target
|
44
66
|
q3 = Fluent::Config::Element.new('query', nil, {
|
45
67
|
'name' => 'q3_test2',
|
@@ -48,13 +70,13 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
48
70
|
}, [])
|
49
71
|
c2 = Fluent::Config::Element.new('target', 'test2', {
|
50
72
|
'exclude_regexp' => '(f|g)_.*',
|
51
|
-
'
|
73
|
+
'field_float' => 'd1,d2,d3,d4'
|
52
74
|
}, [q3])
|
53
75
|
s2 = @this.new(c2)
|
54
76
|
|
55
77
|
assert_equal 'test2', s2.target
|
56
78
|
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => '(f|g)_.*'}, s2.filter_params)
|
57
|
-
assert_equal({:string => [], :boolean => [], :
|
79
|
+
assert_equal({:string => [], :boolean => [], :integer => [], :float => %w(d1 d2 d3 d4)}, s2.field_definitions)
|
58
80
|
assert_equal 1, s2.query_generators.size
|
59
81
|
assert_equal (30 * 60 / 5), s2.query_generators.map(&:fetch_interval).sort.first
|
60
82
|
end
|
@@ -71,19 +93,19 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
71
93
|
|
72
94
|
assert_equal 'test3', s3.target
|
73
95
|
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s3.filter_params)
|
74
|
-
assert_equal({:string => [], :boolean => [], :
|
96
|
+
assert_equal({:string => [], :boolean => [], :integer => [], :float => []}, s3.field_definitions)
|
75
97
|
assert_equal 1, s3.query_generators.size
|
76
98
|
end
|
77
99
|
|
78
100
|
def test_init_target_without_query
|
79
101
|
c4 = Fluent::Config::Element.new('target', 'test4', {
|
80
|
-
'
|
102
|
+
'field_integer' => 'status'
|
81
103
|
}, [])
|
82
104
|
s4 = @this.new(c4)
|
83
105
|
|
84
106
|
assert_equal 'test4', s4.target
|
85
107
|
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s4.filter_params)
|
86
|
-
assert_equal({:string => [], :boolean => [], :
|
108
|
+
assert_equal({:string => [], :boolean => [], :integer => ['status'], :float => []}, s4.field_definitions)
|
87
109
|
assert_equal 0, s4.query_generators.size
|
88
110
|
end
|
89
111
|
|
@@ -93,7 +115,7 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
93
115
|
|
94
116
|
assert_equal 'test5', s5.target
|
95
117
|
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s5.filter_params)
|
96
|
-
assert_equal({:string => [], :boolean => [], :
|
118
|
+
assert_equal({:string => [], :boolean => [], :integer => [], :float => []}, s5.field_definitions)
|
97
119
|
assert_equal 0, s5.query_generators.size
|
98
120
|
end
|
99
121
|
|
@@ -114,10 +136,8 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
114
136
|
'exclude_regexp' => 'f_.*',
|
115
137
|
'field_string' => 's1,s2,s3',
|
116
138
|
'field_boolean' => 'bool1,bool2',
|
117
|
-
'
|
118
|
-
'
|
119
|
-
'field_float' => 'f1,f2',
|
120
|
-
'field_double' => 'd'
|
139
|
+
'field_integer' => 'i1,i2,i3,i4,num1,num2',
|
140
|
+
'field_float' => 'f1,f2,d',
|
121
141
|
}, [q1,q2])
|
122
142
|
s1 = @this.new(c1)
|
123
143
|
|
@@ -128,7 +148,7 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
128
148
|
}, [])
|
129
149
|
c2 = Fluent::Config::Element.new('target', 'test', {
|
130
150
|
'exclude_regexp' => '(f|g)_.*',
|
131
|
-
'
|
151
|
+
'field_float' => 'd1,d2,d3,d4'
|
132
152
|
}, [q3])
|
133
153
|
s2 = @this.new(c2)
|
134
154
|
|
@@ -137,8 +157,8 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
137
157
|
assert_equal 'test', s.target
|
138
158
|
assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => '(f|g)_.*'}, s.filter_params)
|
139
159
|
assert_equal({
|
140
|
-
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :
|
141
|
-
:float => %w(f1 f2
|
160
|
+
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2),
|
161
|
+
:float => %w(f1 f2 d d1 d2 d3 d4)
|
142
162
|
}, s.field_definitions)
|
143
163
|
assert_equal 3, s.query_generators.size
|
144
164
|
assert_equal (10 * 60 / 5), s.query_generators.map(&:fetch_interval).sort.first
|
@@ -161,10 +181,8 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
161
181
|
'exclude_regexp' => 'f_.*',
|
162
182
|
'field_string' => 's1,s2,s3',
|
163
183
|
'field_boolean' => 'bool1,bool2',
|
164
|
-
'
|
165
|
-
'
|
166
|
-
'field_float' => 'f1,f2',
|
167
|
-
'field_double' => 'd'
|
184
|
+
'field_integer' => 'i1,i2,i3,i4,num1,num2',
|
185
|
+
'field_float' => 'f1,f2,d',
|
168
186
|
}, [q1,q2])
|
169
187
|
s1 = @this.new(c1)
|
170
188
|
|
@@ -176,4 +194,41 @@ class ConfigSectionTest < Test::Unit::TestCase
|
|
176
194
|
assert_equal s1.query_generators.size, s.query_generators.size
|
177
195
|
assert_equal s1.query_generators.map(&:fetch_interval).sort.first, s.query_generators.map(&:fetch_interval).sort.first
|
178
196
|
end
|
197
|
+
|
198
|
+
def test_join_without_query
|
199
|
+
q1 = Fluent::Config::Element.new('query', nil, {
|
200
|
+
'name' => 'q1_${target}',
|
201
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
|
202
|
+
'tag' => 'q1.${target}'
|
203
|
+
}, [])
|
204
|
+
q2 = Fluent::Config::Element.new('query', nil, {
|
205
|
+
'name' => 'q2_${target}',
|
206
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
|
207
|
+
'tag' => 'q2.${target}'
|
208
|
+
}, [])
|
209
|
+
c1 = Fluent::Config::Element.new('default', nil, {
|
210
|
+
'include' => '*',
|
211
|
+
'exclude' => 'flag',
|
212
|
+
'exclude_regexp' => 'f_.*',
|
213
|
+
'field_string' => 's1,s2,s3',
|
214
|
+
'field_boolean' => 'bool1,bool2',
|
215
|
+
'field_integer' => 'i1,i2,i3,i4,num1,num2',
|
216
|
+
'field_float' => 'f1,f2,d',
|
217
|
+
}, [q1,q2])
|
218
|
+
s1 = @this.new(c1, false)
|
219
|
+
|
220
|
+
q3 = Fluent::Config::Element.new('query', nil, {
|
221
|
+
'name' => 'q3_test',
|
222
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"',
|
223
|
+
'tag' => 'q3.test'
|
224
|
+
}, [])
|
225
|
+
c2 = Fluent::Config::Element.new('target', 'test', {
|
226
|
+
'exclude_regexp' => '(f|g)_.*',
|
227
|
+
'field_float' => 'd1,d2,d3,d4'
|
228
|
+
}, [q3])
|
229
|
+
s2 = @this.new(c2, false)
|
230
|
+
|
231
|
+
s = s1 + s2
|
232
|
+
assert_equal 0, s.query_generators.size
|
233
|
+
end
|
179
234
|
end
|
data/test/test_query.rb
CHANGED
@@ -1,12 +1,16 @@
|
|
1
1
|
require 'helper'
|
2
|
-
require 'fluent/plugin/
|
2
|
+
require 'fluent/plugin/norikra/query'
|
3
3
|
|
4
4
|
class QueryTest < Test::Unit::TestCase
|
5
5
|
def test_init
|
6
|
-
q = Fluent::
|
6
|
+
q = Fluent::NorikraPlugin::Query.new('name', nil, 'expression', 'tag', 10)
|
7
7
|
assert_equal 'name', q.name
|
8
|
+
assert_nil q.group
|
8
9
|
assert_equal 'expression', q.expression
|
9
10
|
assert_equal 'tag', q.tag
|
10
11
|
assert_equal 10, q.interval
|
12
|
+
|
13
|
+
q = Fluent::NorikraPlugin::Query.new('name', 'group', 'expression', 'tag', 10)
|
14
|
+
assert_equal 'group', q.group
|
11
15
|
end
|
12
16
|
end
|
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'helper'
|
2
|
-
require 'fluent/plugin/
|
2
|
+
require 'fluent/plugin/norikra/query_generator'
|
3
3
|
|
4
4
|
class QueryGeneratorTest < Test::Unit::TestCase
|
5
5
|
def setup
|
6
|
-
@this = Fluent::
|
6
|
+
@this = Fluent::NorikraPlugin::QueryGenerator
|
7
7
|
end
|
8
8
|
|
9
9
|
def test_replace_target
|
@@ -35,15 +35,20 @@ class QueryGeneratorTest < Test::Unit::TestCase
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def test_generate
|
38
|
-
g = @this.new('query_${target}', 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}')
|
38
|
+
g = @this.new('query_${target}', 'test_group', 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}')
|
39
39
|
q = g.generate('test', 'test')
|
40
40
|
assert_equal 'query_test', q.name
|
41
|
+
assert_equal 'test_group', q.group
|
41
42
|
assert_equal 'SELECT * FROM test.win:time_batch( 10 min ) WHERE x=1', q.expression
|
42
43
|
assert_equal 'tag.test', q.tag
|
44
|
+
|
45
|
+
g = @this.new('query_${target}', nil, 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}')
|
46
|
+
q = g.generate('test', 'test')
|
47
|
+
assert_nil q.group
|
43
48
|
end
|
44
49
|
|
45
50
|
def test_fetch_interval
|
46
|
-
g = @this.new('query_${target}', 'SELECT * FROM ${target}.win:time_batch( 12 min ) WHERE x=1', 'tag.${target}')
|
51
|
+
g = @this.new('query_${target}', nil, 'SELECT * FROM ${target}.win:time_batch( 12 min ) WHERE x=1', 'tag.${target}')
|
47
52
|
assert_equal (12*60/5), g.fetch_interval
|
48
53
|
end
|
49
54
|
end
|
data/test/test_record_filter.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'helper'
|
2
|
-
require 'fluent/plugin/
|
2
|
+
require 'fluent/plugin/norikra/record_filter'
|
3
3
|
|
4
4
|
class RecordFilterTest < Test::Unit::TestCase
|
5
5
|
def setup
|
6
|
-
@this = Fluent::
|
6
|
+
@this = Fluent::NorikraPlugin::RecordFilter
|
7
7
|
end
|
8
8
|
|
9
9
|
def test_errors
|
data/test/test_target.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
require 'helper'
|
2
|
-
require 'fluent/plugin/
|
2
|
+
require 'fluent/plugin/norikra/target'
|
3
3
|
|
4
4
|
class TargetTest < Test::Unit::TestCase
|
5
5
|
def setup
|
6
|
-
@this = Fluent::
|
6
|
+
@this = Fluent::NorikraPlugin::Target
|
7
7
|
end
|
8
8
|
|
9
9
|
def test_target_name_escape
|
@@ -34,12 +34,10 @@ class TargetTest < Test::Unit::TestCase
|
|
34
34
|
'exclude_regexp' => 'f_.*',
|
35
35
|
'field_string' => 's1,s2,s3',
|
36
36
|
'field_boolean' => 'bool1,bool2',
|
37
|
-
'
|
38
|
-
'
|
39
|
-
'field_float' => 'f1,f2',
|
40
|
-
'field_double' => 'd'
|
37
|
+
'field_integer' => 'i1,i2,i3,i4,num1,num2',
|
38
|
+
'field_float' => 'f1,f2,d',
|
41
39
|
}, [Q1,Q2])
|
42
|
-
S1 = Fluent::
|
40
|
+
S1 = Fluent::NorikraPlugin::ConfigSection.new(C1)
|
43
41
|
|
44
42
|
Q3 = Fluent::Config::Element.new('query', nil, {
|
45
43
|
'name' => 'q3_test',
|
@@ -48,17 +46,17 @@ class TargetTest < Test::Unit::TestCase
|
|
48
46
|
}, [])
|
49
47
|
C2 = Fluent::Config::Element.new('target', 'test', {
|
50
48
|
'exclude_regexp' => '(f|g)_.*',
|
51
|
-
'
|
49
|
+
'field_float' => 'd1,d2,d3,d4'
|
52
50
|
}, [Q3])
|
53
|
-
S2 = Fluent::
|
51
|
+
S2 = Fluent::NorikraPlugin::ConfigSection.new(C2)
|
54
52
|
|
55
53
|
def test_instanciate
|
56
54
|
t = @this.new('test', S1 + S2)
|
57
55
|
|
58
56
|
assert_equal 'test', t.name
|
59
57
|
assert_equal({
|
60
|
-
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :
|
61
|
-
:float => %w(f1 f2
|
58
|
+
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :integer => %w(i1 i2 i3 i4 num1 num2),
|
59
|
+
:float => %w(f1 f2 d d1 d2 d3 d4)
|
62
60
|
}, t.fields)
|
63
61
|
assert_equal 3, t.queries.size
|
64
62
|
|
@@ -70,9 +68,9 @@ class TargetTest < Test::Unit::TestCase
|
|
70
68
|
assert_equal({
|
71
69
|
's1' => 'string', 's2' => 'string', 's3' => 'string',
|
72
70
|
'bool1' => 'boolean', 'bool2' => 'boolean',
|
73
|
-
'i1' => '
|
71
|
+
'i1' => 'integer', 'i2' => 'integer', 'i3' => 'integer', 'i4' => 'integer', 'num1' => 'integer', 'num2' => 'integer',
|
74
72
|
'f1' => 'float', 'f2' => 'float',
|
75
|
-
'd' => '
|
73
|
+
'd' => 'float', 'd1' => 'float', 'd2' => 'float', 'd3' => 'float', 'd4' => 'float'
|
76
74
|
}, t.reserve_fields)
|
77
75
|
end
|
78
76
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-norikra
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-
|
11
|
+
date: 2013-12-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: norikra-client
|
@@ -81,15 +81,30 @@ files:
|
|
81
81
|
- README.md
|
82
82
|
- Rakefile
|
83
83
|
- example/blank.conf
|
84
|
+
- example/blank.rb
|
84
85
|
- example/blank2.conf
|
85
86
|
- example/example1.conf
|
86
87
|
- example/test1.conf
|
88
|
+
- example/test1.rb
|
87
89
|
- example/test2.conf
|
90
|
+
- example/test_in_out.rb
|
88
91
|
- fluent-plugin-norikra.gemspec
|
92
|
+
- lib/fluent/plugin/in_norikra.rb
|
93
|
+
- lib/fluent/plugin/norikra/config_section.rb
|
94
|
+
- lib/fluent/plugin/norikra/fetch_request.rb
|
95
|
+
- lib/fluent/plugin/norikra/input.rb
|
96
|
+
- lib/fluent/plugin/norikra/output.rb
|
97
|
+
- lib/fluent/plugin/norikra/query.rb
|
98
|
+
- lib/fluent/plugin/norikra/query_generator.rb
|
99
|
+
- lib/fluent/plugin/norikra/record_filter.rb
|
100
|
+
- lib/fluent/plugin/norikra/target.rb
|
89
101
|
- lib/fluent/plugin/norikra_target.rb
|
90
102
|
- lib/fluent/plugin/out_norikra.rb
|
103
|
+
- lib/fluent/plugin/out_norikra_filter.rb
|
91
104
|
- test/helper.rb
|
105
|
+
- test/plugin/test_in_norikra.rb
|
92
106
|
- test/plugin/test_out_norikra.rb
|
107
|
+
- test/plugin/test_out_norikra_filter.rb
|
93
108
|
- test/test_config_section.rb
|
94
109
|
- test/test_query.rb
|
95
110
|
- test/test_query_generator.rb
|
@@ -121,7 +136,9 @@ specification_version: 4
|
|
121
136
|
summary: Fluentd plugin to do CEP with norikra
|
122
137
|
test_files:
|
123
138
|
- test/helper.rb
|
139
|
+
- test/plugin/test_in_norikra.rb
|
124
140
|
- test/plugin/test_out_norikra.rb
|
141
|
+
- test/plugin/test_out_norikra_filter.rb
|
125
142
|
- test/test_config_section.rb
|
126
143
|
- test/test_query.rb
|
127
144
|
- test/test_query_generator.rb
|