fluent-plugin-norikra 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +13 -0
- data/README.md +179 -0
- data/Rakefile +10 -0
- data/example/blank.conf +15 -0
- data/example/example1.conf +68 -0
- data/example/test1.conf +36 -0
- data/fluent-plugin-norikra.gemspec +23 -0
- data/lib/fluent/plugin/norikra_target.rb +217 -0
- data/lib/fluent/plugin/out_norikra.rb +388 -0
- data/test/helper.rb +29 -0
- data/test/test_config_section.rb +179 -0
- data/test/test_query.rb +12 -0
- data/test/test_query_generator.rb +49 -0
- data/test/test_record_filter.rb +81 -0
- data/test/test_target.rb +66 -0
- metadata +124 -0
@@ -0,0 +1,388 @@
|
|
1
|
+
module Fluent
|
2
|
+
class NorikraOutput < Fluent::BufferedOutput
|
3
|
+
Fluent::Plugin.register_output('norikra', self)
|
4
|
+
|
5
|
+
config_set_default :flush_interval, 1 # 1sec
|
6
|
+
|
7
|
+
config_param :norikra, :string, :default => 'localhost:26571'
|
8
|
+
|
9
|
+
config_param :connect_timeout, :integer, :default => nil
|
10
|
+
config_param :send_timeout, :integer, :default => nil
|
11
|
+
config_param :receive_timeout, :integer, :default => nil
|
12
|
+
|
13
|
+
#<server>
|
14
|
+
attr_reader :execute_server, :execute_server_path
|
15
|
+
|
16
|
+
config_param :remove_tag_prefix, :string, :default => nil
|
17
|
+
|
18
|
+
config_param :target_map_tag, :bool, :default => false
|
19
|
+
config_param :target_map_key, :string, :default => nil
|
20
|
+
config_param :target_string, :string, :default => nil
|
21
|
+
|
22
|
+
# <default>
|
23
|
+
# <target TARGET>
|
24
|
+
|
25
|
+
# <events>
|
26
|
+
attr_reader :event_method, :event_tag_generator, :event_sweep_interval
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
super
|
30
|
+
require_relative 'norikra_target'
|
31
|
+
require 'norikra/client'
|
32
|
+
end
|
33
|
+
|
34
|
+
def configure(conf)
|
35
|
+
super
|
36
|
+
|
37
|
+
@host,@port = @norikra.split(':', 2)
|
38
|
+
@port = @port.to_i
|
39
|
+
|
40
|
+
if !@target_map_tag && @target_map_key.nil? && @target_string.nil?
|
41
|
+
raise Fluent::ConfigError, 'target naming not specified (target_map_tag/target_map_key/target_string)'
|
42
|
+
end
|
43
|
+
@target_generator = case
|
44
|
+
when @target_string
|
45
|
+
lambda {|tag,record| @target_string}
|
46
|
+
when @target_map_key
|
47
|
+
lambda {|tag,record| record[@target_map_key]}
|
48
|
+
when @target_map_tag
|
49
|
+
lambda {|tag,record| tag.gsub(/^#{@remove_tag_prefix}(\.)?/, '')}
|
50
|
+
else
|
51
|
+
raise Fluent::ConfigError, "no one way specified to decide target"
|
52
|
+
end
|
53
|
+
|
54
|
+
# target map already prepared (opened, and related queries registered)
|
55
|
+
@target_map = {} # 'target' => instance of Fluent::NorikraOutput::Target
|
56
|
+
|
57
|
+
# for conversion from query_name to tag
|
58
|
+
@query_map = {} # 'query_name' => instance of Fluent::NorikraOutput::Query
|
59
|
+
|
60
|
+
@default_target = ConfigSection.new(Fluent::Config::Element.new('default', nil, {}, []))
|
61
|
+
@config_targets = {}
|
62
|
+
|
63
|
+
@execute_server = false
|
64
|
+
|
65
|
+
event_section = nil
|
66
|
+
conf.elements.each do |element|
|
67
|
+
case element.name
|
68
|
+
when 'default'
|
69
|
+
@default_target = ConfigSection.new(element)
|
70
|
+
when 'target'
|
71
|
+
c = ConfigSection.new(element)
|
72
|
+
@config_targets[c.target] = c
|
73
|
+
when 'server'
|
74
|
+
@execute_server = Fluent::Config.bool_value(element['execute'])
|
75
|
+
@execute_server_path = element['path']
|
76
|
+
when 'event'
|
77
|
+
event_section = element
|
78
|
+
else
|
79
|
+
raise Fluent::ConfigError, "unknown configuration section name for this plugin: #{element.name}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
@event_method = @event_tag_generator = @event_sweep_interval = nil
|
84
|
+
if event_section
|
85
|
+
@event_method = case event_section['method']
|
86
|
+
when 'sweep' then :sweep
|
87
|
+
when 'listen'
|
88
|
+
raise Fluent::ConfigError, "not implemeneted now"
|
89
|
+
else
|
90
|
+
raise Fluent::ConfigError, "unknown method #{event_section['method']}"
|
91
|
+
end
|
92
|
+
unless event_section['tag']
|
93
|
+
raise Fluent::ConfigError, "<event> section needs 'tag' configuration"
|
94
|
+
end
|
95
|
+
tag_prefix = if event_section.has_key?('tag_prefix')
|
96
|
+
event_section['tag_prefix'] + (event_section['tag_prefix'] =~ /\.$/ ? '' : '.')
|
97
|
+
else
|
98
|
+
''
|
99
|
+
end
|
100
|
+
tag_by, tag_arg = event_section['tag'].split(/ +/, 2)
|
101
|
+
@event_tag_generator = case tag_by
|
102
|
+
when 'query_name' then lambda{|query_name,record| tag_prefix + query_name}
|
103
|
+
when 'field' then lambda{|query_name,record| tag_prefix + record[tag_arg]}
|
104
|
+
when 'string' then lambda{|query_name,record| tag_prefix + tag_arg}
|
105
|
+
else
|
106
|
+
raise Fluent::ConfigError, "unknown tag configuration specified:#{event_section['tag']}"
|
107
|
+
end
|
108
|
+
@event_sweep_interval = Fluent::Config.time_value(event_section['sweep_interval'] || '10s')
|
109
|
+
end
|
110
|
+
|
111
|
+
@mutex = Mutex.new
|
112
|
+
end
|
113
|
+
|
114
|
+
def client(opts={})
|
115
|
+
Norikra::Client.new(@host, @port, {
|
116
|
+
:connect_timeout => opts[:connect_timeout] || @connect_timeout,
|
117
|
+
:send_timeout => opts[:send_timeout] || @send_timeout,
|
118
|
+
:receive_timeout => opts[:receive_timeout] || @receive_timeout,
|
119
|
+
})
|
120
|
+
end
|
121
|
+
|
122
|
+
def start
|
123
|
+
super
|
124
|
+
|
125
|
+
@norikra_started = false
|
126
|
+
|
127
|
+
if @execute_server
|
128
|
+
@norikra_pid = nil
|
129
|
+
@norikra_thread = Thread.new(&method(:server_starter))
|
130
|
+
# @norikra_started will be set in server_starter
|
131
|
+
else
|
132
|
+
@norikra_pid = nil
|
133
|
+
@norikra_thread = nil
|
134
|
+
@norikra_started = true
|
135
|
+
end
|
136
|
+
|
137
|
+
# register worker thread
|
138
|
+
@register_queue = []
|
139
|
+
@register_thread = Thread.new(&method(:register_worker))
|
140
|
+
|
141
|
+
# fetch worker thread
|
142
|
+
@fetch_queue = []
|
143
|
+
@fetch_thread = Thread.new(&method(:fetch_worker))
|
144
|
+
|
145
|
+
# for sweep
|
146
|
+
if @event_method
|
147
|
+
@fetch_queue.push(FetchRequest.new(nil, @event_sweep_interval))
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def shutdown
|
152
|
+
@register_thread.kill
|
153
|
+
@fetch_thread.kill
|
154
|
+
Process.kill(:TERM, @norikra_pid) if @execute_server
|
155
|
+
|
156
|
+
@register_thread.join
|
157
|
+
@fetch_thread.join
|
158
|
+
begin
|
159
|
+
counter = 0
|
160
|
+
while !Process.waitpid(@norikra_pid, Process::WNOHANG)
|
161
|
+
sleep 1
|
162
|
+
break if counter > 3
|
163
|
+
end
|
164
|
+
rescue Errno::ECHILD
|
165
|
+
# norikra server process exited.
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def server_starter
|
170
|
+
$log.info "starting Norikra server process #{@host}:#{@port}"
|
171
|
+
@norikra_pid = fork do
|
172
|
+
ENV.keys.select{|k| k =~ /^(RUBY|GEM|BUNDLE|RBENV|RVM|rvm)/}.each {|k| ENV.delete(k)}
|
173
|
+
exec [@execute_server_path, 'norikra(fluentd)'], 'start', '-H', @host, '-P', @port.to_s
|
174
|
+
end
|
175
|
+
connecting = true
|
176
|
+
$log.info "trying to confirm norikra server status..."
|
177
|
+
while connecting
|
178
|
+
begin
|
179
|
+
$log.debug "start to connect norikra server #{@host}:#{@port}"
|
180
|
+
client(:connect_timeout => 1, :send_timeout => 1, :receive_timeout => 1).targets
|
181
|
+
# discard result: no exceptions is success
|
182
|
+
connecting = false
|
183
|
+
next
|
184
|
+
rescue HTTPClient::TimeoutError
|
185
|
+
$log.debug "Norikra server test connection timeout. retrying..."
|
186
|
+
rescue Errno::ECONNREFUSED
|
187
|
+
$log.debug "Norikra server test connection refused. retrying..."
|
188
|
+
rescue => e
|
189
|
+
$log.error "unknown error in confirming norikra server, #{e.class}:#{e.message}"
|
190
|
+
end
|
191
|
+
sleep 3
|
192
|
+
end
|
193
|
+
$log.info "confirmed that norikra server #{@host}:#{@port} started."
|
194
|
+
@norikra_started = true
|
195
|
+
end
|
196
|
+
|
197
|
+
def register_worker
|
198
|
+
while sleep(0.25)
|
199
|
+
next unless @norikra_started
|
200
|
+
|
201
|
+
c = client()
|
202
|
+
|
203
|
+
targets = @register_queue.shift(10)
|
204
|
+
targets.each do |t|
|
205
|
+
next if @target_map[t.name]
|
206
|
+
|
207
|
+
$log.debug "Preparing norikra target #{t.name} on #{@host}:#{@port}"
|
208
|
+
if prepare_target(c, t)
|
209
|
+
$log.debug "success to prepare target #{t.name} on #{@host}:#{@port}"
|
210
|
+
# success
|
211
|
+
t.queries.each do |query|
|
212
|
+
@query_map[query.name] = query
|
213
|
+
insert_fetch_queue(FetchRequest.new(query)) unless query.tag.empty? || @event_method
|
214
|
+
end
|
215
|
+
@target_map[t.name] = t
|
216
|
+
else
|
217
|
+
$log.error "Failed to prepare norikra data for target:#{t.name}"
|
218
|
+
@norikra_started.push(t)
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
def fetch_worker
|
225
|
+
while sleep(1)
|
226
|
+
next unless @norikra_started
|
227
|
+
next if @fetch_queue.first.nil? || @fetch_queue.first.time > Time.now
|
228
|
+
|
229
|
+
now = Time.now
|
230
|
+
while @fetch_queue.first.time <= now
|
231
|
+
req = @fetch_queue.shift
|
232
|
+
if req.query.nil?
|
233
|
+
sweep()
|
234
|
+
else
|
235
|
+
fetch(req.query)
|
236
|
+
end
|
237
|
+
insert_fetch_queue(req)
|
238
|
+
end
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
def format_stream(tag, es)
|
243
|
+
tobe_registered_target_names = []
|
244
|
+
|
245
|
+
out = ''
|
246
|
+
|
247
|
+
es.each do |time,record|
|
248
|
+
target = @target_generator.call(tag, record)
|
249
|
+
|
250
|
+
t = @target_map[target]
|
251
|
+
unless t || tobe_registered_target_names.include?(target)
|
252
|
+
t = Target.new(target, @default_target + @config_targets[target])
|
253
|
+
@register_queue.push(t)
|
254
|
+
tobe_registered_target_names.push(target)
|
255
|
+
end
|
256
|
+
|
257
|
+
event = t.filter(record)
|
258
|
+
|
259
|
+
out << [target,event].to_msgpack
|
260
|
+
end
|
261
|
+
|
262
|
+
out
|
263
|
+
end
|
264
|
+
|
265
|
+
def prepared?(target_names)
|
266
|
+
@norikra_started && target_names.reduce(true){|r,t| r && @target_map[t]}
|
267
|
+
end
|
268
|
+
|
269
|
+
def write(chunk)
|
270
|
+
events_map = {} # target => [event]
|
271
|
+
chunk.msgpack_each do |target, event|
|
272
|
+
events_map[target] ||= []
|
273
|
+
events_map[target].push(event)
|
274
|
+
end
|
275
|
+
|
276
|
+
unless prepared?(events_map.keys)
|
277
|
+
raise RuntimeError, "norikra server is not ready for this targets: #{events_map.keys.join(',')}"
|
278
|
+
end
|
279
|
+
|
280
|
+
c = client()
|
281
|
+
|
282
|
+
events_map.each do |target, events|
|
283
|
+
c.send(target, events)
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def prepare_target(client, target)
|
288
|
+
# target open and reserve fields
|
289
|
+
$log.debug "Going to prepare about target"
|
290
|
+
begin
|
291
|
+
unless client.targets.include?(target.name)
|
292
|
+
$log.debug "opening target #{target.name}"
|
293
|
+
client.open(target.name, target.reserve_fields)
|
294
|
+
$log.debug "opening target #{target.name}, done."
|
295
|
+
end
|
296
|
+
|
297
|
+
reserving = target.reserve_fields
|
298
|
+
reserved = []
|
299
|
+
client.fields(target.name).each do |field|
|
300
|
+
if reserving[field['name']]
|
301
|
+
reserved.push(field['name'])
|
302
|
+
if reserving[field['name']] != field['type']
|
303
|
+
$log.warn "field type mismatch, reserving:#{reserving[field['name']]} but reserved:#{field['type']}"
|
304
|
+
end
|
305
|
+
end
|
306
|
+
end
|
307
|
+
|
308
|
+
reserving.each do |fieldname,type|
|
309
|
+
client.reserve(target, fieldname, type) unless reserved.include?(fieldname)
|
310
|
+
end
|
311
|
+
rescue => e
|
312
|
+
$log.error "failed to prepare target:#{target.name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
|
313
|
+
return false
|
314
|
+
end
|
315
|
+
|
316
|
+
# query registration
|
317
|
+
begin
|
318
|
+
registered = Hash[client.queries.map{|q| [q['name'], q['expression']]}]
|
319
|
+
target.queries.each do |query|
|
320
|
+
if registered.has_key?(query.name) # query already registered
|
321
|
+
if registered[query.name] != query.expression
|
322
|
+
$log.warn "query name and expression mismatch, check norikra server status. target query name:#{query.name}"
|
323
|
+
end
|
324
|
+
next
|
325
|
+
end
|
326
|
+
client.register(query.name, query.expression)
|
327
|
+
end
|
328
|
+
rescue => e
|
329
|
+
$log.warn "failed to register query", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
class FetchRequest
|
334
|
+
attr_accessor :time, :query
|
335
|
+
def initialize(query, interval=nil)
|
336
|
+
@query = query
|
337
|
+
@interval = interval || query.interval
|
338
|
+
@time = Time.now + @interval
|
339
|
+
end
|
340
|
+
def <=>(other)
|
341
|
+
self.time <=> other.time
|
342
|
+
end
|
343
|
+
def next!
|
344
|
+
@time = Time.now + @interval
|
345
|
+
end
|
346
|
+
end
|
347
|
+
|
348
|
+
def insert_fetch_queue(request)
|
349
|
+
@mutex.synchronize do
|
350
|
+
request.next!
|
351
|
+
# if @fetch_queue.size > 0
|
352
|
+
# next_pos = @fetch_queue.bsearch{|req| req.time > request.time}
|
353
|
+
# @fetch_queue.insert(next_pos, request)
|
354
|
+
# else
|
355
|
+
# @fetch_queue.push(request)
|
356
|
+
# end
|
357
|
+
@fetch_queue.push(request)
|
358
|
+
@fetch_queue.sort!
|
359
|
+
end
|
360
|
+
rescue => e
|
361
|
+
$log.error "unknown log encountered", :error_class => e.class, :message => e.message
|
362
|
+
end
|
363
|
+
|
364
|
+
def sweep
|
365
|
+
begin
|
366
|
+
client().sweep.each do |query_name, event_array|
|
367
|
+
query = @query_map[query_name]
|
368
|
+
event_array.each do |time,event|
|
369
|
+
tag = (query && !query.tag.empty?) ? query.tag : @event_tag_generator.call(query_name, event)
|
370
|
+
Fluent::Engine.emit(tag, time, event)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
rescue => e
|
374
|
+
$log.error "failed to sweep", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
def fetch(query)
|
379
|
+
begin
|
380
|
+
client().event(query.name).each do |time,event| # [[time(int from epoch), event], ...]
|
381
|
+
Fluent::Engine.emit(query.tag, time, event)
|
382
|
+
end
|
383
|
+
rescue => e
|
384
|
+
$log.error "failed to fetch for query:#{query.name}", :norikra => "#{@host}:#{@port}", :error => e.class, :message => e.message
|
385
|
+
end
|
386
|
+
end
|
387
|
+
end
|
388
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler'
|
3
|
+
begin
|
4
|
+
Bundler.setup(:default, :development)
|
5
|
+
rescue Bundler::BundlerError => e
|
6
|
+
$stderr.puts e.message
|
7
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
8
|
+
exit e.status_code
|
9
|
+
end
|
10
|
+
require 'test/unit'
|
11
|
+
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
14
|
+
|
15
|
+
require 'fluent/test'
|
16
|
+
unless ENV.has_key?('VERBOSE')
|
17
|
+
nulllogger = Object.new
|
18
|
+
nulllogger.instance_eval {|obj|
|
19
|
+
def method_missing(method, *args)
|
20
|
+
# pass
|
21
|
+
end
|
22
|
+
}
|
23
|
+
$log = nulllogger
|
24
|
+
end
|
25
|
+
|
26
|
+
require 'fluent/plugin/out_norikra'
|
27
|
+
|
28
|
+
class Test::Unit::TestCase
|
29
|
+
end
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/plugin/norikra_target'
|
3
|
+
|
4
|
+
class ConfigSectionTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
@this = Fluent::NorikraOutput::ConfigSection
|
7
|
+
end
|
8
|
+
|
9
|
+
def test_init_default
|
10
|
+
q1 = Fluent::Config::Element.new('query', nil, {
|
11
|
+
'name' => 'q1_${target}',
|
12
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
|
13
|
+
'tag' => 'q1.${target}'
|
14
|
+
}, [])
|
15
|
+
q2 = Fluent::Config::Element.new('query', nil, {
|
16
|
+
'name' => 'q2_${target}',
|
17
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
|
18
|
+
'tag' => 'q2.${target}'
|
19
|
+
}, [])
|
20
|
+
c1 = Fluent::Config::Element.new('default', nil, {
|
21
|
+
'include' => '*',
|
22
|
+
'exclude' => 'flag',
|
23
|
+
'exclude_regexp' => 'f_.*',
|
24
|
+
'field_string' => 's1,s2,s3',
|
25
|
+
'field_boolean' => 'bool1,bool2',
|
26
|
+
'field_int' => 'i1,i2,i3,i4',
|
27
|
+
'field_long' => 'num1,num2',
|
28
|
+
'field_float' => 'f1,f2',
|
29
|
+
'field_double' => 'd'
|
30
|
+
}, [q1,q2])
|
31
|
+
s1 = @this.new(c1)
|
32
|
+
|
33
|
+
assert_nil s1.target
|
34
|
+
assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => 'f_.*'}, s1.filter_params)
|
35
|
+
assert_equal({
|
36
|
+
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :int => %w(i1 i2 i3 i4), :long => %w(num1 num2),
|
37
|
+
:float => %w(f1 f2), :double => %w(d)
|
38
|
+
}, s1.field_definitions)
|
39
|
+
assert_equal 2, s1.query_generators.size
|
40
|
+
assert_equal (10 * 60 / 5), s1.query_generators.map(&:fetch_interval).sort.first
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_init_target
|
44
|
+
q3 = Fluent::Config::Element.new('query', nil, {
|
45
|
+
'name' => 'q3_test2',
|
46
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"',
|
47
|
+
'tag' => 'q3.test2'
|
48
|
+
}, [])
|
49
|
+
c2 = Fluent::Config::Element.new('target', 'test2', {
|
50
|
+
'exclude_regexp' => '(f|g)_.*',
|
51
|
+
'field_double' => 'd1,d2,d3,d4'
|
52
|
+
}, [q3])
|
53
|
+
s2 = @this.new(c2)
|
54
|
+
|
55
|
+
assert_equal 'test2', s2.target
|
56
|
+
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => '(f|g)_.*'}, s2.filter_params)
|
57
|
+
assert_equal({:string => [], :boolean => [], :int => [], :long => [], :float => [], :double => %w(d1 d2 d3 d4)}, s2.field_definitions)
|
58
|
+
assert_equal 1, s2.query_generators.size
|
59
|
+
assert_equal (30 * 60 / 5), s2.query_generators.map(&:fetch_interval).sort.first
|
60
|
+
end
|
61
|
+
|
62
|
+
def test_init_target_query_only
|
63
|
+
q4 = Fluent::Config::Element.new('query', nil, {
|
64
|
+
'name' => 'q4_test3',
|
65
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q4 > 10',
|
66
|
+
'tag' => 'q4.test3',
|
67
|
+
'fetch_interval' => '1s'
|
68
|
+
}, [])
|
69
|
+
c3 = Fluent::Config::Element.new('target', 'test3', {}, [q4])
|
70
|
+
s3 = @this.new(c3)
|
71
|
+
|
72
|
+
assert_equal 'test3', s3.target
|
73
|
+
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s3.filter_params)
|
74
|
+
assert_equal({:string => [], :boolean => [], :int => [], :long => [], :float => [], :double => []}, s3.field_definitions)
|
75
|
+
assert_equal 1, s3.query_generators.size
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_init_target_without_query
|
79
|
+
c4 = Fluent::Config::Element.new('target', 'test4', {
|
80
|
+
'field_int' => 'status'
|
81
|
+
}, [])
|
82
|
+
s4 = @this.new(c4)
|
83
|
+
|
84
|
+
assert_equal 'test4', s4.target
|
85
|
+
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s4.filter_params)
|
86
|
+
assert_equal({:string => [], :boolean => [], :int => ['status'], :long => [], :float => [], :double => []}, s4.field_definitions)
|
87
|
+
assert_equal 0, s4.query_generators.size
|
88
|
+
end
|
89
|
+
|
90
|
+
def test_init_target_blank
|
91
|
+
c5 = Fluent::Config::Element.new('target', 'test5', {}, [])
|
92
|
+
s5 = @this.new(c5)
|
93
|
+
|
94
|
+
assert_equal 'test5', s5.target
|
95
|
+
assert_equal({:include => nil, :include_regexp => nil, :exclude => nil, :exclude_regexp => nil}, s5.filter_params)
|
96
|
+
assert_equal({:string => [], :boolean => [], :int => [], :long => [], :float => [], :double => []}, s5.field_definitions)
|
97
|
+
assert_equal 0, s5.query_generators.size
|
98
|
+
end
|
99
|
+
|
100
|
+
def test_join
|
101
|
+
q1 = Fluent::Config::Element.new('query', nil, {
|
102
|
+
'name' => 'q1_${target}',
|
103
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
|
104
|
+
'tag' => 'q1.${target}'
|
105
|
+
}, [])
|
106
|
+
q2 = Fluent::Config::Element.new('query', nil, {
|
107
|
+
'name' => 'q2_${target}',
|
108
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
|
109
|
+
'tag' => 'q2.${target}'
|
110
|
+
}, [])
|
111
|
+
c1 = Fluent::Config::Element.new('default', nil, {
|
112
|
+
'include' => '*',
|
113
|
+
'exclude' => 'flag',
|
114
|
+
'exclude_regexp' => 'f_.*',
|
115
|
+
'field_string' => 's1,s2,s3',
|
116
|
+
'field_boolean' => 'bool1,bool2',
|
117
|
+
'field_int' => 'i1,i2,i3,i4',
|
118
|
+
'field_long' => 'num1,num2',
|
119
|
+
'field_float' => 'f1,f2',
|
120
|
+
'field_double' => 'd'
|
121
|
+
}, [q1,q2])
|
122
|
+
s1 = @this.new(c1)
|
123
|
+
|
124
|
+
q3 = Fluent::Config::Element.new('query', nil, {
|
125
|
+
'name' => 'q3_test',
|
126
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(30 min) WHERE q3="/"',
|
127
|
+
'tag' => 'q3.test'
|
128
|
+
}, [])
|
129
|
+
c2 = Fluent::Config::Element.new('target', 'test', {
|
130
|
+
'exclude_regexp' => '(f|g)_.*',
|
131
|
+
'field_double' => 'd1,d2,d3,d4'
|
132
|
+
}, [q3])
|
133
|
+
s2 = @this.new(c2)
|
134
|
+
|
135
|
+
s = s1 + s2
|
136
|
+
|
137
|
+
assert_equal 'test', s.target
|
138
|
+
assert_equal({:include => '*', :include_regexp => nil, :exclude => 'flag', :exclude_regexp => '(f|g)_.*'}, s.filter_params)
|
139
|
+
assert_equal({
|
140
|
+
:string => %w(s1 s2 s3), :boolean => %w(bool1 bool2), :int => %w(i1 i2 i3 i4), :long => %w(num1 num2),
|
141
|
+
:float => %w(f1 f2), :double => %w(d d1 d2 d3 d4)
|
142
|
+
}, s.field_definitions)
|
143
|
+
assert_equal 3, s.query_generators.size
|
144
|
+
assert_equal (10 * 60 / 5), s.query_generators.map(&:fetch_interval).sort.first
|
145
|
+
end
|
146
|
+
|
147
|
+
def test_join_with_nil
|
148
|
+
q1 = Fluent::Config::Element.new('query', nil, {
|
149
|
+
'name' => 'q1_${target}',
|
150
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(10 min) WHERE q1',
|
151
|
+
'tag' => 'q1.${target}'
|
152
|
+
}, [])
|
153
|
+
q2 = Fluent::Config::Element.new('query', nil, {
|
154
|
+
'name' => 'q2_${target}',
|
155
|
+
'expression' => 'SELECT * FROM ${target}.win:time_batch(50 min) WHERE q2.length() > 0',
|
156
|
+
'tag' => 'q2.${target}'
|
157
|
+
}, [])
|
158
|
+
c1 = Fluent::Config::Element.new('default', nil, {
|
159
|
+
'include' => '*',
|
160
|
+
'exclude' => 'flag',
|
161
|
+
'exclude_regexp' => 'f_.*',
|
162
|
+
'field_string' => 's1,s2,s3',
|
163
|
+
'field_boolean' => 'bool1,bool2',
|
164
|
+
'field_int' => 'i1,i2,i3,i4',
|
165
|
+
'field_long' => 'num1,num2',
|
166
|
+
'field_float' => 'f1,f2',
|
167
|
+
'field_double' => 'd'
|
168
|
+
}, [q1,q2])
|
169
|
+
s1 = @this.new(c1)
|
170
|
+
|
171
|
+
s = s1 + nil
|
172
|
+
|
173
|
+
assert_equal 'dummy', s.target
|
174
|
+
assert_equal s1.filter_params, s.filter_params
|
175
|
+
assert_equal s1.field_definitions, s.field_definitions
|
176
|
+
assert_equal s1.query_generators.size, s.query_generators.size
|
177
|
+
assert_equal s1.query_generators.map(&:fetch_interval).sort.first, s.query_generators.map(&:fetch_interval).sort.first
|
178
|
+
end
|
179
|
+
end
|
data/test/test_query.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/plugin/norikra_target'
|
3
|
+
|
4
|
+
class QueryTest < Test::Unit::TestCase
|
5
|
+
def test_init
|
6
|
+
q = Fluent::NorikraOutput::Query.new('name', 'expression', 'tag', 10)
|
7
|
+
assert_equal 'name', q.name
|
8
|
+
assert_equal 'expression', q.expression
|
9
|
+
assert_equal 'tag', q.tag
|
10
|
+
assert_equal 10, q.interval
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'fluent/plugin/norikra_target'
|
3
|
+
|
4
|
+
class QueryGeneratorTest < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
@this = Fluent::NorikraOutput::QueryGenerator
|
7
|
+
end
|
8
|
+
|
9
|
+
def test_replace_target
|
10
|
+
expected = 'SELECT * FROM replaced.win:time_batch(10 hours) WHERE x=1'
|
11
|
+
assert_equal expected, @this.replace_target('replaced', 'SELECT * FROM ${target}.win:time_batch(10 hours) WHERE x=1')
|
12
|
+
end
|
13
|
+
|
14
|
+
def test_parse_time_period
|
15
|
+
assert_equal [0,0,0,0,0,0,0,0], @this.parse_time_period('')
|
16
|
+
|
17
|
+
assert_equal [10,0,0,0,0,0,0,0], @this.parse_time_period(' 10 year')
|
18
|
+
assert_equal [0,11,0,0,0,0,0,0], @this.parse_time_period('11 month')
|
19
|
+
assert_equal [0,233,0,0,0,0,0,0], @this.parse_time_period('233 months')
|
20
|
+
assert_equal [0,0,10,0,0,0,0,0], @this.parse_time_period('10 weeks')
|
21
|
+
assert_equal [0,0,0,1,0,0,0,0], @this.parse_time_period('1 day')
|
22
|
+
assert_equal [0,0,0,201,0,0,0,0], @this.parse_time_period('201 days')
|
23
|
+
assert_equal [0,0,0,0,1,0,0,0], @this.parse_time_period('1 hour')
|
24
|
+
assert_equal [0,0,0,0,11,0,0,0], @this.parse_time_period('11 hours')
|
25
|
+
assert_equal [0,0,0,0,0,2,0,0], @this.parse_time_period('2 minutes')
|
26
|
+
assert_equal [0,0,0,0,0,1,0,0], @this.parse_time_period('1 min')
|
27
|
+
assert_equal [0,0,0,0,0,133,0,0], @this.parse_time_period('133 minute')
|
28
|
+
assert_equal [0,0,0,0,0,0,12,0], @this.parse_time_period('12 sec')
|
29
|
+
assert_equal [0,0,0,0,0,0,1,0], @this.parse_time_period('1 second')
|
30
|
+
assert_equal [0,0,0,0,0,0,256,0], @this.parse_time_period(' 256 seconds')
|
31
|
+
assert_equal [0,0,0,0,0,0,0,1], @this.parse_time_period('1 msec')
|
32
|
+
assert_equal [0,0,0,0,0,0,0,111], @this.parse_time_period('111 milliseconds')
|
33
|
+
|
34
|
+
assert_equal [1,12,4,365,23,59,60,0], @this.parse_time_period('1 year 12 months 4 weeks 365 days 23 hours 59 min 60 seconds')
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_generate
|
38
|
+
g = @this.new('query_${target}', 'SELECT * FROM ${target}.win:time_batch( 10 min ) WHERE x=1', 'tag.${target}')
|
39
|
+
q = g.generate('test')
|
40
|
+
assert_equal 'query_test', q.name
|
41
|
+
assert_equal 'SELECT * FROM test.win:time_batch( 10 min ) WHERE x=1', q.expression
|
42
|
+
assert_equal 'tag.test', q.tag
|
43
|
+
end
|
44
|
+
|
45
|
+
def test_fetch_interval
|
46
|
+
g = @this.new('query_${target}', 'SELECT * FROM ${target}.win:time_batch( 12 min ) WHERE x=1', 'tag.${target}')
|
47
|
+
assert_equal (12*60/5), g.fetch_interval
|
48
|
+
end
|
49
|
+
end
|