fluent-plugin-unit-time-filter 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6e0bf31be1cb31c459450da1fe3d767a6ed82f3d
|
4
|
+
data.tar.gz: 48f82a08b5bd76cbe50f5d930a297d5ba67f74c4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7dd5dedbbcbfaf1425baddaef19c51856ddfcdc149afcbd44d5b2d144dfa1806745ef72b063a0d6f7acc41763d36579fa7217ddae1a739286f93a6c149476457
|
7
|
+
data.tar.gz: 24a6577e6917f9c79f92521030c483f5884e449c47df6ec114c5b76693164a7fd9ab8de49e20a25a9153b6b95316d0a85252862865dde4a33b1edd63aa3f6e77
|
data/README.md
CHANGED
@@ -5,6 +5,10 @@ Plug-in to aggregate by unit time
|
|
5
5
|
[](http://badge.fury.io/rb/fluent-plugin-unit-time-filter)
|
6
6
|
[](https://drone.io/bitbucket.org/winebarrel/fluent-plugin-unit-time-filter/latest)
|
7
7
|
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
$ gem install fluent-plugin-unit-time-filter
|
11
|
+
|
8
12
|
## Configuration
|
9
13
|
|
10
14
|
```
|
@@ -63,11 +67,3 @@ fluentd outputs the following:
|
|
63
67
|
2014-02-08 00:39:00 +0900 filtered.my.data: {"count":41}
|
64
68
|
...
|
65
69
|
```
|
66
|
-
|
67
|
-
## Contributing
|
68
|
-
|
69
|
-
1. Fork it
|
70
|
-
2. Create your feature branch (`git checkout -b my-new-feature`)
|
71
|
-
3. Commit your changes (`git commit -am 'Add some feature'`)
|
72
|
-
4. Push to the branch (`git push origin my-new-feature`)
|
73
|
-
5. Create new Pull Request
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "fluent-plugin-unit-time-filter"
|
4
|
-
spec.version = "0.1.
|
4
|
+
spec.version = "0.1.2"
|
5
5
|
spec.authors = ["Genki Sugawara"]
|
6
6
|
spec.email = ["sgwr_dts@yahoo.co.jp"]
|
7
7
|
spec.description = %q{Plug-in to aggregate by unit time}
|
@@ -3,6 +3,10 @@ require 'fluent/plugin/unit_time_filter_buffer'
|
|
3
3
|
class Fluent::UnitTimeFilterOutput < Fluent::Output
|
4
4
|
Fluent::Plugin.register_output('unit_time_filter', self)
|
5
5
|
|
6
|
+
unless method_defined?(:log)
|
7
|
+
define_method("log") { $log }
|
8
|
+
end
|
9
|
+
|
6
10
|
config_param :filter_path, :type => :string
|
7
11
|
config_param :unit_sec, :type => :size, :default => 1
|
8
12
|
config_param :prefix, :type => :string, :default => 'filtered'
|
@@ -48,6 +52,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
|
|
48
52
|
:pass_hash_row => @pass_hash_row,
|
49
53
|
:hash_row_time_key => @hash_row_time_key,
|
50
54
|
:hash_row_tag_key => @hash_row_tag_key,
|
55
|
+
:log => log,
|
51
56
|
})
|
52
57
|
|
53
58
|
Thread.current[BUFFER_KEY] = buf
|
@@ -57,6 +62,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
|
|
57
62
|
yield(buf)
|
58
63
|
rescue Exception => e
|
59
64
|
Thread.current[BUFFER_KEY] = nil
|
65
|
+
log.error(e)
|
60
66
|
raise e
|
61
67
|
end
|
62
68
|
end
|
@@ -4,6 +4,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
|
|
4
4
|
|
5
5
|
def initialize(options)
|
6
6
|
@init_queue = []
|
7
|
+
@log = options[:log]
|
7
8
|
|
8
9
|
@fiber = Fiber.new do |tag, es|
|
9
10
|
@init_queue << [tag, es]
|
@@ -43,7 +44,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
|
|
43
44
|
prev_time = time
|
44
45
|
result
|
45
46
|
}.each {|records|
|
46
|
-
records = records.
|
47
|
+
records = records.map {|i| i[1] }.inject(:+)
|
47
48
|
records = conv_to_hash_rows_if_needed(records, options)
|
48
49
|
|
49
50
|
time = get_time(records.first, options)
|
@@ -64,7 +65,11 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
|
|
64
65
|
|
65
66
|
def emit_records(prefix, tag, time, records)
|
66
67
|
records.each do |record|
|
67
|
-
|
68
|
+
if record.kind_of?(Hash)
|
69
|
+
Fluent::Engine.emit("#{prefix}.#{tag}", time, record)
|
70
|
+
else
|
71
|
+
@log.warn("Record must be Hash: #{record.inspect} (#{record.class})")
|
72
|
+
end
|
68
73
|
end
|
69
74
|
end
|
70
75
|
|
@@ -235,7 +235,7 @@ describe Fluent::UnitTimeFilterOutput do
|
|
235
235
|
end
|
236
236
|
|
237
237
|
describe 'when an error happened' do
|
238
|
-
it 'filter name should be
|
238
|
+
it 'filter name should be included in the error' do
|
239
239
|
filter = <<-EOS
|
240
240
|
proc {|rs|
|
241
241
|
raise 'Any error message'
|
@@ -243,6 +243,8 @@ describe Fluent::UnitTimeFilterOutput do
|
|
243
243
|
EOS
|
244
244
|
|
245
245
|
run_driver(:filter => filter, :tempfile => 'any_filter.rb') do |d|
|
246
|
+
d.instance.log.should_receive(:error) {|e| expect(e.message).to eq('Any error message') }
|
247
|
+
|
246
248
|
begin
|
247
249
|
(0...10).each do |i|
|
248
250
|
d.emit({"key#{i}" => "val#{i}"}, time + i)
|
@@ -255,4 +257,109 @@ describe Fluent::UnitTimeFilterOutput do
|
|
255
257
|
end
|
256
258
|
end
|
257
259
|
end
|
260
|
+
|
261
|
+
describe 'when an invalid record was included' do
|
262
|
+
it 'should skip the bad records (Filter returns a Hash)' do
|
263
|
+
filter = <<-EOS
|
264
|
+
proc {|rs|
|
265
|
+
if rs.any? {|i| i[2].has_key?('return_nil')}
|
266
|
+
nil
|
267
|
+
else
|
268
|
+
{
|
269
|
+
'count' => rs.count,
|
270
|
+
'inspect' => rs.inspect
|
271
|
+
}
|
272
|
+
end
|
273
|
+
}
|
274
|
+
EOS
|
275
|
+
|
276
|
+
run_driver(:filter => filter) do |d|
|
277
|
+
d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: nil (NilClass)') }
|
278
|
+
|
279
|
+
(0...10).each do |i|
|
280
|
+
d.emit({"key#{i}" => "val#{i}"}, time + i)
|
281
|
+
d.emit({"key#{i}_" => "val#{i}_"}, time + i)
|
282
|
+
end
|
283
|
+
|
284
|
+
d.emit({"return_nil" => 1}, time + 10)
|
285
|
+
|
286
|
+
(10...20).each do |i|
|
287
|
+
d.emit({"key#{i}" => "val#{i}"}, time + i)
|
288
|
+
d.emit({"key#{i}_" => "val#{i}_"}, time + i)
|
289
|
+
end
|
290
|
+
|
291
|
+
expect(d.emits).to eq(
|
292
|
+
[["filtered.test.default", 1391832855, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832855, {\"key0\"=>\"val0\"}], [\"test.default\", 1391832855, {\"key0_\"=>\"val0_\"}]]"}],
|
293
|
+
["filtered.test.default", 1391832856, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832856, {\"key1\"=>\"val1\"}], [\"test.default\", 1391832856, {\"key1_\"=>\"val1_\"}]]"}],
|
294
|
+
["filtered.test.default", 1391832857, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832857, {\"key2\"=>\"val2\"}], [\"test.default\", 1391832857, {\"key2_\"=>\"val2_\"}]]"}],
|
295
|
+
["filtered.test.default", 1391832858, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832858, {\"key3\"=>\"val3\"}], [\"test.default\", 1391832858, {\"key3_\"=>\"val3_\"}]]"}],
|
296
|
+
["filtered.test.default", 1391832859, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832859, {\"key4\"=>\"val4\"}], [\"test.default\", 1391832859, {\"key4_\"=>\"val4_\"}]]"}],
|
297
|
+
["filtered.test.default", 1391832860, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832860, {\"key5\"=>\"val5\"}], [\"test.default\", 1391832860, {\"key5_\"=>\"val5_\"}]]"}],
|
298
|
+
["filtered.test.default", 1391832861, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832861, {\"key6\"=>\"val6\"}], [\"test.default\", 1391832861, {\"key6_\"=>\"val6_\"}]]"}],
|
299
|
+
["filtered.test.default", 1391832862, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832862, {\"key7\"=>\"val7\"}], [\"test.default\", 1391832862, {\"key7_\"=>\"val7_\"}]]"}],
|
300
|
+
["filtered.test.default", 1391832863, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832863, {\"key8\"=>\"val8\"}], [\"test.default\", 1391832863, {\"key8_\"=>\"val8_\"}]]"}],
|
301
|
+
["filtered.test.default", 1391832864, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832864, {\"key9\"=>\"val9\"}], [\"test.default\", 1391832864, {\"key9_\"=>\"val9_\"}]]"}],
|
302
|
+
["filtered.test.default", 1391832866, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832866, {\"key11\"=>\"val11\"}], [\"test.default\", 1391832866, {\"key11_\"=>\"val11_\"}]]"}],
|
303
|
+
["filtered.test.default", 1391832867, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832867, {\"key12\"=>\"val12\"}], [\"test.default\", 1391832867, {\"key12_\"=>\"val12_\"}]]"}],
|
304
|
+
["filtered.test.default", 1391832868, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832868, {\"key13\"=>\"val13\"}], [\"test.default\", 1391832868, {\"key13_\"=>\"val13_\"}]]"}],
|
305
|
+
["filtered.test.default", 1391832869, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832869, {\"key14\"=>\"val14\"}], [\"test.default\", 1391832869, {\"key14_\"=>\"val14_\"}]]"}],
|
306
|
+
["filtered.test.default", 1391832870, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832870, {\"key15\"=>\"val15\"}], [\"test.default\", 1391832870, {\"key15_\"=>\"val15_\"}]]"}],
|
307
|
+
["filtered.test.default", 1391832871, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832871, {\"key16\"=>\"val16\"}], [\"test.default\", 1391832871, {\"key16_\"=>\"val16_\"}]]"}],
|
308
|
+
["filtered.test.default", 1391832872, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832872, {\"key17\"=>\"val17\"}], [\"test.default\", 1391832872, {\"key17_\"=>\"val17_\"}]]"}]]
|
309
|
+
)
|
310
|
+
end
|
311
|
+
end
|
312
|
+
|
313
|
+
it 'should skip the bad records (Filter returns an Array)' do
|
314
|
+
filter = <<-EOS
|
315
|
+
proc {|rs|
|
316
|
+
if rs.any? {|i| i[2].has_key?('return_nil')}
|
317
|
+
[nil, 1]
|
318
|
+
else
|
319
|
+
{
|
320
|
+
'count' => rs.count,
|
321
|
+
'inspect' => rs.inspect
|
322
|
+
}
|
323
|
+
end
|
324
|
+
}
|
325
|
+
EOS
|
326
|
+
|
327
|
+
run_driver(:filter => filter) do |d|
|
328
|
+
d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: nil (NilClass)') }
|
329
|
+
d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: 1 (Fixnum)') }
|
330
|
+
|
331
|
+
(0...10).each do |i|
|
332
|
+
d.emit({"key#{i}" => "val#{i}"}, time + i)
|
333
|
+
d.emit({"key#{i}_" => "val#{i}_"}, time + i)
|
334
|
+
end
|
335
|
+
|
336
|
+
d.emit({"return_nil" => 1}, time + 10)
|
337
|
+
|
338
|
+
(10...20).each do |i|
|
339
|
+
d.emit({"key#{i}" => "val#{i}"}, time + i)
|
340
|
+
d.emit({"key#{i}_" => "val#{i}_"}, time + i)
|
341
|
+
end
|
342
|
+
|
343
|
+
expect(d.emits).to eq(
|
344
|
+
[["filtered.test.default", 1391832855, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832855, {\"key0\"=>\"val0\"}], [\"test.default\", 1391832855, {\"key0_\"=>\"val0_\"}]]"}],
|
345
|
+
["filtered.test.default", 1391832856, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832856, {\"key1\"=>\"val1\"}], [\"test.default\", 1391832856, {\"key1_\"=>\"val1_\"}]]"}],
|
346
|
+
["filtered.test.default", 1391832857, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832857, {\"key2\"=>\"val2\"}], [\"test.default\", 1391832857, {\"key2_\"=>\"val2_\"}]]"}],
|
347
|
+
["filtered.test.default", 1391832858, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832858, {\"key3\"=>\"val3\"}], [\"test.default\", 1391832858, {\"key3_\"=>\"val3_\"}]]"}],
|
348
|
+
["filtered.test.default", 1391832859, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832859, {\"key4\"=>\"val4\"}], [\"test.default\", 1391832859, {\"key4_\"=>\"val4_\"}]]"}],
|
349
|
+
["filtered.test.default", 1391832860, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832860, {\"key5\"=>\"val5\"}], [\"test.default\", 1391832860, {\"key5_\"=>\"val5_\"}]]"}],
|
350
|
+
["filtered.test.default", 1391832861, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832861, {\"key6\"=>\"val6\"}], [\"test.default\", 1391832861, {\"key6_\"=>\"val6_\"}]]"}],
|
351
|
+
["filtered.test.default", 1391832862, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832862, {\"key7\"=>\"val7\"}], [\"test.default\", 1391832862, {\"key7_\"=>\"val7_\"}]]"}],
|
352
|
+
["filtered.test.default", 1391832863, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832863, {\"key8\"=>\"val8\"}], [\"test.default\", 1391832863, {\"key8_\"=>\"val8_\"}]]"}],
|
353
|
+
["filtered.test.default", 1391832864, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832864, {\"key9\"=>\"val9\"}], [\"test.default\", 1391832864, {\"key9_\"=>\"val9_\"}]]"}],
|
354
|
+
["filtered.test.default", 1391832866, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832866, {\"key11\"=>\"val11\"}], [\"test.default\", 1391832866, {\"key11_\"=>\"val11_\"}]]"}],
|
355
|
+
["filtered.test.default", 1391832867, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832867, {\"key12\"=>\"val12\"}], [\"test.default\", 1391832867, {\"key12_\"=>\"val12_\"}]]"}],
|
356
|
+
["filtered.test.default", 1391832868, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832868, {\"key13\"=>\"val13\"}], [\"test.default\", 1391832868, {\"key13_\"=>\"val13_\"}]]"}],
|
357
|
+
["filtered.test.default", 1391832869, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832869, {\"key14\"=>\"val14\"}], [\"test.default\", 1391832869, {\"key14_\"=>\"val14_\"}]]"}],
|
358
|
+
["filtered.test.default", 1391832870, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832870, {\"key15\"=>\"val15\"}], [\"test.default\", 1391832870, {\"key15_\"=>\"val15_\"}]]"}],
|
359
|
+
["filtered.test.default", 1391832871, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832871, {\"key16\"=>\"val16\"}], [\"test.default\", 1391832871, {\"key16_\"=>\"val16_\"}]]"}],
|
360
|
+
["filtered.test.default", 1391832872, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832872, {\"key17\"=>\"val17\"}], [\"test.default\", 1391832872, {\"key17_\"=>\"val17_\"}]]"}]]
|
361
|
+
)
|
362
|
+
end
|
363
|
+
end
|
364
|
+
end
|
258
365
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-unit-time-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Genki Sugawara
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-02-
|
11
|
+
date: 2014-02-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|