fluent-plugin-unit-time-filter 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 180c6e4465e1da97a1646bf9d82b9df431d363cf
4
- data.tar.gz: 74a04b0c568bef90caff5691948abee16ed147cd
3
+ metadata.gz: 6e0bf31be1cb31c459450da1fe3d767a6ed82f3d
4
+ data.tar.gz: 48f82a08b5bd76cbe50f5d930a297d5ba67f74c4
5
5
  SHA512:
6
- metadata.gz: b512af5d84a5593ec90d250e3eb1d4e89a8937a9b909f59bd71e831419c2e2c0454bd32924016a28e6afbf7cb9e986d48576968423ab3e8f3a183802dd8e17da
7
- data.tar.gz: 6de88e2b63dade6980d063724ad88de35e4700b70d773709febde8fc32db77d5d5df34ea3997e8867bf54974d22d779520f1569faa8bb15643d43e1224933271
6
+ metadata.gz: 7dd5dedbbcbfaf1425baddaef19c51856ddfcdc149afcbd44d5b2d144dfa1806745ef72b063a0d6f7acc41763d36579fa7217ddae1a739286f93a6c149476457
7
+ data.tar.gz: 24a6577e6917f9c79f92521030c483f5884e449c47df6ec114c5b76693164a7fd9ab8de49e20a25a9153b6b95316d0a85252862865dde4a33b1edd63aa3f6e77
data/README.md CHANGED
@@ -5,6 +5,10 @@ Plug-in to aggregate by unit time
5
5
  [![Gem Version](https://badge.fury.io/rb/fluent-plugin-unit-time-filter.png)](http://badge.fury.io/rb/fluent-plugin-unit-time-filter)
6
6
  [![Build Status](https://drone.io/bitbucket.org/winebarrel/fluent-plugin-unit-time-filter/status.png)](https://drone.io/bitbucket.org/winebarrel/fluent-plugin-unit-time-filter/latest)
7
7
 
8
+ ## Installation
9
+
10
+ $ gem install fluent-plugin-unit-time-filter
11
+
8
12
  ## Configuration
9
13
 
10
14
  ```
@@ -63,11 +67,3 @@ fluentd outputs the following:
63
67
  2014-02-08 00:39:00 +0900 filtered.my.data: {"count":41}
64
68
  ...
65
69
  ```
66
-
67
- ## Contributing
68
-
69
- 1. Fork it
70
- 2. Create your feature branch (`git checkout -b my-new-feature`)
71
- 3. Commit your changes (`git commit -am 'Add some feature'`)
72
- 4. Push to the branch (`git push origin my-new-feature`)
73
- 5. Create new Pull Request
@@ -1,7 +1,7 @@
1
1
  # coding: utf-8
2
2
  Gem::Specification.new do |spec|
3
3
  spec.name = "fluent-plugin-unit-time-filter"
4
- spec.version = "0.1.1"
4
+ spec.version = "0.1.2"
5
5
  spec.authors = ["Genki Sugawara"]
6
6
  spec.email = ["sgwr_dts@yahoo.co.jp"]
7
7
  spec.description = %q{Plug-in to aggregate by unit time}
@@ -3,6 +3,10 @@ require 'fluent/plugin/unit_time_filter_buffer'
3
3
  class Fluent::UnitTimeFilterOutput < Fluent::Output
4
4
  Fluent::Plugin.register_output('unit_time_filter', self)
5
5
 
6
+ unless method_defined?(:log)
7
+ define_method("log") { $log }
8
+ end
9
+
6
10
  config_param :filter_path, :type => :string
7
11
  config_param :unit_sec, :type => :size, :default => 1
8
12
  config_param :prefix, :type => :string, :default => 'filtered'
@@ -48,6 +52,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
48
52
  :pass_hash_row => @pass_hash_row,
49
53
  :hash_row_time_key => @hash_row_time_key,
50
54
  :hash_row_tag_key => @hash_row_tag_key,
55
+ :log => log,
51
56
  })
52
57
 
53
58
  Thread.current[BUFFER_KEY] = buf
@@ -57,6 +62,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
57
62
  yield(buf)
58
63
  rescue Exception => e
59
64
  Thread.current[BUFFER_KEY] = nil
65
+ log.error(e)
60
66
  raise e
61
67
  end
62
68
  end
@@ -4,6 +4,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
4
4
 
5
5
  def initialize(options)
6
6
  @init_queue = []
7
+ @log = options[:log]
7
8
 
8
9
  @fiber = Fiber.new do |tag, es|
9
10
  @init_queue << [tag, es]
@@ -43,7 +44,7 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
43
44
  prev_time = time
44
45
  result
45
46
  }.each {|records|
46
- records = records.inject([]) {|r, i| r + i[1] }
47
+ records = records.map {|i| i[1] }.inject(:+)
47
48
  records = conv_to_hash_rows_if_needed(records, options)
48
49
 
49
50
  time = get_time(records.first, options)
@@ -64,7 +65,11 @@ class Fluent::UnitTimeFilterOutput < Fluent::Output
64
65
 
65
66
  def emit_records(prefix, tag, time, records)
66
67
  records.each do |record|
67
- Fluent::Engine.emit("#{prefix}.#{tag}", time, record)
68
+ if record.kind_of?(Hash)
69
+ Fluent::Engine.emit("#{prefix}.#{tag}", time, record)
70
+ else
71
+ @log.warn("Record must be Hash: #{record.inspect} (#{record.class})")
72
+ end
68
73
  end
69
74
  end
70
75
 
@@ -235,7 +235,7 @@ describe Fluent::UnitTimeFilterOutput do
235
235
  end
236
236
 
237
237
  describe 'when an error happened' do
238
- it 'filter name should be to be included in the error' do
238
+ it 'filter name should be included in the error' do
239
239
  filter = <<-EOS
240
240
  proc {|rs|
241
241
  raise 'Any error message'
@@ -243,6 +243,8 @@ describe Fluent::UnitTimeFilterOutput do
243
243
  EOS
244
244
 
245
245
  run_driver(:filter => filter, :tempfile => 'any_filter.rb') do |d|
246
+ d.instance.log.should_receive(:error) {|e| expect(e.message).to eq('Any error message') }
247
+
246
248
  begin
247
249
  (0...10).each do |i|
248
250
  d.emit({"key#{i}" => "val#{i}"}, time + i)
@@ -255,4 +257,109 @@ describe Fluent::UnitTimeFilterOutput do
255
257
  end
256
258
  end
257
259
  end
260
+
261
+ describe 'when an invalid record was included' do
262
+ it 'should skip the bad records (Filter returns a Hash)' do
263
+ filter = <<-EOS
264
+ proc {|rs|
265
+ if rs.any? {|i| i[2].has_key?('return_nil')}
266
+ nil
267
+ else
268
+ {
269
+ 'count' => rs.count,
270
+ 'inspect' => rs.inspect
271
+ }
272
+ end
273
+ }
274
+ EOS
275
+
276
+ run_driver(:filter => filter) do |d|
277
+ d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: nil (NilClass)') }
278
+
279
+ (0...10).each do |i|
280
+ d.emit({"key#{i}" => "val#{i}"}, time + i)
281
+ d.emit({"key#{i}_" => "val#{i}_"}, time + i)
282
+ end
283
+
284
+ d.emit({"return_nil" => 1}, time + 10)
285
+
286
+ (10...20).each do |i|
287
+ d.emit({"key#{i}" => "val#{i}"}, time + i)
288
+ d.emit({"key#{i}_" => "val#{i}_"}, time + i)
289
+ end
290
+
291
+ expect(d.emits).to eq(
292
+ [["filtered.test.default", 1391832855, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832855, {\"key0\"=>\"val0\"}], [\"test.default\", 1391832855, {\"key0_\"=>\"val0_\"}]]"}],
293
+ ["filtered.test.default", 1391832856, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832856, {\"key1\"=>\"val1\"}], [\"test.default\", 1391832856, {\"key1_\"=>\"val1_\"}]]"}],
294
+ ["filtered.test.default", 1391832857, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832857, {\"key2\"=>\"val2\"}], [\"test.default\", 1391832857, {\"key2_\"=>\"val2_\"}]]"}],
295
+ ["filtered.test.default", 1391832858, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832858, {\"key3\"=>\"val3\"}], [\"test.default\", 1391832858, {\"key3_\"=>\"val3_\"}]]"}],
296
+ ["filtered.test.default", 1391832859, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832859, {\"key4\"=>\"val4\"}], [\"test.default\", 1391832859, {\"key4_\"=>\"val4_\"}]]"}],
297
+ ["filtered.test.default", 1391832860, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832860, {\"key5\"=>\"val5\"}], [\"test.default\", 1391832860, {\"key5_\"=>\"val5_\"}]]"}],
298
+ ["filtered.test.default", 1391832861, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832861, {\"key6\"=>\"val6\"}], [\"test.default\", 1391832861, {\"key6_\"=>\"val6_\"}]]"}],
299
+ ["filtered.test.default", 1391832862, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832862, {\"key7\"=>\"val7\"}], [\"test.default\", 1391832862, {\"key7_\"=>\"val7_\"}]]"}],
300
+ ["filtered.test.default", 1391832863, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832863, {\"key8\"=>\"val8\"}], [\"test.default\", 1391832863, {\"key8_\"=>\"val8_\"}]]"}],
301
+ ["filtered.test.default", 1391832864, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832864, {\"key9\"=>\"val9\"}], [\"test.default\", 1391832864, {\"key9_\"=>\"val9_\"}]]"}],
302
+ ["filtered.test.default", 1391832866, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832866, {\"key11\"=>\"val11\"}], [\"test.default\", 1391832866, {\"key11_\"=>\"val11_\"}]]"}],
303
+ ["filtered.test.default", 1391832867, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832867, {\"key12\"=>\"val12\"}], [\"test.default\", 1391832867, {\"key12_\"=>\"val12_\"}]]"}],
304
+ ["filtered.test.default", 1391832868, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832868, {\"key13\"=>\"val13\"}], [\"test.default\", 1391832868, {\"key13_\"=>\"val13_\"}]]"}],
305
+ ["filtered.test.default", 1391832869, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832869, {\"key14\"=>\"val14\"}], [\"test.default\", 1391832869, {\"key14_\"=>\"val14_\"}]]"}],
306
+ ["filtered.test.default", 1391832870, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832870, {\"key15\"=>\"val15\"}], [\"test.default\", 1391832870, {\"key15_\"=>\"val15_\"}]]"}],
307
+ ["filtered.test.default", 1391832871, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832871, {\"key16\"=>\"val16\"}], [\"test.default\", 1391832871, {\"key16_\"=>\"val16_\"}]]"}],
308
+ ["filtered.test.default", 1391832872, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832872, {\"key17\"=>\"val17\"}], [\"test.default\", 1391832872, {\"key17_\"=>\"val17_\"}]]"}]]
309
+ )
310
+ end
311
+ end
312
+
313
+ it 'should skip the bad records (Filter returns an Array)' do
314
+ filter = <<-EOS
315
+ proc {|rs|
316
+ if rs.any? {|i| i[2].has_key?('return_nil')}
317
+ [nil, 1]
318
+ else
319
+ {
320
+ 'count' => rs.count,
321
+ 'inspect' => rs.inspect
322
+ }
323
+ end
324
+ }
325
+ EOS
326
+
327
+ run_driver(:filter => filter) do |d|
328
+ d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: nil (NilClass)') }
329
+ d.instance.log.should_receive(:warn) {|msg| expect(msg).to eq('Record must be Hash: 1 (Fixnum)') }
330
+
331
+ (0...10).each do |i|
332
+ d.emit({"key#{i}" => "val#{i}"}, time + i)
333
+ d.emit({"key#{i}_" => "val#{i}_"}, time + i)
334
+ end
335
+
336
+ d.emit({"return_nil" => 1}, time + 10)
337
+
338
+ (10...20).each do |i|
339
+ d.emit({"key#{i}" => "val#{i}"}, time + i)
340
+ d.emit({"key#{i}_" => "val#{i}_"}, time + i)
341
+ end
342
+
343
+ expect(d.emits).to eq(
344
+ [["filtered.test.default", 1391832855, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832855, {\"key0\"=>\"val0\"}], [\"test.default\", 1391832855, {\"key0_\"=>\"val0_\"}]]"}],
345
+ ["filtered.test.default", 1391832856, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832856, {\"key1\"=>\"val1\"}], [\"test.default\", 1391832856, {\"key1_\"=>\"val1_\"}]]"}],
346
+ ["filtered.test.default", 1391832857, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832857, {\"key2\"=>\"val2\"}], [\"test.default\", 1391832857, {\"key2_\"=>\"val2_\"}]]"}],
347
+ ["filtered.test.default", 1391832858, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832858, {\"key3\"=>\"val3\"}], [\"test.default\", 1391832858, {\"key3_\"=>\"val3_\"}]]"}],
348
+ ["filtered.test.default", 1391832859, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832859, {\"key4\"=>\"val4\"}], [\"test.default\", 1391832859, {\"key4_\"=>\"val4_\"}]]"}],
349
+ ["filtered.test.default", 1391832860, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832860, {\"key5\"=>\"val5\"}], [\"test.default\", 1391832860, {\"key5_\"=>\"val5_\"}]]"}],
350
+ ["filtered.test.default", 1391832861, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832861, {\"key6\"=>\"val6\"}], [\"test.default\", 1391832861, {\"key6_\"=>\"val6_\"}]]"}],
351
+ ["filtered.test.default", 1391832862, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832862, {\"key7\"=>\"val7\"}], [\"test.default\", 1391832862, {\"key7_\"=>\"val7_\"}]]"}],
352
+ ["filtered.test.default", 1391832863, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832863, {\"key8\"=>\"val8\"}], [\"test.default\", 1391832863, {\"key8_\"=>\"val8_\"}]]"}],
353
+ ["filtered.test.default", 1391832864, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832864, {\"key9\"=>\"val9\"}], [\"test.default\", 1391832864, {\"key9_\"=>\"val9_\"}]]"}],
354
+ ["filtered.test.default", 1391832866, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832866, {\"key11\"=>\"val11\"}], [\"test.default\", 1391832866, {\"key11_\"=>\"val11_\"}]]"}],
355
+ ["filtered.test.default", 1391832867, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832867, {\"key12\"=>\"val12\"}], [\"test.default\", 1391832867, {\"key12_\"=>\"val12_\"}]]"}],
356
+ ["filtered.test.default", 1391832868, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832868, {\"key13\"=>\"val13\"}], [\"test.default\", 1391832868, {\"key13_\"=>\"val13_\"}]]"}],
357
+ ["filtered.test.default", 1391832869, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832869, {\"key14\"=>\"val14\"}], [\"test.default\", 1391832869, {\"key14_\"=>\"val14_\"}]]"}],
358
+ ["filtered.test.default", 1391832870, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832870, {\"key15\"=>\"val15\"}], [\"test.default\", 1391832870, {\"key15_\"=>\"val15_\"}]]"}],
359
+ ["filtered.test.default", 1391832871, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832871, {\"key16\"=>\"val16\"}], [\"test.default\", 1391832871, {\"key16_\"=>\"val16_\"}]]"}],
360
+ ["filtered.test.default", 1391832872, {"count"=>2, "inspect"=>"[[\"test.default\", 1391832872, {\"key17\"=>\"val17\"}], [\"test.default\", 1391832872, {\"key17_\"=>\"val17_\"}]]"}]]
361
+ )
362
+ end
363
+ end
364
+ end
258
365
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-unit-time-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Genki Sugawara
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-02-09 00:00:00.000000000 Z
11
+ date: 2014-02-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd