fluent-plugin-flowcounter 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 373e8131d63b116d0e7bb4364fd93a19356bb141
4
- data.tar.gz: 2000acaab8a219b70da435c4f77469155cbdd300
3
+ metadata.gz: 416a10332d7ab178acadf8d6894305482d798040
4
+ data.tar.gz: c6a281878e6a7a9a8ae2cf8449945e8acdd7bbb0
5
5
  SHA512:
6
- metadata.gz: 715cd4f1dfeb133f64f334608df853f8efc2cf0debd31a87f63642dd0b82cf4c774126935e821a345bce8f348e8aea71a56409646b93c35f6cf51edb72df4a8a
7
- data.tar.gz: 96e775ec5a88dc718a21676fe630236b50b7eba183a3bfeac47a0e66883292dd3b29b9ca16a261673eaf14340db56fd5ec48aa1f36341e1f3851568b8d0c5339
6
+ metadata.gz: 8908f3cab634a1c7b728fd9f469993ba9e4e34194f6d6c6751a40c0d2bcdc1eb0e38317bc1b2f6f72aa489d9693d323690049b93b3c0d894cb552be86c710aac
7
+ data.tar.gz: e40bc4a1c056d4372df6172e303d65966f1c4d2d0286e444f6d02c2e561a103807962d53f3eb393d6777393cc0a28fdc49e2ca3be9f150516b7f78edd887b54a
@@ -4,3 +4,4 @@ rvm:
4
4
  - 2.0.0
5
5
  - 2.1
6
6
  - 2.2
7
+ - 2.3.0
data/README.md CHANGED
@@ -3,9 +3,9 @@
3
3
  Count metrics below about matches. This is a plugin for [Fluentd](http://fluentd.org)
4
4
 
5
5
  * Messages per second/minute/hour/day
6
- * Bytes per second/minute/hour/day
6
+ * Bytes per second/minute/hour/day (optional)
7
7
  * Messages per second (average every second/minute/hour/day)
8
- * Bytes per second (average every second/minute/hour/day)
8
+ * Bytes per second (average every second/minute/hour/day) (optional)
9
9
 
10
10
  FlowCounterOutput emits messages contains results data, so you can output these message (with 'flowcount' tag by default) to any outputs you want.
11
11
 
@@ -19,17 +19,21 @@ Or, output result data with for each tags (with `output_style tagged`)
19
19
 
20
20
  `input_tag_remove_prefix` option available if you want to remove tag prefix from output field names.
21
21
 
22
+ If you want to count only records, omit `count_keys` configuration.
23
+
24
+ {"tag":"test", "count":300, "count_rate":5}
25
+
22
26
  ## Configuration
23
27
 
24
28
  Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default).
25
29
 
26
30
  <match **>
27
- type copy
31
+ @type copy
28
32
  <store>
29
33
  # original output configurations...
30
34
  </store>
31
35
  <store>
32
- type flowcounter
36
+ @type flowcounter
33
37
  count_keys field1,field2
34
38
  </store>
35
39
  </match>
@@ -41,12 +45,12 @@ Counts from fields 'field1' and 'field2', per minute(default), aggregates per ta
41
45
  Counts from field 'message', per hour, aggregates all tags, output with tag 'fluentd.traffic'.
42
46
 
43
47
  <match **>
44
- type copy
48
+ @type copy
45
49
  <store>
46
50
  # original output configurations...
47
51
  </store>
48
52
  <store>
49
- type flowcounter
53
+ @type flowcounter
50
54
  count_keys message
51
55
  unit hour
52
56
  aggregate all
@@ -61,17 +65,26 @@ Counts from field 'message', per hour, aggregates all tags, output with tag 'flu
61
65
  To count with all fields in messages, specify 'count_keys *'.
62
66
 
63
67
  <match target.**>
64
- type flowcounter
68
+ @type flowcounter
65
69
  count_keys *
66
70
  unit hour
67
71
  aggregate all
68
72
  tag fluentd.traffic
69
73
  </match>
70
74
 
75
+ To count records only (without bytes), omit `count_keys` (it runs in better performance.)
76
+
77
+ <match target.**>
78
+ @type flowcounter
79
+ unit hour
80
+ aggregate all
81
+ tag fluentd.traffic
82
+ </match>
83
+
71
84
  Use '${hostname}' if you want your hostname in tag.
72
85
 
73
86
  <match target.**>
74
- type flowcounter
87
+ @type flowcounter
75
88
  count_keys *
76
89
  tag fluentd.node.${hostname}
77
90
  </match>
@@ -1,7 +1,7 @@
1
1
  # -*- encoding: utf-8 -*-
2
2
  Gem::Specification.new do |gem|
3
3
  gem.name = "fluent-plugin-flowcounter"
4
- gem.version = "0.3.2"
4
+ gem.version = "0.4.0"
5
5
  gem.authors = ["TAGOMORI Satoshi"]
6
6
  gem.email = ["tagomoris@gmail.com"]
7
7
  gem.summary = %q{Fluent plugin to count message flow}
@@ -13,13 +13,13 @@ class Fluent::FlowCounterOutput < Fluent::Output
13
13
  define_method("router") { ::Fluent::Engine }
14
14
  end
15
15
 
16
- config_param :unit, :string, :default => 'minute'
17
- config_param :aggregate, :string, :default => 'tag'
18
- config_param :output_style, :string, :default => 'joined'
19
- config_param :tag, :string, :default => 'flowcount'
20
- config_param :input_tag_remove_prefix, :string, :default => nil
21
- config_param :count_keys, :string
22
- config_param :delimiter, :string, :default => '_'
16
+ config_param :unit, :string, default: 'minute'
17
+ config_param :aggregate, :string, default: 'tag'
18
+ config_param :output_style, :string, default: 'joined'
19
+ config_param :tag, :string, default: 'flowcount'
20
+ config_param :input_tag_remove_prefix, :string, default: nil
21
+ config_param :count_keys, :string, default: nil
22
+ config_param :delimiter, :string, default: '_'
23
23
 
24
24
  include Fluent::Mixin::ConfigPlaceholders
25
25
 
@@ -66,8 +66,13 @@ class Fluent::FlowCounterOutput < Fluent::Output
66
66
  @removed_prefix_string = @input_tag_remove_prefix + '.'
67
67
  @removed_length = @removed_prefix_string.length
68
68
  end
69
- @count_keys = @count_keys.split(',')
70
- @count_all = (@count_keys == ['*'])
69
+ if @count_keys
70
+ @count_keys = @count_keys.split(',')
71
+ @count_all = (@count_keys == ['*'])
72
+ @count_bytes = true
73
+ else
74
+ @count_bytes = false
75
+ end
71
76
 
72
77
  @counts = count_initialized
73
78
  @mutex = Mutex.new
@@ -86,7 +91,11 @@ class Fluent::FlowCounterOutput < Fluent::Output
86
91
 
87
92
  def count_initialized(keys=nil)
88
93
  if @aggregate == :all
89
- {'count' => 0, 'bytes' => 0}
94
+ if @count_bytes
95
+ {'count' => 0, 'bytes' => 0}
96
+ else
97
+ {'count' => 0}
98
+ end
90
99
  elsif keys
91
100
  values = Array.new(keys.length){|i| 0 }
92
101
  Hash[[keys, values].transpose]
@@ -100,11 +109,11 @@ class Fluent::FlowCounterOutput < Fluent::Output
100
109
  b = 'bytes'
101
110
  if @aggregate == :tag
102
111
  c = name + delimiter + 'count'
103
- b = name + delimiter + 'bytes'
112
+ b = name + delimiter + 'bytes' if @count_bytes
104
113
  end
105
114
  @mutex.synchronize {
106
115
  @counts[c] = (@counts[c] || 0) + counts
107
- @counts[b] = (@counts[b] || 0) + bytes
116
+ @counts[b] = (@counts[b] || 0) + bytes if @count_bytes
108
117
  }
109
118
  end
110
119
 
@@ -127,8 +136,10 @@ class Fluent::FlowCounterOutput < Fluent::Output
127
136
  names.map {|name|
128
137
  counts = {
129
138
  'count' => flushed[name + delimiter + 'count'],
130
- 'bytes' => flushed[name + delimiter + 'bytes'],
131
139
  }
140
+ if @count_bytes
141
+ counts['bytes'] = flushed[name + delimiter + 'bytes']
142
+ end
132
143
  data = generate_output(counts, step)
133
144
  data['tag'] = name
134
145
  data
@@ -175,12 +186,12 @@ class Fluent::FlowCounterOutput < Fluent::Output
175
186
  if @count_all
176
187
  es.each {|time,record|
177
188
  c += 1
178
- b += record.to_msgpack.bytesize
189
+ b += record.to_msgpack.bytesize if @count_bytes
179
190
  }
180
191
  else
181
192
  es.each {|time,record|
182
193
  c += 1
183
- b += @count_keys.inject(0){|s,k| s + (record[k] || FOR_MISSING).bytesize}
194
+ b += @count_keys.inject(0){|s,k| s + (record[k] || FOR_MISSING).bytesize} if @count_bytes
184
195
  }
185
196
  end
186
197
  countup(name, c, b)
@@ -4,7 +4,7 @@ class FlowCounterOutputTest < Test::Unit::TestCase
4
4
  def setup
5
5
  Fluent::Test.setup
6
6
  end
7
-
7
+
8
8
  CONFIG = %[
9
9
  unit day
10
10
  aggregate tag
@@ -18,9 +18,9 @@ count_keys message
18
18
  end
19
19
 
20
20
  def test_configure
21
- assert_raise(Fluent::ConfigError) {
22
- d = create_driver('')
23
- }
21
+ d = create_driver('')
22
+ assert !(d.instance.instance_eval{ @count_bytes })
23
+
24
24
  assert_raise(Fluent::ConfigError) {
25
25
  d = create_driver %[
26
26
  count_keys message,message2
@@ -235,7 +235,7 @@ count_keys message
235
235
  ], 'test.tag2')
236
236
  time = Time.now.to_i
237
237
  d2.run do
238
- 60.times do
238
+ 60.times do
239
239
  d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
240
240
  d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
241
241
  d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
@@ -262,7 +262,7 @@ count_keys message
262
262
  ], 'test.tag3')
263
263
  time = Time.now.to_i
264
264
  d3.run do
265
- 60.times do
265
+ 60.times do
266
266
  d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
267
267
  d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
268
268
  d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
@@ -304,6 +304,45 @@ count_keys message
304
304
  assert_equal 15.0, r1[0]['bytes_rate']
305
305
  end
306
306
 
307
+ def test_emit_not_to_count_bytes
308
+ d1 = create_driver( %[
309
+ unit day
310
+ aggregate tag
311
+ tag flowcount
312
+ input_tag_remove_prefix test
313
+ ], 'test.tag1')
314
+ time = Time.parse("2012-01-02 13:14:15").to_i
315
+ d1.run do
316
+ 3600.times do
317
+ d1.emit({'message'=> 'a' * 100})
318
+ d1.emit({'message'=> 'b' * 100})
319
+ d1.emit({'message'=> 'c' * 100})
320
+ end
321
+ end
322
+ r1 = d1.instance.flush(3600 * 24)
323
+ assert_equal 3600*3, r1['tag1_count']
324
+ assert_nil r1['tag1_bytes']
325
+ assert_equal (300/24.0).floor / 100.0, r1['tag1_count_rate'] # 3 * 3600 / (60 * 60 * 24) as xx.xx
326
+ assert_nil r1['tag1_bytes_rate']
327
+
328
+ d3 = create_driver( %[
329
+ unit minute
330
+ aggregate all
331
+ tag flow
332
+ ], 'test.tag1')
333
+ time = Time.parse("2012-01-02 13:14:15").to_i
334
+ d3.run do
335
+ 60.times do
336
+ d3.emit({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
337
+ end
338
+ end
339
+ r3 = d3.instance.flush(60)
340
+ assert_equal 60, r3['count']
341
+ assert_nil r3['bytes']
342
+ assert_equal 1.0, r3['count_rate']
343
+ assert_nil r3['bytes_rate']
344
+ end
345
+
307
346
  def test_emit_records_without_specified_field
308
347
  d3 = create_driver( %[
309
348
  unit minute
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-flowcounter
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - TAGOMORI Satoshi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-31 00:00:00.000000000 Z
11
+ date: 2016-03-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -105,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
105
105
  version: '0'
106
106
  requirements: []
107
107
  rubyforge_project:
108
- rubygems_version: 2.4.5
108
+ rubygems_version: 2.5.1
109
109
  signing_key:
110
110
  specification_version: 4
111
111
  summary: Fluent plugin to count message flow