fluent-plugin-flowcounter 0.3.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/README.md +21 -8
- data/fluent-plugin-flowcounter.gemspec +1 -1
- data/lib/fluent/plugin/out_flowcounter.rb +26 -15
- data/test/plugin/test_out_flowcounter.rb +45 -6
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 416a10332d7ab178acadf8d6894305482d798040
|
4
|
+
data.tar.gz: c6a281878e6a7a9a8ae2cf8449945e8acdd7bbb0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8908f3cab634a1c7b728fd9f469993ba9e4e34194f6d6c6751a40c0d2bcdc1eb0e38317bc1b2f6f72aa489d9693d323690049b93b3c0d894cb552be86c710aac
|
7
|
+
data.tar.gz: e40bc4a1c056d4372df6172e303d65966f1c4d2d0286e444f6d02c2e561a103807962d53f3eb393d6777393cc0a28fdc49e2ca3be9f150516b7f78edd887b54a
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -3,9 +3,9 @@
|
|
3
3
|
Count metrics below about matches. This is a plugin for [Fluentd](http://fluentd.org)
|
4
4
|
|
5
5
|
* Messages per second/minute/hour/day
|
6
|
-
* Bytes per second/minute/hour/day
|
6
|
+
* Bytes per second/minute/hour/day (optional)
|
7
7
|
* Messages per second (average every second/minute/hour/day)
|
8
|
-
* Bytes per second (average every second/minute/hour/day)
|
8
|
+
* Bytes per second (average every second/minute/hour/day) (optional)
|
9
9
|
|
10
10
|
FlowCounterOutput emits messages contains results data, so you can output these message (with 'flowcount' tag by default) to any outputs you want.
|
11
11
|
|
@@ -19,17 +19,21 @@ Or, output result data with for each tags (with `output_style tagged`)
|
|
19
19
|
|
20
20
|
`input_tag_remove_prefix` option available if you want to remove tag prefix from output field names.
|
21
21
|
|
22
|
+
If you want to count only records, omit `count_keys` configuration.
|
23
|
+
|
24
|
+
{"tag":"test", "count":300, "count_rate":5}
|
25
|
+
|
22
26
|
## Configuration
|
23
27
|
|
24
28
|
Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default).
|
25
29
|
|
26
30
|
<match **>
|
27
|
-
type copy
|
31
|
+
@type copy
|
28
32
|
<store>
|
29
33
|
# original output configurations...
|
30
34
|
</store>
|
31
35
|
<store>
|
32
|
-
type flowcounter
|
36
|
+
@type flowcounter
|
33
37
|
count_keys field1,field2
|
34
38
|
</store>
|
35
39
|
</match>
|
@@ -41,12 +45,12 @@ Counts from fields 'field1' and 'field2', per minute(default), aggregates per ta
|
|
41
45
|
Counts from field 'message', per hour, aggregates all tags, output with tag 'fluentd.traffic'.
|
42
46
|
|
43
47
|
<match **>
|
44
|
-
type copy
|
48
|
+
@type copy
|
45
49
|
<store>
|
46
50
|
# original output configurations...
|
47
51
|
</store>
|
48
52
|
<store>
|
49
|
-
type flowcounter
|
53
|
+
@type flowcounter
|
50
54
|
count_keys message
|
51
55
|
unit hour
|
52
56
|
aggregate all
|
@@ -61,17 +65,26 @@ Counts from field 'message', per hour, aggregates all tags, output with tag 'flu
|
|
61
65
|
To count with all fields in messages, specify 'count_keys *'.
|
62
66
|
|
63
67
|
<match target.**>
|
64
|
-
type flowcounter
|
68
|
+
@type flowcounter
|
65
69
|
count_keys *
|
66
70
|
unit hour
|
67
71
|
aggregate all
|
68
72
|
tag fluentd.traffic
|
69
73
|
</match>
|
70
74
|
|
75
|
+
To count records only (without bytes), omit `count_keys` (it runs in better performance.)
|
76
|
+
|
77
|
+
<match target.**>
|
78
|
+
@type flowcounter
|
79
|
+
unit hour
|
80
|
+
aggregate all
|
81
|
+
tag fluentd.traffic
|
82
|
+
</match>
|
83
|
+
|
71
84
|
Use '${hostname}' if you want your hostname in tag.
|
72
85
|
|
73
86
|
<match target.**>
|
74
|
-
type flowcounter
|
87
|
+
@type flowcounter
|
75
88
|
count_keys *
|
76
89
|
tag fluentd.node.${hostname}
|
77
90
|
</match>
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-flowcounter"
|
4
|
-
gem.version = "0.
|
4
|
+
gem.version = "0.4.0"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com"]
|
7
7
|
gem.summary = %q{Fluent plugin to count message flow}
|
@@ -13,13 +13,13 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
13
13
|
define_method("router") { ::Fluent::Engine }
|
14
14
|
end
|
15
15
|
|
16
|
-
config_param :unit, :string, :
|
17
|
-
config_param :aggregate, :string, :
|
18
|
-
config_param :output_style, :string, :
|
19
|
-
config_param :tag, :string, :
|
20
|
-
config_param :input_tag_remove_prefix, :string, :
|
21
|
-
config_param :count_keys, :string
|
22
|
-
config_param :delimiter, :string, :
|
16
|
+
config_param :unit, :string, default: 'minute'
|
17
|
+
config_param :aggregate, :string, default: 'tag'
|
18
|
+
config_param :output_style, :string, default: 'joined'
|
19
|
+
config_param :tag, :string, default: 'flowcount'
|
20
|
+
config_param :input_tag_remove_prefix, :string, default: nil
|
21
|
+
config_param :count_keys, :string, default: nil
|
22
|
+
config_param :delimiter, :string, default: '_'
|
23
23
|
|
24
24
|
include Fluent::Mixin::ConfigPlaceholders
|
25
25
|
|
@@ -66,8 +66,13 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
66
66
|
@removed_prefix_string = @input_tag_remove_prefix + '.'
|
67
67
|
@removed_length = @removed_prefix_string.length
|
68
68
|
end
|
69
|
-
|
70
|
-
|
69
|
+
if @count_keys
|
70
|
+
@count_keys = @count_keys.split(',')
|
71
|
+
@count_all = (@count_keys == ['*'])
|
72
|
+
@count_bytes = true
|
73
|
+
else
|
74
|
+
@count_bytes = false
|
75
|
+
end
|
71
76
|
|
72
77
|
@counts = count_initialized
|
73
78
|
@mutex = Mutex.new
|
@@ -86,7 +91,11 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
86
91
|
|
87
92
|
def count_initialized(keys=nil)
|
88
93
|
if @aggregate == :all
|
89
|
-
|
94
|
+
if @count_bytes
|
95
|
+
{'count' => 0, 'bytes' => 0}
|
96
|
+
else
|
97
|
+
{'count' => 0}
|
98
|
+
end
|
90
99
|
elsif keys
|
91
100
|
values = Array.new(keys.length){|i| 0 }
|
92
101
|
Hash[[keys, values].transpose]
|
@@ -100,11 +109,11 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
100
109
|
b = 'bytes'
|
101
110
|
if @aggregate == :tag
|
102
111
|
c = name + delimiter + 'count'
|
103
|
-
b = name + delimiter + 'bytes'
|
112
|
+
b = name + delimiter + 'bytes' if @count_bytes
|
104
113
|
end
|
105
114
|
@mutex.synchronize {
|
106
115
|
@counts[c] = (@counts[c] || 0) + counts
|
107
|
-
@counts[b] = (@counts[b] || 0) + bytes
|
116
|
+
@counts[b] = (@counts[b] || 0) + bytes if @count_bytes
|
108
117
|
}
|
109
118
|
end
|
110
119
|
|
@@ -127,8 +136,10 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
127
136
|
names.map {|name|
|
128
137
|
counts = {
|
129
138
|
'count' => flushed[name + delimiter + 'count'],
|
130
|
-
'bytes' => flushed[name + delimiter + 'bytes'],
|
131
139
|
}
|
140
|
+
if @count_bytes
|
141
|
+
counts['bytes'] = flushed[name + delimiter + 'bytes']
|
142
|
+
end
|
132
143
|
data = generate_output(counts, step)
|
133
144
|
data['tag'] = name
|
134
145
|
data
|
@@ -175,12 +186,12 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
175
186
|
if @count_all
|
176
187
|
es.each {|time,record|
|
177
188
|
c += 1
|
178
|
-
b += record.to_msgpack.bytesize
|
189
|
+
b += record.to_msgpack.bytesize if @count_bytes
|
179
190
|
}
|
180
191
|
else
|
181
192
|
es.each {|time,record|
|
182
193
|
c += 1
|
183
|
-
b += @count_keys.inject(0){|s,k| s + (record[k] || FOR_MISSING).bytesize}
|
194
|
+
b += @count_keys.inject(0){|s,k| s + (record[k] || FOR_MISSING).bytesize} if @count_bytes
|
184
195
|
}
|
185
196
|
end
|
186
197
|
countup(name, c, b)
|
@@ -4,7 +4,7 @@ class FlowCounterOutputTest < Test::Unit::TestCase
|
|
4
4
|
def setup
|
5
5
|
Fluent::Test.setup
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
CONFIG = %[
|
9
9
|
unit day
|
10
10
|
aggregate tag
|
@@ -18,9 +18,9 @@ count_keys message
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def test_configure
|
21
|
-
|
22
|
-
|
23
|
-
|
21
|
+
d = create_driver('')
|
22
|
+
assert !(d.instance.instance_eval{ @count_bytes })
|
23
|
+
|
24
24
|
assert_raise(Fluent::ConfigError) {
|
25
25
|
d = create_driver %[
|
26
26
|
count_keys message,message2
|
@@ -235,7 +235,7 @@ count_keys message
|
|
235
235
|
], 'test.tag2')
|
236
236
|
time = Time.now.to_i
|
237
237
|
d2.run do
|
238
|
-
60.times do
|
238
|
+
60.times do
|
239
239
|
d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
|
240
240
|
d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
|
241
241
|
d2.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
|
@@ -262,7 +262,7 @@ count_keys message
|
|
262
262
|
], 'test.tag3')
|
263
263
|
time = Time.now.to_i
|
264
264
|
d3.run do
|
265
|
-
60.times do
|
265
|
+
60.times do
|
266
266
|
d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
|
267
267
|
d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
|
268
268
|
d3.emit({'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'})
|
@@ -304,6 +304,45 @@ count_keys message
|
|
304
304
|
assert_equal 15.0, r1[0]['bytes_rate']
|
305
305
|
end
|
306
306
|
|
307
|
+
def test_emit_not_to_count_bytes
|
308
|
+
d1 = create_driver( %[
|
309
|
+
unit day
|
310
|
+
aggregate tag
|
311
|
+
tag flowcount
|
312
|
+
input_tag_remove_prefix test
|
313
|
+
], 'test.tag1')
|
314
|
+
time = Time.parse("2012-01-02 13:14:15").to_i
|
315
|
+
d1.run do
|
316
|
+
3600.times do
|
317
|
+
d1.emit({'message'=> 'a' * 100})
|
318
|
+
d1.emit({'message'=> 'b' * 100})
|
319
|
+
d1.emit({'message'=> 'c' * 100})
|
320
|
+
end
|
321
|
+
end
|
322
|
+
r1 = d1.instance.flush(3600 * 24)
|
323
|
+
assert_equal 3600*3, r1['tag1_count']
|
324
|
+
assert_nil r1['tag1_bytes']
|
325
|
+
assert_equal (300/24.0).floor / 100.0, r1['tag1_count_rate'] # 3 * 3600 / (60 * 60 * 24) as xx.xx
|
326
|
+
assert_nil r1['tag1_bytes_rate']
|
327
|
+
|
328
|
+
d3 = create_driver( %[
|
329
|
+
unit minute
|
330
|
+
aggregate all
|
331
|
+
tag flow
|
332
|
+
], 'test.tag1')
|
333
|
+
time = Time.parse("2012-01-02 13:14:15").to_i
|
334
|
+
d3.run do
|
335
|
+
60.times do
|
336
|
+
d3.emit({'f1'=>'1'*10, 'f2'=>'2'*20, 'f3'=>'3'*10})
|
337
|
+
end
|
338
|
+
end
|
339
|
+
r3 = d3.instance.flush(60)
|
340
|
+
assert_equal 60, r3['count']
|
341
|
+
assert_nil r3['bytes']
|
342
|
+
assert_equal 1.0, r3['count_rate']
|
343
|
+
assert_nil r3['bytes_rate']
|
344
|
+
end
|
345
|
+
|
307
346
|
def test_emit_records_without_specified_field
|
308
347
|
d3 = create_driver( %[
|
309
348
|
unit minute
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-flowcounter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -105,7 +105,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
105
105
|
version: '0'
|
106
106
|
requirements: []
|
107
107
|
rubyforge_project:
|
108
|
-
rubygems_version: 2.
|
108
|
+
rubygems_version: 2.5.1
|
109
109
|
signing_key:
|
110
110
|
specification_version: 4
|
111
111
|
summary: Fluent plugin to count message flow
|