fluent-plugin-flowcounter 0.1.8 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -0
- data/{README.rdoc → README.md} +19 -18
- data/fluent-plugin-flowcounter.gemspec +2 -2
- data/lib/fluent/plugin/out_flowcounter.rb +31 -1
- data/test/helper.rb +0 -1
- data/test/plugin/test_out_flowcounter.rb +40 -0
- metadata +8 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3639330b2bfa7435b5a737363733c630d12a8d6c
|
4
|
+
data.tar.gz: f32013615c744fa27fcd6e2014ef055753c99ec9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4c9b9cbde88a539fac3d12dcc6983f33061b1e9c1e39f36cef2ac8e109726b683a9b343ff2123619f2d72a273741b205547f8025349be80270438b34254e6911
|
7
|
+
data.tar.gz: 526a506b799a7167e88543281fe2f5c5547614b09018a0c8b8ef2a45281a315a5093fdf8e55be231db5a16aaa06f1a158a215d64beb2891d3d5638a5d3cfab99
|
data/.travis.yml
ADDED
data/{README.rdoc → README.md}
RENAMED
@@ -1,26 +1,25 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
== Component
|
4
|
-
|
5
|
-
=== FlowCounterOutput
|
1
|
+
# fluent-plugin-flowcounter
|
6
2
|
|
7
3
|
Count metricses below about matches.
|
8
4
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
5
|
+
* Messages per minute/hour/day
|
6
|
+
* Bytes per minute/hour/day
|
7
|
+
* Messages per second (average every minute/hour/day)
|
8
|
+
* Bytes per second (average every minute/hour/day)
|
13
9
|
|
14
10
|
FlowCounterOutput emits messages contains results data, so you can output these message (with 'flowcount' tag by default) to any outputs you want.
|
15
11
|
|
16
12
|
output ex1 (aggragates all inputs): {"count":300, "bytes":3660, "count_rate":5, "bytes_rate":61}
|
17
13
|
output ex2 (aggragates per tag): {"test_count":300, "test_bytes":3660, "test_count_rate":5, "test_bytes_rate":61, "service1_count":180, "service1_bytes":7260, "service1_count_rate":3, "service1_bytes_rate":121}
|
18
14
|
|
19
|
-
|
15
|
+
Or, output result data with for each tags (with `output_style tagged`)
|
16
|
+
|
17
|
+
{"tag":"test", "count":300, "bytes":3660, "count_rate":5, "bytes_rate":61}
|
18
|
+
{"tag":"service1", "count":180, "bytes":7260, "count_rate":3, "bytes_rate":121}
|
20
19
|
|
21
|
-
|
20
|
+
`input_tag_remove_prefix` option available if you want to remove tag prefix from output field names.
|
22
21
|
|
23
|
-
|
22
|
+
## Configuration
|
24
23
|
|
25
24
|
Counts from fields 'field1' and 'field2', per minute(default), aggregates per tags(default), output with tag 'flowcount'(default).
|
26
25
|
|
@@ -77,12 +76,14 @@ Use '${hostname}' if you want your hostname in tag.
|
|
77
76
|
tag fluentd.node.${hostname}
|
78
77
|
</match>
|
79
78
|
|
80
|
-
|
79
|
+
## TODO
|
81
80
|
|
82
|
-
|
83
|
-
|
81
|
+
* consider what to do next
|
82
|
+
* patches welcome!
|
84
83
|
|
85
|
-
|
84
|
+
## Copyright
|
86
85
|
|
87
|
-
|
88
|
-
|
86
|
+
* Copyright
|
87
|
+
* Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
|
88
|
+
* License
|
89
|
+
* Apache License, Version 2.0
|
@@ -1,12 +1,13 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-flowcounter"
|
4
|
-
gem.version = "0.1.
|
4
|
+
gem.version = "0.1.9"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com"]
|
7
7
|
gem.summary = %q{Fluent plugin to count message flow}
|
8
8
|
gem.description = %q{Plugin to counts messages/bytes that matches, per minutes/hours/days}
|
9
9
|
gem.homepage = "https://github.com/tagomoris/fluent-plugin-flowcounter"
|
10
|
+
gem.license = "APLv2"
|
10
11
|
|
11
12
|
gem.files = `git ls-files`.split($\)
|
12
13
|
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
@@ -14,7 +15,6 @@ Gem::Specification.new do |gem|
|
|
14
15
|
gem.require_paths = ["lib"]
|
15
16
|
|
16
17
|
gem.add_development_dependency "rake"
|
17
|
-
gem.add_development_dependency "shoulda"
|
18
18
|
gem.add_runtime_dependency "fluentd"
|
19
19
|
gem.add_runtime_dependency "fluent-mixin-config-placeholders"
|
20
20
|
end
|
@@ -5,6 +5,7 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
5
5
|
|
6
6
|
config_param :unit, :string, :default => 'minute'
|
7
7
|
config_param :aggregate, :string, :default => 'tag'
|
8
|
+
config_param :output_style, :string, :default => 'joined'
|
8
9
|
config_param :tag, :string, :default => 'flowcount'
|
9
10
|
config_param :input_tag_remove_prefix, :string, :default => nil
|
10
11
|
config_param :count_keys, :string
|
@@ -31,6 +32,15 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
31
32
|
else
|
32
33
|
raise Fluent::ConfigError, "flowcounter aggregate allows tag/all"
|
33
34
|
end
|
35
|
+
@output_style = case @output_style
|
36
|
+
when 'joined' then :joined
|
37
|
+
when 'tagged' then :tagged
|
38
|
+
else
|
39
|
+
raise Fluent::ConfigError, "flowcounter output_style allows joined/tagged"
|
40
|
+
end
|
41
|
+
if @output_style == :tagged and @aggregate != :tag
|
42
|
+
raise Fluent::ConfigError, "flowcounter aggregate must be 'tag' when output_style is 'tagged'"
|
43
|
+
end
|
34
44
|
if @input_tag_remove_prefix
|
35
45
|
@removed_prefix_string = @input_tag_remove_prefix + '.'
|
36
46
|
@removed_length = @removed_prefix_string.length
|
@@ -90,8 +100,28 @@ class Fluent::FlowCounterOutput < Fluent::Output
|
|
90
100
|
generate_output(flushed, step)
|
91
101
|
end
|
92
102
|
|
103
|
+
def tagged_flush(step)
|
104
|
+
flushed,@counts = @counts,count_initialized(@counts.keys)
|
105
|
+
names = flushed.keys.select {|x| x.end_with?('_count')}.map {|x| x.chomp('_count')}
|
106
|
+
names.map {|name|
|
107
|
+
counts = {
|
108
|
+
'count' => flushed[name + '_count'],
|
109
|
+
'bytes' => flushed[name + '_bytes'],
|
110
|
+
}
|
111
|
+
data = generate_output(counts, step)
|
112
|
+
data['tag'] = name
|
113
|
+
data
|
114
|
+
}
|
115
|
+
end
|
116
|
+
|
93
117
|
def flush_emit(step)
|
94
|
-
|
118
|
+
if @output_style == :tagged
|
119
|
+
tagged_flush(step).each do |data|
|
120
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, data)
|
121
|
+
end
|
122
|
+
else
|
123
|
+
Fluent::Engine.emit(@tag, Fluent::Engine.now, flush(step))
|
124
|
+
end
|
95
125
|
end
|
96
126
|
|
97
127
|
def start_watch
|
data/test/helper.rb
CHANGED
@@ -32,6 +32,7 @@ count_keys message
|
|
32
32
|
]
|
33
33
|
assert_equal :minute, d.instance.unit
|
34
34
|
assert_equal :tag, d.instance.aggregate
|
35
|
+
assert_equal :joined, d.instance.output_style
|
35
36
|
assert_equal 'flowcount', d.instance.tag
|
36
37
|
assert_nil d.instance.input_tag_remove_prefix
|
37
38
|
assert_equal ['message'], d.instance.count_keys
|
@@ -41,6 +42,7 @@ count_keys message
|
|
41
42
|
]
|
42
43
|
assert_equal :minute, d.instance.unit
|
43
44
|
assert_equal :tag, d.instance.aggregate
|
45
|
+
assert_equal :joined, d.instance.output_style
|
44
46
|
assert_equal 'flowcount', d.instance.tag
|
45
47
|
assert_nil d.instance.input_tag_remove_prefix
|
46
48
|
assert_equal ['field1', 'field2'], d.instance.count_keys
|
@@ -51,6 +53,18 @@ count_keys message
|
|
51
53
|
]
|
52
54
|
assert_equal :hour, d.instance.unit
|
53
55
|
assert_equal :tag, d.instance.aggregate
|
56
|
+
assert_equal :joined, d.instance.output_style
|
57
|
+
assert_equal 'flowcount', d.instance.tag
|
58
|
+
assert_nil d.instance.input_tag_remove_prefix
|
59
|
+
assert_equal ['message'], d.instance.count_keys
|
60
|
+
|
61
|
+
d = create_driver %[
|
62
|
+
output_style tagged
|
63
|
+
count_keys message
|
64
|
+
]
|
65
|
+
assert_equal :minute, d.instance.unit
|
66
|
+
assert_equal :tag, d.instance.aggregate
|
67
|
+
assert_equal :tagged, d.instance.output_style
|
54
68
|
assert_equal 'flowcount', d.instance.tag
|
55
69
|
assert_nil d.instance.input_tag_remove_prefix
|
56
70
|
assert_equal ['message'], d.instance.count_keys
|
@@ -64,6 +78,7 @@ count_keys message
|
|
64
78
|
]
|
65
79
|
assert_equal :day, d.instance.unit
|
66
80
|
assert_equal :all, d.instance.aggregate
|
81
|
+
assert_equal :joined, d.instance.output_style
|
67
82
|
assert_equal 'test.flowcount', d.instance.tag
|
68
83
|
assert_equal 'test', d.instance.input_tag_remove_prefix
|
69
84
|
assert_equal ['message'], d.instance.count_keys
|
@@ -77,6 +92,7 @@ count_keys message
|
|
77
92
|
]
|
78
93
|
assert_equal :day, d.instance.unit
|
79
94
|
assert_equal :all, d.instance.aggregate
|
95
|
+
assert_equal :joined, d.instance.output_style
|
80
96
|
assert_equal 'test.flowcount', d.instance.tag
|
81
97
|
assert_equal 'test', d.instance.input_tag_remove_prefix
|
82
98
|
assert d.instance.count_all
|
@@ -245,4 +261,28 @@ count_keys message
|
|
245
261
|
msgpack_size = {'f1' => 'abcde', 'f2' => 'vwxyz', 'f3' => '0123456789'}.to_msgpack.bytesize * 5 * 60
|
246
262
|
assert_equal msgpack_size, data[2]['bytes']
|
247
263
|
end
|
264
|
+
|
265
|
+
def test_emit_tagged
|
266
|
+
d1 = create_driver( %[
|
267
|
+
unit minute
|
268
|
+
aggregate tag
|
269
|
+
output_style tagged
|
270
|
+
tag flow
|
271
|
+
input_tag_remove_prefix test
|
272
|
+
count_keys *
|
273
|
+
], 'test.tag1')
|
274
|
+
time = Time.parse("2012-01-02 13:14:15").to_i
|
275
|
+
d1.run do
|
276
|
+
60.times do
|
277
|
+
d1.emit({'message'=> 'hello'})
|
278
|
+
end
|
279
|
+
end
|
280
|
+
r1 = d1.instance.tagged_flush(60)
|
281
|
+
assert_equal 1, r1.length
|
282
|
+
assert_equal 'tag1', r1[0]['tag']
|
283
|
+
assert_equal 60, r1[0]['count']
|
284
|
+
assert_equal 60*15, r1[0]['bytes']
|
285
|
+
assert_equal 1.0, r1[0]['count_rate']
|
286
|
+
assert_equal 15.0, r1[0]['bytes_rate']
|
287
|
+
end
|
248
288
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-flowcounter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-
|
11
|
+
date: 2013-09-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -24,20 +24,6 @@ dependencies:
|
|
24
24
|
- - '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: shoulda
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - '>='
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '0'
|
34
|
-
type: :development
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - '>='
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: fluentd
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -75,17 +61,19 @@ extra_rdoc_files: []
|
|
75
61
|
files:
|
76
62
|
- .document
|
77
63
|
- .gitignore
|
64
|
+
- .travis.yml
|
78
65
|
- AUTHORS
|
79
66
|
- Gemfile
|
80
67
|
- LICENSE.txt
|
81
|
-
- README.
|
68
|
+
- README.md
|
82
69
|
- Rakefile
|
83
70
|
- fluent-plugin-flowcounter.gemspec
|
84
71
|
- lib/fluent/plugin/out_flowcounter.rb
|
85
72
|
- test/helper.rb
|
86
73
|
- test/plugin/test_out_flowcounter.rb
|
87
74
|
homepage: https://github.com/tagomoris/fluent-plugin-flowcounter
|
88
|
-
licenses:
|
75
|
+
licenses:
|
76
|
+
- APLv2
|
89
77
|
metadata: {}
|
90
78
|
post_install_message:
|
91
79
|
rdoc_options: []
|
@@ -103,10 +91,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
103
91
|
version: '0'
|
104
92
|
requirements: []
|
105
93
|
rubyforge_project:
|
106
|
-
rubygems_version: 2.0.
|
94
|
+
rubygems_version: 2.0.3
|
107
95
|
signing_key:
|
108
96
|
specification_version: 4
|
109
97
|
summary: Fluent plugin to count message flow
|
110
98
|
test_files:
|
111
99
|
- test/helper.rb
|
112
100
|
- test/plugin/test_out_flowcounter.rb
|
101
|
+
has_rdoc:
|