fluent-plugin-datacounter 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +3 -3
- data/fluent-plugin-datacounter.gemspec +1 -1
- data/lib/fluent/plugin/out_datacounter.rb +9 -2
- data/test/plugin/test_out_datacounter.rb +39 -0
- metadata +17 -7
data/README.rdoc
CHANGED
@@ -28,7 +28,7 @@ Count messages that have attribute 'referer' as 'google.com', 'yahoo.com' and 'f
|
|
28
28
|
unit minute
|
29
29
|
aggregate all
|
30
30
|
count_key referer
|
31
|
-
# patternX: X(1-
|
31
|
+
# patternX: X(1-20)
|
32
32
|
pattern1 google google.com
|
33
33
|
pattern2 yahoo yahoo.com
|
34
34
|
pattern3 facebook facebook.com
|
@@ -41,7 +41,7 @@ Or, more exact match pattern, output per tags (default 'aggregate tag'), per hou
|
|
41
41
|
type datacounter
|
42
42
|
unit hour
|
43
43
|
count_key referer
|
44
|
-
# patternX: X(1-
|
44
|
+
# patternX: X(1-20)
|
45
45
|
pattern1 google ^http://www\.google\.com/.*
|
46
46
|
pattern2 yahoo ^http://www\.yahoo\.com/.*
|
47
47
|
pattern3 twitter ^https://twitter.com/.*
|
@@ -54,7 +54,7 @@ HTTP status code patterns.
|
|
54
54
|
count_interval 1m # just same as 'unit minute' and 'count_interval 60s'
|
55
55
|
# you can also specify '30s', '5m', '2h' ....
|
56
56
|
count_key status
|
57
|
-
# patternX: X(1-
|
57
|
+
# patternX: X(1-20)
|
58
58
|
pattern1 2xx ^2\d\d$
|
59
59
|
pattern2 3xx ^3\d\d$
|
60
60
|
pattern3 404 ^404$ # we want only 404 counts...
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-datacounter"
|
6
|
-
s.version = "0.2.
|
6
|
+
s.version = "0.2.1"
|
7
7
|
s.authors = ["TAGOMORI Satoshi"]
|
8
8
|
s.email = ["tagomoris@gmail.com"]
|
9
9
|
s.homepage = "https://github.com/tagomoris/fluent-plugin-datacounter"
|
@@ -1,6 +1,8 @@
|
|
1
1
|
class Fluent::DataCounterOutput < Fluent::Output
|
2
2
|
Fluent::Plugin.register_output('datacounter', self)
|
3
3
|
|
4
|
+
PATTERN_MAX_NUM = 20
|
5
|
+
|
4
6
|
config_param :count_interval, :time, :default => nil
|
5
7
|
config_param :unit, :string, :default => 'minute'
|
6
8
|
config_param :aggregate, :string, :default => 'tag'
|
@@ -10,7 +12,7 @@ class Fluent::DataCounterOutput < Fluent::Output
|
|
10
12
|
|
11
13
|
# pattern0 reserved as unmatched counts
|
12
14
|
config_param :pattern1, :string # string: NAME REGEXP
|
13
|
-
(2..
|
15
|
+
(2..PATTERN_MAX_NUM).each do |i|
|
14
16
|
config_param ('pattern' + i.to_s).to_sym, :string, :default => nil # NAME REGEXP
|
15
17
|
end
|
16
18
|
|
@@ -42,7 +44,12 @@ class Fluent::DataCounterOutput < Fluent::Output
|
|
42
44
|
|
43
45
|
@patterns = [[0, 'unmatched', nil]]
|
44
46
|
pattern_names = ['unmatched']
|
45
|
-
|
47
|
+
|
48
|
+
invalids = conf.keys.select{|k| k =~ /^pattern(\d+)$/}.select{|arg| arg =~ /^pattern(\d+)/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
|
49
|
+
if invalids.size > 0
|
50
|
+
$log.warn "invalid number patterns (valid pattern number:1-20):" + invalids.join(",")
|
51
|
+
end
|
52
|
+
(1..PATTERN_MAX_NUM).each do |i|
|
46
53
|
next unless conf["pattern#{i}"]
|
47
54
|
name,regexp = conf["pattern#{i}"].split(' ', 2)
|
48
55
|
@patterns.push([i, name, Regexp.new(regexp)])
|
@@ -177,6 +177,45 @@ class DataCounterOutputTest < Test::Unit::TestCase
|
|
177
177
|
assert_equal 80.0, r2['hoge_percentage']
|
178
178
|
end
|
179
179
|
|
180
|
+
def test_pattern_num
|
181
|
+
assert_equal 20, Fluent::DataCounterOutput::PATTERN_MAX_NUM
|
182
|
+
|
183
|
+
conf = %[
|
184
|
+
aggregate all
|
185
|
+
count_key field
|
186
|
+
]
|
187
|
+
(1..20).each do |i|
|
188
|
+
conf += "pattern#{i} name#{i} ^#{i}$\n"
|
189
|
+
end
|
190
|
+
d = create_driver(conf, 'test.max')
|
191
|
+
d.run do
|
192
|
+
(1..20).each do |j|
|
193
|
+
d.emit({'field' => j})
|
194
|
+
end
|
195
|
+
end
|
196
|
+
r = d.instance.flush(60)
|
197
|
+
assert_equal 1, r['name1_count']
|
198
|
+
assert_equal 1, r['name2_count']
|
199
|
+
assert_equal 1, r['name3_count']
|
200
|
+
assert_equal 1, r['name4_count']
|
201
|
+
assert_equal 1, r['name5_count']
|
202
|
+
assert_equal 1, r['name6_count']
|
203
|
+
assert_equal 1, r['name7_count']
|
204
|
+
assert_equal 1, r['name8_count']
|
205
|
+
assert_equal 1, r['name9_count']
|
206
|
+
assert_equal 1, r['name10_count']
|
207
|
+
assert_equal 1, r['name11_count']
|
208
|
+
assert_equal 1, r['name12_count']
|
209
|
+
assert_equal 1, r['name13_count']
|
210
|
+
assert_equal 1, r['name14_count']
|
211
|
+
assert_equal 1, r['name15_count']
|
212
|
+
assert_equal 1, r['name16_count']
|
213
|
+
assert_equal 1, r['name17_count']
|
214
|
+
assert_equal 1, r['name18_count']
|
215
|
+
assert_equal 1, r['name19_count']
|
216
|
+
assert_equal 1, r['name20_count']
|
217
|
+
end
|
218
|
+
|
180
219
|
def test_emit
|
181
220
|
d1 = create_driver(CONFIG, 'test.tag1')
|
182
221
|
d1.run do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-datacounter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-03-23 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
16
|
-
requirement:
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,15 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :development
|
23
23
|
prerelease: false
|
24
|
-
version_requirements:
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
25
30
|
- !ruby/object:Gem::Dependency
|
26
31
|
name: fluentd
|
27
|
-
requirement:
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
28
33
|
none: false
|
29
34
|
requirements:
|
30
35
|
- - ! '>='
|
@@ -32,7 +37,12 @@ dependencies:
|
|
32
37
|
version: '0'
|
33
38
|
type: :runtime
|
34
39
|
prerelease: false
|
35
|
-
version_requirements:
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
36
46
|
description: Output filter plugin to count messages that matches specified conditions
|
37
47
|
email:
|
38
48
|
- tagomoris@gmail.com
|
@@ -70,7 +80,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
70
80
|
version: '0'
|
71
81
|
requirements: []
|
72
82
|
rubyforge_project: fluent-plugin-datacounter
|
73
|
-
rubygems_version: 1.8.
|
83
|
+
rubygems_version: 1.8.19
|
74
84
|
signing_key:
|
75
85
|
specification_version: 3
|
76
86
|
summary: Output filter plugin to count messages that matches specified conditions
|