fluent-plugin-redis-counter 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG.md +4 -0
- data/README.md +9 -0
- data/VERSION +1 -1
- data/fluent-plugin-redis-counter.gemspec +2 -2
- data/lib/fluent/plugin/out_redis_counter.rb +77 -52
- data/test/plugin/out_redis_counter.rb +80 -11
- metadata +2 -2
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -28,6 +28,15 @@ fluent-plugin-redis-counter is hosted by [RubyGems.org](https://rubygems.org/).
|
|
28
28
|
count_key foo-status2xx # key-name for Redis
|
29
29
|
count_value 1 # count-up amount(default: 1, negative value is allowed)
|
30
30
|
</pattern>
|
31
|
+
|
32
|
+
# time-dependent redis keyname
|
33
|
+
# for example, "foo-status2xx-%Y-%m-%d" will be formatted to "foo-status2xx-2012-06-21".
|
34
|
+
# rules for placeholder(%Y, etc.) is similar to out_file plugin.
|
35
|
+
<pattern>
|
36
|
+
match_status ^2[0-9][0-9]$
|
37
|
+
count_key_format foo-statux2xx-%d
|
38
|
+
localtime # time-zone(default: localtime, it can be "utc" or "localtime")
|
39
|
+
</pattern>
|
31
40
|
</match>
|
32
41
|
|
33
42
|
# Example
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.1
|
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-redis-counter"
|
6
|
-
s.version = "0.1.
|
6
|
+
s.version = "0.1.1"
|
7
7
|
s.description = "fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys."
|
8
8
|
s.platform = Gem::Platform::RUBY
|
9
9
|
s.authors = ["Buntaro Okada"]
|
10
|
-
s.date = %q{2012-06-
|
10
|
+
s.date = %q{2012-06-21}
|
11
11
|
s.email = "kbinani.bt@gmail.com"
|
12
12
|
s.homepage = "https://github.com/kbinani/fluent-plugin-redis-counter"
|
13
13
|
s.summary = "Redis counter plugin for fluent"
|
@@ -15,32 +15,14 @@ module Fluent
|
|
15
15
|
@port = conf.has_key?('port') ? conf['port'].to_i : 6379
|
16
16
|
@db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
|
17
17
|
@patterns = []
|
18
|
-
conf.elements.select { |
|
19
|
-
|
20
|
-
}.each { |
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
if e.has_key?('count_value')
|
26
|
-
begin
|
27
|
-
count_value = Integer(e['count_value'])
|
28
|
-
rescue
|
29
|
-
raise Fluent::ConfigError, 'invalid "count_value", integer required.'
|
30
|
-
end
|
18
|
+
conf.elements.select { |element|
|
19
|
+
element.name == 'pattern'
|
20
|
+
}.each { |element|
|
21
|
+
begin
|
22
|
+
@patterns << Pattern.new(element)
|
23
|
+
rescue RedisCounterException => e
|
24
|
+
raise Fluent::ConfigError, e.message
|
31
25
|
end
|
32
|
-
matches = {}
|
33
|
-
e.each_key { |key|
|
34
|
-
if key =~ /^match_/
|
35
|
-
name = key['match_'.size .. key.size]
|
36
|
-
matches[name] = Regexp.new(e[key])
|
37
|
-
end
|
38
|
-
}
|
39
|
-
@patterns << {
|
40
|
-
'matches' => matches,
|
41
|
-
'count_key' => e['count_key'],
|
42
|
-
'count_value' => count_value
|
43
|
-
}
|
44
26
|
}
|
45
27
|
end
|
46
28
|
|
@@ -57,7 +39,7 @@ module Fluent
|
|
57
39
|
end
|
58
40
|
|
59
41
|
def format(tag, time, record)
|
60
|
-
record.to_msgpack
|
42
|
+
[tag, time, record].to_msgpack
|
61
43
|
end
|
62
44
|
|
63
45
|
def write(chunk)
|
@@ -65,43 +47,86 @@ module Fluent
|
|
65
47
|
table.default = 0
|
66
48
|
chunk.open { |io|
|
67
49
|
begin
|
68
|
-
MessagePack::Unpacker.new(io).each { |
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
50
|
+
MessagePack::Unpacker.new(io).each { |message|
|
51
|
+
(tag, time, record) = message
|
52
|
+
@patterns.select { |pattern|
|
53
|
+
pattern.is_match?(record)
|
54
|
+
}.each{ |pattern|
|
55
|
+
table[pattern.get_count_key(time)] += pattern.count_value
|
56
|
+
}
|
73
57
|
}
|
74
58
|
rescue EOFError
|
75
59
|
# EOFError always occured when reached end of chunk.
|
76
60
|
end
|
77
61
|
}
|
78
|
-
table.
|
79
|
-
|
80
|
-
|
81
|
-
|
62
|
+
table.each_pair.select { |key, value|
|
63
|
+
value != 0
|
64
|
+
}.each { |key, value|
|
65
|
+
@redis.incrby(key, value)
|
82
66
|
}
|
83
67
|
end
|
84
68
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
69
|
+
class RedisCounterException < Exception
|
70
|
+
end
|
71
|
+
|
72
|
+
class Pattern
|
73
|
+
attr_reader :matches, :count_value
|
74
|
+
|
75
|
+
def initialize(conf_element)
|
76
|
+
if !conf_element.has_key?('count_key') && !conf_element.has_key?('count_key_format')
|
77
|
+
raise RedisCounterException, '"count_key" or "count_key_format" is required.'
|
78
|
+
end
|
79
|
+
if conf_element.has_key?('count_key') && conf_element.has_key?('count_key_format')
|
80
|
+
raise RedisCounterException, 'both "count_key" and "count_key_format" are specified.'
|
81
|
+
end
|
82
|
+
|
83
|
+
if conf_element.has_key?('count_key')
|
84
|
+
@count_key = conf_element['count_key']
|
85
|
+
else
|
86
|
+
if conf_element.has_key?('localtime') && conf_element.has_key?('utc')
|
87
|
+
raise RedisCounterException, 'both "localtime" and "utc" are specified.'
|
88
|
+
end
|
89
|
+
is_localtime = true
|
90
|
+
if conf_element.has_key?('utc')
|
91
|
+
is_localtime = false
|
92
|
+
end
|
93
|
+
@count_key_format = TimeFormatter.new(conf_element['count_key_format'], is_localtime)
|
94
|
+
end
|
95
|
+
|
96
|
+
@count_value = 1
|
97
|
+
if conf_element.has_key?('count_value')
|
98
|
+
begin
|
99
|
+
@count_value = Integer(conf_element['count_value'])
|
100
|
+
rescue
|
101
|
+
raise RedisCounterException, 'invalid "count_value", integer required.'
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
@matches = {}
|
106
|
+
conf_element.each_pair.select { |key, value|
|
107
|
+
key =~ /^match_/
|
108
|
+
}.each { |key, value|
|
109
|
+
name = key['match_'.size .. key.size]
|
110
|
+
@matches[name] = Regexp.new(value)
|
111
|
+
}
|
112
|
+
end
|
113
|
+
|
114
|
+
def is_match?(record)
|
115
|
+
@matches.each_pair{ |key, value|
|
116
|
+
if !record.has_key?(key) || !(record[key] =~ value)
|
117
|
+
return false
|
98
118
|
end
|
99
119
|
}
|
100
|
-
|
101
|
-
|
120
|
+
return true
|
121
|
+
end
|
122
|
+
|
123
|
+
def get_count_key(time)
|
124
|
+
if @count_key_format == nil
|
125
|
+
@count_key
|
126
|
+
else
|
127
|
+
@count_key_format.format(time)
|
102
128
|
end
|
103
|
-
|
104
|
-
return nil
|
129
|
+
end
|
105
130
|
end
|
106
131
|
end
|
107
132
|
end
|
@@ -16,6 +16,7 @@ class RedisCounterTest < Test::Unit::TestCase
|
|
16
16
|
)
|
17
17
|
redis.del("a")
|
18
18
|
redis.del("b")
|
19
|
+
redis.del("foo-2012-06-21")
|
19
20
|
redis.quit
|
20
21
|
end
|
21
22
|
|
@@ -50,15 +51,15 @@ class RedisCounterTest < Test::Unit::TestCase
|
|
50
51
|
assert_equal 1, driver.instance.db_number
|
51
52
|
assert_equal 2, driver.instance.patterns.size
|
52
53
|
|
53
|
-
assert_equal 2, driver.instance.patterns[0]
|
54
|
-
assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0]
|
55
|
-
assert_equal Regexp.new('^https'), driver.instance.patterns[0]
|
56
|
-
assert_equal 'status-normal', driver.instance.patterns[0]
|
57
|
-
assert_equal 1, driver.instance.patterns[0]
|
54
|
+
assert_equal 2, driver.instance.patterns[0].matches.size
|
55
|
+
assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0].matches['status']
|
56
|
+
assert_equal Regexp.new('^https'), driver.instance.patterns[0].matches['url']
|
57
|
+
assert_equal 'status-normal', driver.instance.patterns[0].get_count_key(Time.now.to_i)
|
58
|
+
assert_equal 1, driver.instance.patterns[0].count_value
|
58
59
|
|
59
|
-
assert_equal 0, driver.instance.patterns[1]
|
60
|
-
assert_equal 'foo', driver.instance.patterns[1]
|
61
|
-
assert_equal 2, driver.instance.patterns[1]
|
60
|
+
assert_equal 0, driver.instance.patterns[1].matches.size
|
61
|
+
assert_equal 'foo', driver.instance.patterns[1].get_count_key(Time.now.to_i)
|
62
|
+
assert_equal 2, driver.instance.patterns[1].count_value
|
62
63
|
end
|
63
64
|
|
64
65
|
def test_configure_count_key_required
|
@@ -70,7 +71,58 @@ class RedisCounterTest < Test::Unit::TestCase
|
|
70
71
|
]
|
71
72
|
flunk
|
72
73
|
rescue Fluent::ConfigError => e
|
73
|
-
assert_equal '"count_key" is required.', e.message
|
74
|
+
assert_equal '"count_key" or "count_key_format" is required.', e.message
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_configure_count_key_duplicated
|
79
|
+
begin
|
80
|
+
create_driver %[
|
81
|
+
<pattern>
|
82
|
+
count_key foo
|
83
|
+
count_key_format foo-%Y
|
84
|
+
</pattern>
|
85
|
+
]
|
86
|
+
flunk
|
87
|
+
rescue Fluent::ConfigError => e
|
88
|
+
assert_equal 'both "count_key" and "count_key_format" are specified.', e.message
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def test_configure_count_key_format_utc
|
93
|
+
driver = create_driver %[
|
94
|
+
<pattern>
|
95
|
+
count_key_format foo-%Y-%m-%d-%H-%M-%S
|
96
|
+
utc
|
97
|
+
</pattern>
|
98
|
+
]
|
99
|
+
time = Time.parse('2011-06-21 03:12:01 UTC').to_i
|
100
|
+
assert_equal 'foo-2011-06-21-03-12-01', driver.instance.patterns[0].get_count_key(time)
|
101
|
+
end
|
102
|
+
|
103
|
+
def test_configure_count_key_format_localtime
|
104
|
+
driver = create_driver %[
|
105
|
+
<pattern>
|
106
|
+
count_key_format foo-%Y-%m-%d-%H-%M-%S
|
107
|
+
localtime
|
108
|
+
</pattern>
|
109
|
+
]
|
110
|
+
local_time = Time.parse('2012-06-21 03:12:00').to_i
|
111
|
+
assert_equal 'foo-2012-06-21-03-12-00', driver.instance.patterns[0].get_count_key(local_time)
|
112
|
+
end
|
113
|
+
|
114
|
+
def test_configure_duplicated_timezone
|
115
|
+
begin
|
116
|
+
create_driver %[
|
117
|
+
<pattern>
|
118
|
+
count_key_format foo%Y
|
119
|
+
localtime
|
120
|
+
utc
|
121
|
+
</pattern>
|
122
|
+
]
|
123
|
+
flunk
|
124
|
+
rescue Fluent::ConfigError => e
|
125
|
+
assert_equal 'both "localtime" and "utc" are specified.', e.message
|
74
126
|
end
|
75
127
|
end
|
76
128
|
|
@@ -89,8 +141,9 @@ class RedisCounterTest < Test::Unit::TestCase
|
|
89
141
|
end
|
90
142
|
|
91
143
|
def test_format
|
92
|
-
|
93
|
-
@d.
|
144
|
+
time = Time.parse('2012-06-21 01:55:00 UTC').to_i
|
145
|
+
@d.emit({"a" => 1}, time)
|
146
|
+
@d.expect_format(['test', time, {"a" => 1}].to_msgpack)
|
94
147
|
@d.run
|
95
148
|
end
|
96
149
|
|
@@ -114,4 +167,20 @@ class RedisCounterTest < Test::Unit::TestCase
|
|
114
167
|
assert_nil driver.instance.redis.get("b")
|
115
168
|
end
|
116
169
|
|
170
|
+
def test_write_with_timeformat
|
171
|
+
driver = create_driver %[
|
172
|
+
db_number 1
|
173
|
+
<pattern>
|
174
|
+
match_a ^2[0-9][0-9]$
|
175
|
+
count_key_format foo-%Y-%m-%d
|
176
|
+
count_value 2
|
177
|
+
</pattern>
|
178
|
+
]
|
179
|
+
time = Time.parse('2012-06-21 03:01:00 UTC').to_i
|
180
|
+
driver.emit({"a" => "200"}, time)
|
181
|
+
driver.run
|
182
|
+
|
183
|
+
assert_equal '2', driver.instance.redis.get("foo-2012-06-21")
|
184
|
+
end
|
185
|
+
|
117
186
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-redis-counter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-06-
|
12
|
+
date: 2012-06-21 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|