fluent-plugin-redis-counter 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.travis.yml +6 -0
- data/CHANGELOG.md +3 -0
- data/README.md +28 -11
- data/VERSION +1 -1
- data/fluent-plugin-redis-counter.gemspec +3 -3
- data/lib/fluent/plugin/out_redis_counter.rb +58 -14
- data/test/plugin/out_redis_counter.rb +73 -21
- metadata +4 -4
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -2,6 +2,12 @@
|
|
2
2
|
|
3
3
|
fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys.
|
4
4
|
|
5
|
+
# Installation
|
6
|
+
|
7
|
+
fluent-plugin-redis-counter is hosted by [RubyGems.org](https://rubygems.org/).
|
8
|
+
|
9
|
+
$fluent-gem install fluent-plugin-redis-counter
|
10
|
+
|
5
11
|
# Configuration
|
6
12
|
|
7
13
|
<match redis_counter.**>
|
@@ -12,6 +18,16 @@ fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys.
|
|
12
18
|
|
13
19
|
# database number is optional.
|
14
20
|
db_number 0 # 0 is default
|
21
|
+
|
22
|
+
# match condition
|
23
|
+
# this pattern matches {"status": "200", "url": "http://foo.example.com"} and then,
|
24
|
+
# increment Redis key "foo-status2xx" by calling Redis::incrby( "foo-status2xx", 1 ).
|
25
|
+
<pattern>
|
26
|
+
match_status ^2[0-9][0-9]$ # matches with {"status": "200", ...
|
27
|
+
match_url ^http:\/\/foo\. # matches with {"url": "http://foo.example.com", ...
|
28
|
+
count_key foo-status2xx # key-name for Redis
|
29
|
+
count_value 1 # count-up amount(default: 1, negative value is allowed)
|
30
|
+
</pattern>
|
15
31
|
</match>
|
16
32
|
|
17
33
|
# Example
|
@@ -26,29 +42,30 @@ prepare a conf file ("fluent.conf") in current directory like this:
|
|
26
42
|
host localhost
|
27
43
|
port 6379
|
28
44
|
db_number 0
|
45
|
+
<pattern>
|
46
|
+
match_status ^2[0-9][0-9]$
|
47
|
+
match_url ^http:\/\/foo\.
|
48
|
+
count_key foo
|
49
|
+
</pattern>
|
29
50
|
</match>
|
30
51
|
|
31
52
|
run commands for test:
|
32
53
|
|
33
54
|
$redis-server 2>&1 >/dev/null &
|
34
55
|
[1] 879
|
35
|
-
$redis-cli
|
36
|
-
redis 127.0.0.1:6379>del foo
|
56
|
+
$echo del foo | redis-cli -h localhost -p 6379 -n 0
|
37
57
|
(integer) 0
|
38
|
-
redis 127.0.0.1:6379>exit
|
39
58
|
$fluentd -c ./fluent.conf 2>&1 >/dev/null &
|
40
59
|
[2] 889
|
41
|
-
$echo {\"foo\"
|
42
|
-
$echo {\"foo\"
|
60
|
+
$echo {\"status\": \"200\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
|
61
|
+
$echo {\"status\": \"500\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
|
43
62
|
$kill -s HUP 889
|
44
|
-
$redis-cli
|
45
|
-
|
46
|
-
"3"
|
47
|
-
redis 127.0.0.1:6379>
|
63
|
+
$echo get foo | redis-cli -h localhost -p 6379 -n 0
|
64
|
+
"1"
|
48
65
|
|
49
66
|
# Copyright
|
50
|
-
- Copyright
|
51
|
-
- Copyright
|
67
|
+
- Copyright © 2012 Buntaro Okada
|
68
|
+
- Copyright © 2011-2012 Yuki Nishijima
|
52
69
|
|
53
70
|
# License
|
54
71
|
- Apache License, Version 2.0
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0
|
1
|
+
0.1.0
|
@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-redis-counter"
|
6
|
-
s.version = "0.0
|
6
|
+
s.version = "0.1.0"
|
7
7
|
s.description = "fluent-plugin-redis-counter is a fluent plugin to count-up/down redis keys."
|
8
8
|
s.platform = Gem::Platform::RUBY
|
9
9
|
s.authors = ["Buntaro Okada"]
|
10
|
-
s.date = %q{2012-06-
|
10
|
+
s.date = %q{2012-06-19}
|
11
11
|
s.email = "kbinani.bt@gmail.com"
|
12
12
|
s.homepage = "https://github.com/kbinani/fluent-plugin-redis-counter"
|
13
13
|
s.summary = "Redis counter plugin for fluent"
|
@@ -18,5 +18,5 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.require_paths = ["lib"]
|
19
19
|
|
20
20
|
s.add_dependency %q<fluentd>, ["~> 0.10.0"]
|
21
|
-
s.add_dependency %q<redis>, ["
|
21
|
+
s.add_dependency %q<redis>, [">= 2.2.2"]
|
22
22
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module Fluent
|
2
2
|
class RedisCounterOutput < BufferedOutput
|
3
3
|
Fluent::Plugin.register_output('redis_counter', self)
|
4
|
-
attr_reader :host, :port, :db_number, :redis
|
4
|
+
attr_reader :host, :port, :db_number, :redis, :patterns
|
5
5
|
|
6
6
|
def initialize
|
7
7
|
super
|
@@ -14,6 +14,34 @@ module Fluent
|
|
14
14
|
@host = conf.has_key?('host') ? conf['host'] : 'localhost'
|
15
15
|
@port = conf.has_key?('port') ? conf['port'].to_i : 6379
|
16
16
|
@db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
|
17
|
+
@patterns = []
|
18
|
+
conf.elements.select { |e|
|
19
|
+
e.name == 'pattern'
|
20
|
+
}.each { |e|
|
21
|
+
if e.has_key?('count_key') == false
|
22
|
+
raise Fluent::ConfigError, '"count_key" is required.'
|
23
|
+
end
|
24
|
+
count_value = 1
|
25
|
+
if e.has_key?('count_value')
|
26
|
+
begin
|
27
|
+
count_value = Integer(e['count_value'])
|
28
|
+
rescue
|
29
|
+
raise Fluent::ConfigError, 'invalid "count_value", integer required.'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
matches = {}
|
33
|
+
e.each_key { |key|
|
34
|
+
if key =~ /^match_/
|
35
|
+
name = key['match_'.size .. key.size]
|
36
|
+
matches[name] = Regexp.new(e[key])
|
37
|
+
end
|
38
|
+
}
|
39
|
+
@patterns << {
|
40
|
+
'matches' => matches,
|
41
|
+
'count_key' => e['count_key'],
|
42
|
+
'count_value' => count_value
|
43
|
+
}
|
44
|
+
}
|
17
45
|
end
|
18
46
|
|
19
47
|
def start
|
@@ -38,26 +66,42 @@ module Fluent
|
|
38
66
|
chunk.open { |io|
|
39
67
|
begin
|
40
68
|
MessagePack::Unpacker.new(io).each { |record|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
table[key] += value
|
46
|
-
end
|
47
|
-
rescue ArgumentError
|
48
|
-
# convert to integer failed, do nothing.
|
49
|
-
rescue TypeError
|
50
|
-
# object can't convert to integer, do nothing.
|
51
|
-
end
|
52
|
-
}
|
69
|
+
matched_pattern = get_matched_pattern(record)
|
70
|
+
if matched_pattern != nil && matched_pattern['count_value'] != 0
|
71
|
+
table[matched_pattern['count_key']] += matched_pattern['count_value']
|
72
|
+
end
|
53
73
|
}
|
54
74
|
rescue EOFError
|
55
75
|
# EOFError always occured when reached end of chunk.
|
56
76
|
end
|
57
77
|
}
|
58
78
|
table.each_key { |key|
|
59
|
-
|
79
|
+
if (value = table[key]) != 0
|
80
|
+
@redis.incrby(key, value)
|
81
|
+
end
|
82
|
+
}
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
def get_matched_pattern(record)
|
87
|
+
@patterns.each { |pattern|
|
88
|
+
all_matched = true
|
89
|
+
pattern['matches'].each_key{ |key|
|
90
|
+
if !record.has_key?(key)
|
91
|
+
all_matched = false
|
92
|
+
break
|
93
|
+
else
|
94
|
+
if !(record[key] =~ pattern['matches'][key])
|
95
|
+
all_matched = false
|
96
|
+
break
|
97
|
+
end
|
98
|
+
end
|
99
|
+
}
|
100
|
+
if all_matched
|
101
|
+
return pattern
|
102
|
+
end
|
60
103
|
}
|
104
|
+
return nil
|
61
105
|
end
|
62
106
|
end
|
63
107
|
end
|
@@ -27,39 +27,91 @@ class RedisCounterTest < Test::Unit::TestCase
|
|
27
27
|
assert_equal 'localhost', @d.instance.host
|
28
28
|
assert_equal 6379, @d.instance.port
|
29
29
|
assert_equal 1, @d.instance.db_number
|
30
|
+
assert_equal 0, @d.instance.patterns.size
|
30
31
|
end
|
31
32
|
|
32
|
-
def
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
33
|
+
def test_configure_pattern
|
34
|
+
driver = create_driver %[
|
35
|
+
host localhost
|
36
|
+
port 6379
|
37
|
+
db_number 1
|
38
|
+
<pattern>
|
39
|
+
match_status ^2[0-9]{2}$
|
40
|
+
match_url ^https
|
41
|
+
count_key status-normal
|
42
|
+
</pattern>
|
43
|
+
<pattern>
|
44
|
+
count_key foo
|
45
|
+
count_value 2
|
46
|
+
</pattern>
|
47
|
+
]
|
48
|
+
assert_equal 'localhost', driver.instance.host
|
49
|
+
assert_equal 6379, driver.instance.port
|
50
|
+
assert_equal 1, driver.instance.db_number
|
51
|
+
assert_equal 2, driver.instance.patterns.size
|
37
52
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
@d.run
|
53
|
+
assert_equal 2, driver.instance.patterns[0]['matches'].size
|
54
|
+
assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0]['matches']['status']
|
55
|
+
assert_equal Regexp.new('^https'), driver.instance.patterns[0]['matches']['url']
|
56
|
+
assert_equal 'status-normal', driver.instance.patterns[0]['count_key']
|
57
|
+
assert_equal 1, driver.instance.patterns[0]['count_value']
|
44
58
|
|
45
|
-
assert_equal
|
46
|
-
assert_equal
|
59
|
+
assert_equal 0, driver.instance.patterns[1]['matches'].size
|
60
|
+
assert_equal 'foo', driver.instance.patterns[1]['count_key']
|
61
|
+
assert_equal 2, driver.instance.patterns[1]['count_value']
|
47
62
|
end
|
48
63
|
|
49
|
-
def
|
50
|
-
|
51
|
-
|
52
|
-
|
64
|
+
def test_configure_count_key_required
|
65
|
+
begin
|
66
|
+
create_driver %[
|
67
|
+
<pattern>
|
68
|
+
count_value 1
|
69
|
+
</pattern>
|
70
|
+
]
|
71
|
+
flunk
|
72
|
+
rescue Fluent::ConfigError => e
|
73
|
+
assert_equal '"count_key" is required.', e.message
|
74
|
+
end
|
75
|
+
end
|
53
76
|
|
54
|
-
|
77
|
+
def test_configure_invalid_count_value
|
78
|
+
begin
|
79
|
+
create_driver %[
|
80
|
+
<pattern>
|
81
|
+
count_key foo
|
82
|
+
count_value a
|
83
|
+
</pattern>
|
84
|
+
]
|
85
|
+
flunk
|
86
|
+
rescue Fluent::ConfigError => e
|
87
|
+
assert_equal 'invalid "count_value", integer required.', e.message
|
88
|
+
end
|
55
89
|
end
|
56
90
|
|
57
|
-
def
|
91
|
+
def test_format
|
58
92
|
@d.emit({"a" => 1})
|
59
|
-
@d.
|
93
|
+
@d.expect_format({"a" => 1}.to_msgpack)
|
60
94
|
@d.run
|
95
|
+
end
|
96
|
+
|
97
|
+
def test_write
|
98
|
+
driver = create_driver %[
|
99
|
+
db_number 1
|
100
|
+
<pattern>
|
101
|
+
match_a ^2[0-9][0-9]$
|
102
|
+
count_key a
|
103
|
+
count_value 2
|
104
|
+
</pattern>
|
105
|
+
]
|
106
|
+
driver.emit({"a" => "value-of-a"})
|
107
|
+
driver.emit({"a" => "200"})
|
108
|
+
driver.emit({"b" => "200"})
|
109
|
+
driver.emit({"aa" => "200"})
|
110
|
+
driver.emit({"a" => "2000"})
|
111
|
+
driver.run
|
61
112
|
|
62
|
-
assert_equal
|
113
|
+
assert_equal '2', driver.instance.redis.get("a")
|
114
|
+
assert_nil driver.instance.redis.get("b")
|
63
115
|
end
|
64
116
|
|
65
117
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-redis-counter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-06-
|
12
|
+
date: 2012-06-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -32,7 +32,7 @@ dependencies:
|
|
32
32
|
requirement: !ruby/object:Gem::Requirement
|
33
33
|
none: false
|
34
34
|
requirements:
|
35
|
-
- -
|
35
|
+
- - ! '>='
|
36
36
|
- !ruby/object:Gem::Version
|
37
37
|
version: 2.2.2
|
38
38
|
type: :runtime
|
@@ -40,7 +40,7 @@ dependencies:
|
|
40
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
41
|
none: false
|
42
42
|
requirements:
|
43
|
-
- -
|
43
|
+
- - ! '>='
|
44
44
|
- !ruby/object:Gem::Version
|
45
45
|
version: 2.2.2
|
46
46
|
description: fluent-plugin-redis-counter is a fluent plugin to count-up/down redis
|