fluent-plugin-redis-multi-type-counter 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +4 -0
- data/.travis.yml +16 -0
- data/CHANGELOG.md +26 -0
- data/Gemfile +4 -0
- data/README.md +119 -0
- data/Rakefile +2 -0
- data/VERSION +1 -0
- data/benchmark/performance_test.rb +99 -0
- data/fluent-plugin-redis-multi-type-counter.gemspec +24 -0
- data/lib/fluent/plugin/out_redis_counter.rb +247 -0
- data/test/plugin/out_redis_counter.rb +372 -0
- metadata +84 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f1e9b2d54944b6575cabdad55ce1250d61c3163a
|
4
|
+
data.tar.gz: 692d3df513e0f23ccc6019f7547092f8a38e2e69
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c750a05f9e29ce9b9a925f055039c35e4e1e010dff5db0fb7f6195974e4a025427257b43eb970501d3b60a2570312aff3315eb42cd534000fa6908be3601c77a
|
7
|
+
data.tar.gz: 6d71cc09fce6a3d3c91c1f92174bd36469602c650d1aaf397e007083b7640f95f2c5df2fee87d63f3d0758f389db860bbdcf69159f5d07303a874ac68c6230f5
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
## 0.1.0
|
2
|
+
* support hash, sorted set type
|
3
|
+
|
4
|
+
######################################################################
|
5
|
+
## fluent-plugin-redis-counter
|
6
|
+
## 0.3.2
|
7
|
+
* add password support for Redis connection.
|
8
|
+
|
9
|
+
## 0.3.1
|
10
|
+
* improve performance
|
11
|
+
|
12
|
+
## 0.3.0
|
13
|
+
* count value can be picked up from JSON data contents.
|
14
|
+
|
15
|
+
## 0.2.0
|
16
|
+
* count key can be built from JSON data contents.
|
17
|
+
|
18
|
+
## 0.1.1
|
19
|
+
* add 'count_key_format' in order to make time-dependent redis keyname
|
20
|
+
* bug fix
|
21
|
+
|
22
|
+
## 0.1.0
|
23
|
+
* design changed.
|
24
|
+
|
25
|
+
## 0.0.1
|
26
|
+
* first version
|
data/Gemfile
ADDED
data/README.md
ADDED
@@ -0,0 +1,119 @@
|
|
1
|
+
# Redis multi-type counter plugin for fluent [![Build Status](https://secure.travis-ci.org/heartsavior/fluent-plugin-redis-multi-type-counter.png)](http://travis-ci.org/heartsavior/fluent-plugin-redis-multi-type-counter)
|
2
|
+
|
3
|
+
fluent-plugin-redis-multi-type-counter is a fluent plugin to count-up/down redis keys, hash, sorted set.
|
4
|
+
|
5
|
+
# Installation
|
6
|
+
|
7
|
+
fluent-plugin-redis-multi-type-counter is hosted by [RubyGems.org](https://rubygems.org/).
|
8
|
+
|
9
|
+
$fluent-gem install fluent-plugin-redis-multi-type-counter
|
10
|
+
|
11
|
+
# Configuration
|
12
|
+
|
13
|
+
<match redis_counter.**>
|
14
|
+
type redis_counter
|
15
|
+
|
16
|
+
host localhost
|
17
|
+
port 6379
|
18
|
+
|
19
|
+
# database number is optional.
|
20
|
+
db_number 0 # 0 is default
|
21
|
+
|
22
|
+
# match condition
|
23
|
+
# this pattern matches {"status": "200", "url": "http://foo.example.com"} and then,
|
24
|
+
# increment Redis key "foo-status2xx" by calling Redis::incrby( "foo-status2xx", 1 ).
|
25
|
+
<pattern>
|
26
|
+
match_status ^2[0-9][0-9]$ # matches with {"status": "200", ...
|
27
|
+
match_url ^http:\/\/foo\. # matches with {"url": "http://foo.example.com", ...
|
28
|
+
count_key foo-status2xx # key-name for Redis
|
29
|
+
count_value 1 # count-up amount(default: 1, negative value is allowed)
|
30
|
+
</pattern>
|
31
|
+
|
32
|
+
# time-dependent redis keyname
|
33
|
+
# for example, "foo-status2xx-%Y-%m-%d" will be formatted to "foo-status2xx-2012-06-21".
|
34
|
+
# rules for placeholder(%Y, etc.) is similar to out_file plugin.
|
35
|
+
<pattern>
|
36
|
+
match_status ^2[0-9][0-9]$
|
37
|
+
count_key_format foo-statux2xx-%d
|
38
|
+
localtime # time-zone(default: localtime, it can be "utc" or "localtime")
|
39
|
+
</pattern>
|
40
|
+
|
41
|
+
# you can also use values from the matched JSON record in the key name, by using the
|
42
|
+
# syntax foo%_{key1}-%_{key2}, where key1 and key2 are keys in the JSON record that
|
43
|
+
# was received by fluentd.
|
44
|
+
# for example, "customer:%_{customer_id}:status2xx" will be formatted to
|
45
|
+
# "customer:123:status2xx" if the JSON record contains a key named "customer_id"
|
46
|
+
# with value 123, like so: {"status": 200, "customer_id": 123 ... }.
|
47
|
+
# these can be combined with the time formatting options in the previous example.
|
48
|
+
<pattern>
|
49
|
+
match_status ^2[0-9][0-9]$
|
50
|
+
count_key_format customer:%_{customer_id}:status2xx-%Y-%m-%d
|
51
|
+
</pattern>
|
52
|
+
|
53
|
+
# you can also sum up key in hash, by configuring count_hash_key_format
|
54
|
+
# syntax is same to count_key_format
|
55
|
+
# for example, {"custom_id": 123, "date": "20131219" ...}.
|
56
|
+
# HINCRBY item_count:123 20131219 1
|
57
|
+
<pattern>
|
58
|
+
count_key_format item_count:%_{item_id}
|
59
|
+
count_hash_key_format %_{date}
|
60
|
+
<pattern>
|
61
|
+
|
62
|
+
# you can also sum up key in sorted set(zset), by configuring count_zset_key_format
|
63
|
+
# syntax and usage is same to count_hash_key_format
|
64
|
+
# for example, {"custom_id": 123, "date": "20131219" ...}.
|
65
|
+
# ZINCRBY item_count:123 1 20131219
|
66
|
+
<pattern>
|
67
|
+
count_key_format item_count:%_{item_id}
|
68
|
+
count_zset_key_format %_{date}
|
69
|
+
<pattern>
|
70
|
+
|
71
|
+
# you can also sum up specified key with count_value_key option.
|
72
|
+
# for example, {"count": 321, "customer_id": 123 ... }.
|
73
|
+
# INCRBY item_count:123 321.
|
74
|
+
<pattern>
|
75
|
+
count_key_format item_count:%_{item_id}
|
76
|
+
count_value_key count
|
77
|
+
</pattern>
|
78
|
+
</match>
|
79
|
+
|
80
|
+
# Example
|
81
|
+
|
82
|
+
prepare a conf file ("fluent.conf") in current directory like this:
|
83
|
+
|
84
|
+
<source>
|
85
|
+
type forward
|
86
|
+
</source>
|
87
|
+
<match debug.**>
|
88
|
+
type redis_counter
|
89
|
+
host localhost
|
90
|
+
port 6379
|
91
|
+
db_number 0
|
92
|
+
<pattern>
|
93
|
+
match_status ^2[0-9][0-9]$
|
94
|
+
match_url ^http:\/\/foo\.
|
95
|
+
count_key foo
|
96
|
+
</pattern>
|
97
|
+
</match>
|
98
|
+
|
99
|
+
run commands for test:
|
100
|
+
|
101
|
+
$redis-server 2>&1 >/dev/null &
|
102
|
+
[1] 879
|
103
|
+
$echo del foo | redis-cli -h localhost -p 6379 -n 0
|
104
|
+
(integer) 0
|
105
|
+
$fluentd -c ./fluent.conf 2>&1 >/dev/null &
|
106
|
+
[2] 889
|
107
|
+
$echo {\"status\": \"200\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
|
108
|
+
$echo {\"status\": \"500\", \"url\": \"http://foo.example.com\"} | fluent-cat debug
|
109
|
+
$kill -s HUP 889
|
110
|
+
$echo get foo | redis-cli -h localhost -p 6379 -n 0
|
111
|
+
"1"
|
112
|
+
|
113
|
+
# Copyright
|
114
|
+
- Copyright © 2014 Jungtaek Lim
|
115
|
+
- Copyright © 2012-2014 Buntaro Okada
|
116
|
+
- Copyright © 2011-2012 Yuki Nishijima
|
117
|
+
|
118
|
+
# License
|
119
|
+
- Apache License, Version 2.0
|
data/Rakefile
ADDED
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
require 'benchmark'
|
3
|
+
|
4
|
+
if ENV["PROFILE"]
|
5
|
+
require 'ruby-prof'
|
6
|
+
end
|
7
|
+
|
8
|
+
$LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
|
9
|
+
require 'fluent/plugin/out_redis_counter'
|
10
|
+
|
11
|
+
module PerformanceTestApp
|
12
|
+
class Benchmarker
|
13
|
+
def initialize
|
14
|
+
Fluent::Test.setup
|
15
|
+
@fluent = Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisCounterOutput).configure(%[
|
16
|
+
host localhost
|
17
|
+
port 6379
|
18
|
+
db_number 1
|
19
|
+
<pattern>
|
20
|
+
match_status ^2[0-9]{2}$
|
21
|
+
count_key_format status-normal:item_id:%_{item_id}
|
22
|
+
count_value 1
|
23
|
+
</pattern>
|
24
|
+
])
|
25
|
+
end
|
26
|
+
|
27
|
+
def prepare
|
28
|
+
10_0000.times do
|
29
|
+
item_id = ( rand * 10_0000 ).to_i
|
30
|
+
@fluent.emit({ "status" => "200", "item_id" => item_id })
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def run
|
35
|
+
result = nil
|
36
|
+
profile = nil
|
37
|
+
Benchmark.bm do |x|
|
38
|
+
result = x.report {
|
39
|
+
if ENV["PROFILE"]
|
40
|
+
profile = RubyProf.profile do
|
41
|
+
@fluent.run
|
42
|
+
end
|
43
|
+
else
|
44
|
+
@fluent.run
|
45
|
+
end
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
if ENV["PROFILE"]
|
50
|
+
profile_printer = RubyProf::GraphPrinter.new(profile)
|
51
|
+
profile_printer.print(STDOUT, {})
|
52
|
+
end
|
53
|
+
$log.info("benchmark result: #{result}")
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
class MemoryWatcher
|
58
|
+
def initialize
|
59
|
+
end
|
60
|
+
|
61
|
+
def run(&block)
|
62
|
+
start_memory = get_memory(Process.pid)
|
63
|
+
pid = Process.fork do
|
64
|
+
block.call
|
65
|
+
end
|
66
|
+
@max_memory = 0
|
67
|
+
th = Thread.new do
|
68
|
+
begin
|
69
|
+
loop do
|
70
|
+
sleep(0.01)
|
71
|
+
msize = get_memory(pid)
|
72
|
+
if msize > @max_memory
|
73
|
+
@max_memory = msize
|
74
|
+
end
|
75
|
+
end
|
76
|
+
ensure
|
77
|
+
$log.info("start memory size:\t#{sprintf("%#10d", start_memory / 1024)}KB")
|
78
|
+
$log.info("max memory size:\t#{sprintf("%#10d", @max_memory / 1024)}KB")
|
79
|
+
end
|
80
|
+
end
|
81
|
+
Process.waitall
|
82
|
+
th.kill
|
83
|
+
end
|
84
|
+
|
85
|
+
def get_memory(pid)
|
86
|
+
`ps -h -o rss= -p #{pid}`.to_i * 1024
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
if $0 == __FILE__
|
92
|
+
parent = PerformanceTestApp::MemoryWatcher.new
|
93
|
+
app = PerformanceTestApp::Benchmarker.new
|
94
|
+
parent.run do
|
95
|
+
app.prepare
|
96
|
+
app.run
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = "fluent-plugin-redis-multi-type-counter"
|
6
|
+
s.version = "0.1.0"
|
7
|
+
s.description = "fluent-plugin-redis-multi-type-counter is a fluent plugin to count-up/down redis keys, hash keys, zset keys"
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Jungtaek Lim"]
|
10
|
+
s.date = %q{2014-01-07}
|
11
|
+
s.email = "kabhwan@gmail.com"
|
12
|
+
s.homepage = "https://github.com/heartsavior/fluent-plugin-redis-multi-type-counter"
|
13
|
+
s.summary = "Redis multi type counter plugin for fluent"
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ["lib"]
|
19
|
+
|
20
|
+
s.add_dependency %q<fluentd>, ["~> 0.10.0"]
|
21
|
+
s.add_dependency %q<redis>, [">= 2.2.2"]
|
22
|
+
|
23
|
+
s.license = 'APACHE2'
|
24
|
+
end
|
@@ -0,0 +1,247 @@
|
|
1
|
+
module Fluent
|
2
|
+
class RedisCounterOutput < BufferedOutput
|
3
|
+
Fluent::Plugin.register_output('redis_counter', self)
|
4
|
+
attr_reader :host, :port, :db_number, :password, :redis, :patterns
|
5
|
+
|
6
|
+
config_param :max_pipelining, :integer, :default => 1000
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super
|
10
|
+
require 'redis'
|
11
|
+
require 'msgpack'
|
12
|
+
end
|
13
|
+
|
14
|
+
def configure(conf)
|
15
|
+
super
|
16
|
+
@host = conf.has_key?('host') ? conf['host'] : 'localhost'
|
17
|
+
@port = conf.has_key?('port') ? conf['port'].to_i : 6379
|
18
|
+
@password = conf.has_key?('password') ? conf['password'] : nil
|
19
|
+
@db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
|
20
|
+
@patterns = []
|
21
|
+
conf.elements.select { |element|
|
22
|
+
element.name == 'pattern'
|
23
|
+
}.each { |element|
|
24
|
+
begin
|
25
|
+
@patterns << Pattern.new(element)
|
26
|
+
rescue RedisCounterException => e
|
27
|
+
raise Fluent::ConfigError, e.message
|
28
|
+
end
|
29
|
+
}
|
30
|
+
end
|
31
|
+
|
32
|
+
def start
|
33
|
+
super
|
34
|
+
@redis = Redis.new(
|
35
|
+
:host => @host, :port => @port,
|
36
|
+
:password => @password,
|
37
|
+
:thread_safe => true, :db => @db_number
|
38
|
+
)
|
39
|
+
end
|
40
|
+
|
41
|
+
def shutdown
|
42
|
+
@redis.quit
|
43
|
+
end
|
44
|
+
|
45
|
+
def format(tag, time, record)
|
46
|
+
[tag, time, record].to_msgpack
|
47
|
+
end
|
48
|
+
|
49
|
+
def write(chunk)
|
50
|
+
table = {}
|
51
|
+
table.default = 0
|
52
|
+
chunk.open { |io|
|
53
|
+
begin
|
54
|
+
MessagePack::Unpacker.new(io).each { |message|
|
55
|
+
(tag, time, record) = message
|
56
|
+
@patterns.select { |pattern|
|
57
|
+
pattern.is_match?(record)
|
58
|
+
}.each{ |pattern|
|
59
|
+
count_key = pattern.get_count_key(time, record)
|
60
|
+
count_hash_key = pattern.get_count_hash_key(record)
|
61
|
+
count_zset_key = pattern.get_count_zset_key(record)
|
62
|
+
|
63
|
+
key = RecordKey.new(count_key, count_hash_key, count_zset_key)
|
64
|
+
table[key] += pattern.get_count_value(record)
|
65
|
+
}
|
66
|
+
}
|
67
|
+
rescue EOFError
|
68
|
+
# EOFError always occured when reached end of chunk.
|
69
|
+
end
|
70
|
+
}
|
71
|
+
|
72
|
+
table.each_pair.select { |key, value|
|
73
|
+
value != 0
|
74
|
+
}.each_slice(@max_pipelining) { |items|
|
75
|
+
@redis.pipelined do
|
76
|
+
items.each do |key, value|
|
77
|
+
if key.count_hash_key != nil
|
78
|
+
@redis.hincrby(key.count_key, key.count_hash_key, value)
|
79
|
+
elsif key.count_zset_key != nil
|
80
|
+
@redis.zincrby(key.count_key, value, key.count_zset_key)
|
81
|
+
else
|
82
|
+
@redis.incrby(key.count_key, value)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
}
|
87
|
+
end
|
88
|
+
|
89
|
+
class RecordKey
|
90
|
+
attr_reader :count_key, :count_hash_key, :count_zset_key
|
91
|
+
|
92
|
+
def initialize(count_key, count_hash_key, count_zset_key)
|
93
|
+
@count_key = count_key
|
94
|
+
@count_hash_key = count_hash_key
|
95
|
+
@count_zset_key = count_zset_key
|
96
|
+
end
|
97
|
+
|
98
|
+
def hash
|
99
|
+
hash_key = ""
|
100
|
+
|
101
|
+
keys = [@count_key, @count_hash_key, @count_zset_key]
|
102
|
+
keys.select { |key|
|
103
|
+
key != nil
|
104
|
+
}.each { |key|
|
105
|
+
hash_key += ("@@@@" + key)
|
106
|
+
}
|
107
|
+
|
108
|
+
hash_key.hash
|
109
|
+
end
|
110
|
+
|
111
|
+
def eql?(other)
|
112
|
+
return @count_key.eql?(other.count_key) && @count_hash_key.eql?(other.count_hash_key) &&
|
113
|
+
@count_zset_key.eql?(other.count_zset_key)
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
class RedisCounterException < Exception
|
118
|
+
end
|
119
|
+
|
120
|
+
class RecordValueFormatter
|
121
|
+
attr_reader :format
|
122
|
+
def initialize(format)
|
123
|
+
@format = format
|
124
|
+
end
|
125
|
+
|
126
|
+
CUSTOM_KEY_EXPRESSION_RE = /(%_\{([^\}]+)\})/
|
127
|
+
|
128
|
+
def key(record)
|
129
|
+
@format.gsub(CUSTOM_KEY_EXPRESSION_RE) do |s|
|
130
|
+
record[$2]
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
class Pattern
|
136
|
+
attr_reader :matches, :count_value, :count_value_key
|
137
|
+
|
138
|
+
def initialize(conf_element)
|
139
|
+
if !conf_element.has_key?('count_key') && !conf_element.has_key?('count_key_format')
|
140
|
+
raise RedisCounterException, '"count_key" or "count_key_format" is required.'
|
141
|
+
end
|
142
|
+
if conf_element.has_key?('count_key') && conf_element.has_key?('count_key_format')
|
143
|
+
raise RedisCounterException, 'both "count_key" and "count_key_format" are specified.'
|
144
|
+
end
|
145
|
+
|
146
|
+
if conf_element.has_key?('count_key')
|
147
|
+
@count_key = conf_element['count_key']
|
148
|
+
else
|
149
|
+
if conf_element.has_key?('localtime') && conf_element.has_key?('utc')
|
150
|
+
raise RedisCounterException, 'both "localtime" and "utc" are specified.'
|
151
|
+
end
|
152
|
+
is_localtime = true
|
153
|
+
if conf_element.has_key?('utc')
|
154
|
+
is_localtime = false
|
155
|
+
end
|
156
|
+
@count_key_format = [conf_element['count_key_format'], is_localtime]
|
157
|
+
@record_formatter_for_count_key = RecordValueFormatter.new(@count_key_format[0])
|
158
|
+
end
|
159
|
+
|
160
|
+
if conf_element.has_key?('count_hash_key_format') && conf_element.has_key?('count_zset_key_format')
|
161
|
+
raise RedisCounterException, 'both "count_hash_key_format" "count_zset_key_format" are specified.'
|
162
|
+
end
|
163
|
+
|
164
|
+
if conf_element.has_key?('count_hash_key_format')
|
165
|
+
@count_hash_key_format = conf_element['count_hash_key_format']
|
166
|
+
@record_formatter_for_count_hash_key = RecordValueFormatter.new(@count_hash_key_format)
|
167
|
+
else
|
168
|
+
@count_hash_key_format = nil
|
169
|
+
end
|
170
|
+
|
171
|
+
if conf_element.has_key?('count_zset_key_format')
|
172
|
+
@count_zset_key_format = conf_element['count_zset_key_format']
|
173
|
+
@record_formatter_for_count_zset_key = RecordValueFormatter.new(@count_zset_key_format)
|
174
|
+
else
|
175
|
+
@count_zset_key_format = nil
|
176
|
+
end
|
177
|
+
|
178
|
+
if conf_element.has_key?('count_value_key')
|
179
|
+
@count_value_key = conf_element['count_value_key']
|
180
|
+
else
|
181
|
+
@count_value = 1
|
182
|
+
if conf_element.has_key?('count_value')
|
183
|
+
begin
|
184
|
+
@count_value = Integer(conf_element['count_value'])
|
185
|
+
rescue
|
186
|
+
raise RedisCounterException, 'invalid "count_value", integer required.'
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
@matches = {}
|
192
|
+
conf_element.each_pair.select { |key, value|
|
193
|
+
key =~ /^match_/
|
194
|
+
}.each { |key, value|
|
195
|
+
name = key['match_'.size .. key.size]
|
196
|
+
@matches[name] = Regexp.new(value)
|
197
|
+
}
|
198
|
+
end
|
199
|
+
|
200
|
+
def is_match?(record)
|
201
|
+
@matches.each_pair{ |key, value|
|
202
|
+
if !record.has_key?(key) || !(record[key] =~ value)
|
203
|
+
return false
|
204
|
+
end
|
205
|
+
}
|
206
|
+
return true
|
207
|
+
end
|
208
|
+
|
209
|
+
def get_count_key(time, record)
|
210
|
+
if @count_key_format == nil
|
211
|
+
@count_key
|
212
|
+
else
|
213
|
+
count_key = @record_formatter_for_count_key.key(record)
|
214
|
+
formatter = TimeFormatter.new(count_key, @count_key_format[1])
|
215
|
+
formatter.format(time)
|
216
|
+
end
|
217
|
+
end
|
218
|
+
|
219
|
+
def get_count_hash_key(record)
|
220
|
+
if @count_hash_key_format == nil
|
221
|
+
return nil
|
222
|
+
else
|
223
|
+
return @record_formatter_for_count_hash_key.key(record)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
def get_count_zset_key(record)
|
228
|
+
if @count_zset_key_format == nil
|
229
|
+
return nil
|
230
|
+
else
|
231
|
+
return @record_formatter_for_count_zset_key.key(record)
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def get_count_value(record)
|
236
|
+
if @count_value_key
|
237
|
+
ret = record[@count_value_key] || 0
|
238
|
+
return ret.kind_of?(Integer) ? ret : 0
|
239
|
+
else
|
240
|
+
if @count_value
|
241
|
+
return @count_value
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
245
|
+
end
|
246
|
+
end
|
247
|
+
end
|
@@ -0,0 +1,372 @@
|
|
1
|
+
require 'fluent/test'
|
2
|
+
|
3
|
+
class RedisCounterTest < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
Fluent::Test.setup
|
6
|
+
require 'fluent/plugin/out_redis_counter'
|
7
|
+
|
8
|
+
@d = create_driver %[
|
9
|
+
host localhost
|
10
|
+
port 6379
|
11
|
+
db_number 1
|
12
|
+
]
|
13
|
+
redis = Redis.new(
|
14
|
+
:host => "localhost", :port => 6379,
|
15
|
+
:thread_safe => true, :db => 1
|
16
|
+
)
|
17
|
+
redis.del("a")
|
18
|
+
redis.del("b")
|
19
|
+
redis.del("foo-2012-06-21")
|
20
|
+
redis.del("item_sum_count:200")
|
21
|
+
redis.del("item_sum_count_by_hash:200")
|
22
|
+
redis.del("item_sum_count_by_zset:200")
|
23
|
+
redis.quit
|
24
|
+
end
|
25
|
+
|
26
|
+
def create_driver(conf = CONFIG)
|
27
|
+
Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisCounterOutput).configure(conf)
|
28
|
+
end
|
29
|
+
|
30
|
+
def test_configure
|
31
|
+
assert_equal 'localhost', @d.instance.host
|
32
|
+
assert_equal 6379, @d.instance.port
|
33
|
+
assert_equal 1, @d.instance.db_number
|
34
|
+
assert_equal 0, @d.instance.patterns.size
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_configure_pattern
|
38
|
+
driver = create_driver %[
|
39
|
+
host localhost
|
40
|
+
port 6379
|
41
|
+
db_number 1
|
42
|
+
<pattern>
|
43
|
+
match_status ^2[0-9]{2}$
|
44
|
+
match_url ^https
|
45
|
+
count_key status-normal
|
46
|
+
</pattern>
|
47
|
+
<pattern>
|
48
|
+
count_key foo
|
49
|
+
count_value 2
|
50
|
+
</pattern>
|
51
|
+
]
|
52
|
+
assert_equal 'localhost', driver.instance.host
|
53
|
+
assert_equal 6379, driver.instance.port
|
54
|
+
assert_equal 1, driver.instance.db_number
|
55
|
+
assert_equal 2, driver.instance.patterns.size
|
56
|
+
|
57
|
+
assert_equal 2, driver.instance.patterns[0].matches.size
|
58
|
+
assert_equal Regexp.new('^2[0-9]{2}$'), driver.instance.patterns[0].matches['status']
|
59
|
+
assert_equal Regexp.new('^https'), driver.instance.patterns[0].matches['url']
|
60
|
+
assert_equal 'status-normal', driver.instance.patterns[0].get_count_key(Time.now.to_i, {})
|
61
|
+
assert_equal 1, driver.instance.patterns[0].count_value
|
62
|
+
|
63
|
+
assert_equal 0, driver.instance.patterns[1].matches.size
|
64
|
+
assert_equal 'foo', driver.instance.patterns[1].get_count_key(Time.now.to_i, {})
|
65
|
+
assert_equal 2, driver.instance.patterns[1].count_value
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_configure_count_key_required
|
69
|
+
begin
|
70
|
+
create_driver %[
|
71
|
+
<pattern>
|
72
|
+
count_value 1
|
73
|
+
</pattern>
|
74
|
+
]
|
75
|
+
flunk
|
76
|
+
rescue Fluent::ConfigError => e
|
77
|
+
assert_equal '"count_key" or "count_key_format" is required.', e.message
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def test_configure_count_key_duplicated
|
82
|
+
begin
|
83
|
+
create_driver %[
|
84
|
+
<pattern>
|
85
|
+
count_key foo
|
86
|
+
count_key_format foo-%Y
|
87
|
+
</pattern>
|
88
|
+
]
|
89
|
+
flunk
|
90
|
+
rescue Fluent::ConfigError => e
|
91
|
+
assert_equal 'both "count_key" and "count_key_format" are specified.', e.message
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def test_configure_count_key_format_utc
|
96
|
+
driver = create_driver %[
|
97
|
+
<pattern>
|
98
|
+
count_key_format foo-%Y-%m-%d-%H-%M-%S
|
99
|
+
utc
|
100
|
+
</pattern>
|
101
|
+
]
|
102
|
+
time = Time.parse('2011-06-21 03:12:01 UTC').to_i
|
103
|
+
assert_equal 'foo-2011-06-21-03-12-01', driver.instance.patterns[0].get_count_key(time, {})
|
104
|
+
end
|
105
|
+
|
106
|
+
def test_configure_count_key_format_localtime
|
107
|
+
driver = create_driver %[
|
108
|
+
<pattern>
|
109
|
+
count_key_format foo-%Y-%m-%d-%H-%M-%S
|
110
|
+
localtime
|
111
|
+
</pattern>
|
112
|
+
]
|
113
|
+
local_time = Time.parse('2012-06-21 03:12:00').to_i
|
114
|
+
assert_equal 'foo-2012-06-21-03-12-00', driver.instance.patterns[0].get_count_key(local_time, {})
|
115
|
+
end
|
116
|
+
|
117
|
+
def test_configure_duplicated_timezone
|
118
|
+
begin
|
119
|
+
create_driver %[
|
120
|
+
<pattern>
|
121
|
+
count_key_format foo%Y
|
122
|
+
localtime
|
123
|
+
utc
|
124
|
+
</pattern>
|
125
|
+
]
|
126
|
+
flunk
|
127
|
+
rescue Fluent::ConfigError => e
|
128
|
+
assert_equal 'both "localtime" and "utc" are specified.', e.message
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def test_configure_count_key_format_with_record_value_formatter
|
133
|
+
driver = create_driver %[
|
134
|
+
<pattern>
|
135
|
+
count_key_format %_{prefix}-foo-%Y-%m-%_{type}-%_{customer_id}
|
136
|
+
localtime
|
137
|
+
</pattern>
|
138
|
+
]
|
139
|
+
local_time = Time.parse('2012-06-21 03:12:00').to_i
|
140
|
+
record = {'prefix' => 'pre', 'type' => 'bar', 'customer_id' => 321}
|
141
|
+
assert_equal 'pre-foo-2012-06-bar-321', driver.instance.patterns[0].get_count_key(local_time, record)
|
142
|
+
end
|
143
|
+
|
144
|
+
def test_configure_count_hash_key
|
145
|
+
driver = create_driver %[
|
146
|
+
<pattern>
|
147
|
+
count_key_format %_{prefix}-foo-bar
|
148
|
+
count_hash_key_format %_{host}-hash-key
|
149
|
+
</pattern>
|
150
|
+
]
|
151
|
+
record = {'prefix' => 'pre', 'host' => 'localhost'}
|
152
|
+
local_time = Time.parse('2012-06-21 03:12:00').to_i
|
153
|
+
assert_equal 'pre-foo-bar', driver.instance.patterns[0].get_count_key(local_time, record)
|
154
|
+
assert_equal 'localhost-hash-key', driver.instance.patterns[0].get_count_hash_key(record)
|
155
|
+
end
|
156
|
+
|
157
|
+
def test_configure_count_zset_key
|
158
|
+
driver = create_driver %[
|
159
|
+
<pattern>
|
160
|
+
count_key_format %_{prefix}-foo-bar
|
161
|
+
count_zset_key_format %_{host}-zset-key
|
162
|
+
</pattern>
|
163
|
+
]
|
164
|
+
record = {'prefix' => 'pre', 'host' => 'localhost'}
|
165
|
+
local_time = Time.parse('2012-06-21 03:12:00').to_i
|
166
|
+
assert_equal 'pre-foo-bar', driver.instance.patterns[0].get_count_key(local_time, record)
|
167
|
+
assert_equal 'localhost-zset-key', driver.instance.patterns[0].get_count_zset_key(record)
|
168
|
+
end
|
169
|
+
|
170
|
+
def test_configure_both_specified_hash_key_and_zset_key
|
171
|
+
begin
|
172
|
+
create_driver %[
|
173
|
+
<pattern>
|
174
|
+
count_key_format %_{prefix}-foo-bar
|
175
|
+
count_hash_key_format %_{host}-hash-key
|
176
|
+
count_zset_key_format %_{host}-zset-key
|
177
|
+
</pattern>
|
178
|
+
]
|
179
|
+
flunk
|
180
|
+
rescue Fluent::ConfigError => e
|
181
|
+
assert_equal 'both "count_hash_key_format" "count_zset_key_format" are specified.', e.message
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
def test_configure_invalid_count_value
|
186
|
+
begin
|
187
|
+
create_driver %[
|
188
|
+
<pattern>
|
189
|
+
count_key foo
|
190
|
+
count_value a
|
191
|
+
</pattern>
|
192
|
+
]
|
193
|
+
flunk
|
194
|
+
rescue Fluent::ConfigError => e
|
195
|
+
assert_equal 'invalid "count_value", integer required.', e.message
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
def test_get_count_value_with_count_value_key
|
200
|
+
driver = create_driver %[
|
201
|
+
<pattern>
|
202
|
+
count_key_format %_{customer_id}
|
203
|
+
count_value_key count
|
204
|
+
</pattern>
|
205
|
+
]
|
206
|
+
record = {'count' => 123, 'customer_id' => 321}
|
207
|
+
assert_equal 123, driver.instance.patterns[0].get_count_value(record)
|
208
|
+
end
|
209
|
+
|
210
|
+
def test_get_count_value_without_count_value_key
|
211
|
+
driver = create_driver %[
|
212
|
+
<pattern>
|
213
|
+
count_key_format %_{customer_id}
|
214
|
+
</pattern>
|
215
|
+
]
|
216
|
+
record = {'count' => 123, 'customer_id' => 321}
|
217
|
+
assert_equal 1, driver.instance.patterns[0].get_count_value(record)
|
218
|
+
end
|
219
|
+
|
220
|
+
def test_format
|
221
|
+
time = Time.parse('2012-06-21 01:55:00 UTC').to_i
|
222
|
+
@d.emit({"a" => 1}, time)
|
223
|
+
@d.expect_format(['test', time, {"a" => 1}].to_msgpack)
|
224
|
+
@d.run
|
225
|
+
end
|
226
|
+
|
227
|
+
def test_write
|
228
|
+
driver = create_driver %[
|
229
|
+
db_number 1
|
230
|
+
<pattern>
|
231
|
+
match_a ^2[0-9][0-9]$
|
232
|
+
count_key a
|
233
|
+
count_value 2
|
234
|
+
</pattern>
|
235
|
+
]
|
236
|
+
driver.emit({"a" => "value-of-a"})
|
237
|
+
driver.emit({"a" => "200"})
|
238
|
+
driver.emit({"b" => "200"})
|
239
|
+
driver.emit({"aa" => "200"})
|
240
|
+
driver.emit({"a" => "2000"})
|
241
|
+
driver.run
|
242
|
+
|
243
|
+
assert_equal '2', driver.instance.redis.get("a")
|
244
|
+
assert_nil driver.instance.redis.get("b")
|
245
|
+
end
|
246
|
+
|
247
|
+
def test_write_with_timeformat
|
248
|
+
driver = create_driver %[
|
249
|
+
db_number 1
|
250
|
+
<pattern>
|
251
|
+
match_a ^2[0-9][0-9]$
|
252
|
+
count_key_format foo-%Y-%m-%d
|
253
|
+
count_value 2
|
254
|
+
</pattern>
|
255
|
+
]
|
256
|
+
time = Time.parse('2012-06-21 03:01:00 UTC').to_i
|
257
|
+
driver.emit({"a" => "200"}, time)
|
258
|
+
driver.run
|
259
|
+
|
260
|
+
assert_equal '2', driver.instance.redis.get("foo-2012-06-21")
|
261
|
+
end
|
262
|
+
|
263
|
+
def test_write_with_count_hash_key
|
264
|
+
def create_driver_for_test
|
265
|
+
driver = create_driver %[
|
266
|
+
db_number 1
|
267
|
+
<pattern>
|
268
|
+
count_key_format item_sum_count_by_hash:%_{item_id}
|
269
|
+
count_hash_key_format %_{from}
|
270
|
+
</pattern>
|
271
|
+
]
|
272
|
+
end
|
273
|
+
|
274
|
+
driver = create_driver_for_test
|
275
|
+
|
276
|
+
time = Time.parse('2012-06-21 03:01:00 UTC').to_i
|
277
|
+
driver.emit({"item_id" => 200, "from" => "US"}, time)
|
278
|
+
driver.run
|
279
|
+
|
280
|
+
assert_equal '1', driver.instance.redis.hget("item_sum_count_by_hash:200", "US")
|
281
|
+
|
282
|
+
driver = create_driver_for_test
|
283
|
+
|
284
|
+
driver.emit({"item_id" => 200, "from" => "CHINA"}, time)
|
285
|
+
driver.run
|
286
|
+
|
287
|
+
assert_equal '1', driver.instance.redis.hget("item_sum_count_by_hash:200", "CHINA")
|
288
|
+
assert_equal '1', driver.instance.redis.hget("item_sum_count_by_hash:200", "US")
|
289
|
+
|
290
|
+
driver = create_driver_for_test
|
291
|
+
|
292
|
+
driver.emit({"item_id" => 200, "from" => "US"}, time)
|
293
|
+
driver.run
|
294
|
+
|
295
|
+
assert_equal '2', driver.instance.redis.hget("item_sum_count_by_hash:200", "US")
|
296
|
+
end
|
297
|
+
|
298
|
+
def test_write_with_count_zset_key
|
299
|
+
def create_driver_for_test
|
300
|
+
driver = create_driver %[
|
301
|
+
db_number 1
|
302
|
+
<pattern>
|
303
|
+
count_key_format item_sum_count_by_zset:%_{item_id}
|
304
|
+
count_zset_key_format %_{from}
|
305
|
+
</pattern>
|
306
|
+
]
|
307
|
+
end
|
308
|
+
|
309
|
+
driver = create_driver_for_test
|
310
|
+
|
311
|
+
time = Time.parse('2012-06-21 03:01:00 UTC').to_i
|
312
|
+
driver.emit({"item_id" => 200, "from" => "US"}, time)
|
313
|
+
driver.run
|
314
|
+
|
315
|
+
assert_equal 1, driver.instance.redis.zscore("item_sum_count_by_zset:200", "US")
|
316
|
+
|
317
|
+
driver = create_driver_for_test
|
318
|
+
|
319
|
+
driver.emit({"item_id" => 200, "from" => "CHINA"}, time)
|
320
|
+
driver.run
|
321
|
+
|
322
|
+
assert_equal 1, driver.instance.redis.zscore("item_sum_count_by_zset:200", "CHINA")
|
323
|
+
assert_equal 1, driver.instance.redis.zscore("item_sum_count_by_zset:200", "US")
|
324
|
+
|
325
|
+
driver = create_driver_for_test
|
326
|
+
|
327
|
+
driver.emit({"item_id" => 200, "from" => "US"}, time)
|
328
|
+
driver.run
|
329
|
+
|
330
|
+
assert_equal 2, driver.instance.redis.zscore("item_sum_count_by_zset:200", "US")
|
331
|
+
end
|
332
|
+
|
333
|
+
|
334
|
+
def test_write_with_count_value_key
|
335
|
+
driver = create_driver %[
|
336
|
+
db_number 1
|
337
|
+
<pattern>
|
338
|
+
count_key_format item_sum_count:%_{item_id}
|
339
|
+
count_value_key count
|
340
|
+
</pattern>
|
341
|
+
]
|
342
|
+
|
343
|
+
time = Time.parse('2012-06-21 03:01:00 UTC').to_i
|
344
|
+
driver.emit({"item_id" => 200, "count" => 123}, time)
|
345
|
+
driver.run
|
346
|
+
|
347
|
+
assert_equal '123', driver.instance.redis.get("item_sum_count:200")
|
348
|
+
|
349
|
+
driver = create_driver %[
|
350
|
+
db_number 1
|
351
|
+
<pattern>
|
352
|
+
count_key_format item_sum_count:%_{item_id}
|
353
|
+
count_value_key count
|
354
|
+
</pattern>
|
355
|
+
]
|
356
|
+
driver.emit({"item_id" => 200, }, time)
|
357
|
+
driver.run
|
358
|
+
assert_equal '123', driver.instance.redis.get("item_sum_count:200"), "it should be ignore when count_value_key does not exists."
|
359
|
+
|
360
|
+
driver = create_driver %[
|
361
|
+
db_number 1
|
362
|
+
<pattern>
|
363
|
+
count_key_format item_sum_count:%_{item_id}
|
364
|
+
count_value_key count
|
365
|
+
</pattern>
|
366
|
+
]
|
367
|
+
driver.emit({"item_id" => 200, "count" => "111aaa"}, time)
|
368
|
+
driver.run
|
369
|
+
assert_equal '123', driver.instance.redis.get("item_sum_count:200"), "it should be ignore when count_value_key is not number"
|
370
|
+
end
|
371
|
+
|
372
|
+
end
|
metadata
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-redis-multi-type-counter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Jungtaek Lim
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-01-07 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.10.0
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.10.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: redis
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 2.2.2
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 2.2.2
|
41
|
+
description: fluent-plugin-redis-multi-type-counter is a fluent plugin to count-up/down
|
42
|
+
redis keys, hash keys, zset keys
|
43
|
+
email: kabhwan@gmail.com
|
44
|
+
executables: []
|
45
|
+
extensions: []
|
46
|
+
extra_rdoc_files: []
|
47
|
+
files:
|
48
|
+
- .gitignore
|
49
|
+
- .travis.yml
|
50
|
+
- CHANGELOG.md
|
51
|
+
- Gemfile
|
52
|
+
- README.md
|
53
|
+
- Rakefile
|
54
|
+
- VERSION
|
55
|
+
- benchmark/performance_test.rb
|
56
|
+
- fluent-plugin-redis-multi-type-counter.gemspec
|
57
|
+
- lib/fluent/plugin/out_redis_counter.rb
|
58
|
+
- test/plugin/out_redis_counter.rb
|
59
|
+
homepage: https://github.com/heartsavior/fluent-plugin-redis-multi-type-counter
|
60
|
+
licenses:
|
61
|
+
- APACHE2
|
62
|
+
metadata: {}
|
63
|
+
post_install_message:
|
64
|
+
rdoc_options: []
|
65
|
+
require_paths:
|
66
|
+
- lib
|
67
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - '>='
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
version: '0'
|
72
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
requirements: []
|
78
|
+
rubyforge_project:
|
79
|
+
rubygems_version: 2.0.3
|
80
|
+
signing_key:
|
81
|
+
specification_version: 4
|
82
|
+
summary: Redis multi type counter plugin for fluent
|
83
|
+
test_files:
|
84
|
+
- test/plugin/out_redis_counter.rb
|