fluent-plugin-redisstore 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +15 -0
- data/AUTHORS +1 -0
- data/Gemfile +3 -0
- data/README.rdoc +35 -0
- data/Rakefile +11 -0
- data/fluent-plugin-redisstore.gemspec +19 -0
- data/lib/fluent/plugin/out_redisstore.rb +202 -0
- metadata +91 -0
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
Mzk2MzIxZmZkY2QwZjcxZjdiY2E4MmE4MDJmZjVjM2EzYmRjNDJjZA==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
ODAzYTRmZWVmY2U4NGM0OGM5MzE5MDQ5OGM0MjRkMTAwZjVjYWFiZg==
|
7
|
+
!binary "U0hBNTEy":
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
NzU0YWQzOGQwYzU1NzhhNWVjY2VhMGFiNzJhYzFiOTA4YjU4NWEyZDRmNzdi
|
10
|
+
MDhhMjBhOTZmMzFlNzM3MTczNzY2M2MwNGQzMDc1ZDUzMTlkZWFkY2UxYmZi
|
11
|
+
MWEzYTNjNDAwOGEyNmMzZjExZWU4N2IwN2ZmMzRmYzViNDNlMDE=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
NTU1NmZjOTIxNzdmODhhZTY5MmJhZDJiYTY0ODg1ZGY2ZWI4N2UyMTNjNGI3
|
14
|
+
YjE1YWNhZjQ1MDgzOGQxYWU1Nzk2MzYyMDZjOWIwMDc1ODk4ZGE1OTgyMTQ0
|
15
|
+
MThjNzYyY2E4MTkzN2I5YjJiMDdhMzEyNTUwYTJmNzQyMGY1NTE=
|
data/AUTHORS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
moaikids
|
data/Gemfile
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
= Redis output plugin for Fluent event collector
|
2
|
+
|
3
|
+
== Overview
|
4
|
+
|
5
|
+
*redisstore* output plugin buffers event logs in local file and upload it to redis storage.
|
6
|
+
|
7
|
+
this sources folked 'fluent-plugin-redis'. (https://github.com/yuki24/fluent-plugin-redis)
|
8
|
+
|
9
|
+
== Installation
|
10
|
+
|
11
|
+
== Configuration
|
12
|
+
|
13
|
+
<match pattern>
|
14
|
+
type redisstore
|
15
|
+
host REDIS_HOST_ADDRESS(string, all , localhost)
|
16
|
+
port REDIS_PORT(int, all, 6379)
|
17
|
+
db_number REDIS_DB_NUMBER(int, all , nil)
|
18
|
+
timeout TIMEOUT(float, all , 5.0)
|
19
|
+
key_prefix prefix_(string, all, '')
|
20
|
+
key_suffix _suffix(string, all , '')
|
21
|
+
store_type zset(zset|set|list|string, all, zset)
|
22
|
+
key_name key(string, all, nil)
|
23
|
+
score_name score(string, all , nil)
|
24
|
+
value_name value(string, all, nill)
|
25
|
+
key_expire 604800(int, all, nil)
|
26
|
+
value_expire 86400(int , zset, nil)
|
27
|
+
value_length 100(int, zset|list, nil)
|
28
|
+
order asc(asc|desc, zset/list, nil)
|
29
|
+
</match>
|
30
|
+
|
31
|
+
== Copyright
|
32
|
+
|
33
|
+
Copyright:: Copyright (c) 2013 moaikids
|
34
|
+
License:: Apache License, Version 2.0
|
35
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
Gem::Specification.new do |gem|
|
3
|
+
gem.name = "fluent-plugin-redisstore"
|
4
|
+
gem.version = "0.0.1"
|
5
|
+
gem.authors = ["moaikids"]
|
6
|
+
gem.licenses = ["Apache License Version 2.0"]
|
7
|
+
gem.summary = %q{Redis(zset/set/list/string) output plugin for Fluentd}
|
8
|
+
gem.description = %q{Redis(zset/set/list/string) output plugin for Fluentd...}
|
9
|
+
gem.homepage = "https://github.com/moaikids/fluent-plugin-redisstore"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.require_paths = ["lib"]
|
15
|
+
|
16
|
+
gem.add_development_dependency "rake"
|
17
|
+
gem.add_development_dependency "shoulda"
|
18
|
+
gem.add_runtime_dependency "fluentd"
|
19
|
+
end
|
@@ -0,0 +1,202 @@
|
|
1
|
+
module Fluent
|
2
|
+
class RedisOutput < BufferedOutput
|
3
|
+
Fluent::Plugin.register_output('redisstore', self)
|
4
|
+
attr_reader :host, :port, :db_number, :redis, :timeout, :key_prefix, :key_suffix, :store_type, :key_name, :fixed_key_value, :score_name, :value_name, :key_expire, :value_expire, :value_length, :order
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
super
|
8
|
+
require 'redis'
|
9
|
+
require 'msgpack'
|
10
|
+
end
|
11
|
+
|
12
|
+
def configure(conf)
|
13
|
+
super
|
14
|
+
|
15
|
+
@host = conf.has_key?('host') ? conf['host'] : 'localhost'
|
16
|
+
@port = conf.has_key?('port') ? conf['port'].to_i : 6379
|
17
|
+
@db_number = conf.has_key?('db_number') ? conf['db_number'].to_i : nil
|
18
|
+
@timeout = conf.has_key?('timeout') ? conf['timeout'].to_f : 5.0
|
19
|
+
|
20
|
+
@key_prefix = conf.has_key?('key_prefix') ? conf['key_prefix'] : ''
|
21
|
+
@key_suffix = conf.has_key?('key_suffix') ? conf['key_suffix'] : ''
|
22
|
+
@store_type = conf.has_key?('store_type') ? conf['store_type'] : 'zset'
|
23
|
+
@key_name = conf['key_name']
|
24
|
+
@fixed_key_value = conf.has_key?('fixed_key_value') ? conf['fixed_key_value'] : nil
|
25
|
+
@score_name = conf['score_name']
|
26
|
+
@value_name = conf['value_name']
|
27
|
+
@key_expire = conf.has_key?('key_expire') ? conf['key_expire'].to_i : -1
|
28
|
+
@value_expire = conf.has_key?('value_expire') ? conf['value_expire'].to_i : -1
|
29
|
+
@value_length = conf.has_key?('value_length') ? conf['value_length'].to_i : -1
|
30
|
+
@order = conf.has_key?('order') ? conf['order'] : 'asc'
|
31
|
+
end
|
32
|
+
|
33
|
+
def start
|
34
|
+
super
|
35
|
+
@redis = Redis.new(:host => @host, :port => @port, :timeout => @timeout,
|
36
|
+
:thread_safe => true, :db => @db_number)
|
37
|
+
end
|
38
|
+
|
39
|
+
def shutdown
|
40
|
+
@redis.quit
|
41
|
+
end
|
42
|
+
|
43
|
+
def format(tag, time, record)
|
44
|
+
identifier = [tag, time].join(".")
|
45
|
+
[identifier, record].to_msgpack
|
46
|
+
end
|
47
|
+
|
48
|
+
def write(chunk)
|
49
|
+
@redis.pipelined {
|
50
|
+
chunk.open { |io|
|
51
|
+
begin
|
52
|
+
MessagePack::Unpacker.new(io).each { |message|
|
53
|
+
begin
|
54
|
+
(tag, record) = message
|
55
|
+
if @store_type == 'zset'
|
56
|
+
operation_for_zset(record)
|
57
|
+
elsif @store_type == 'set'
|
58
|
+
operation_for_set(record)
|
59
|
+
elsif @store_type == 'list'
|
60
|
+
operation_for_list(record)
|
61
|
+
elsif @store_type == 'string'
|
62
|
+
operation_for_string(record)
|
63
|
+
end
|
64
|
+
rescue NoMethodError => e
|
65
|
+
puts e
|
66
|
+
end
|
67
|
+
}
|
68
|
+
rescue EOFError
|
69
|
+
# EOFError always occured when reached end of chunk.
|
70
|
+
end
|
71
|
+
}
|
72
|
+
}
|
73
|
+
end
|
74
|
+
|
75
|
+
def operation_for_zset(record)
|
76
|
+
now = Time.now.to_i
|
77
|
+
if @fixed_key_value
|
78
|
+
k = @fixed_key_value
|
79
|
+
else
|
80
|
+
k = traverse(record, @key_name).to_s
|
81
|
+
end
|
82
|
+
if @score_name
|
83
|
+
s = traverse(record, @score_name)
|
84
|
+
else
|
85
|
+
s = now
|
86
|
+
end
|
87
|
+
v = traverse(record, @value_name)
|
88
|
+
sk = @key_prefix + k + @key_suffix
|
89
|
+
|
90
|
+
@redis.zadd sk , s, v
|
91
|
+
if @key_expire > 0
|
92
|
+
@redis.expire sk , @key_expire
|
93
|
+
end
|
94
|
+
if @value_expire > 0
|
95
|
+
@redis.zremrangebyscore sk , '-inf' , (now - @value_expire)
|
96
|
+
end
|
97
|
+
if @value_length > 0
|
98
|
+
script = generate_zremrangebyrank_script(sk, @value_length, @order)
|
99
|
+
@redis.eval script
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def operation_for_set(record)
|
104
|
+
if @fixed_key_value
|
105
|
+
k = @fixed_key_value
|
106
|
+
else
|
107
|
+
k = traverse(record, @key_name).to_s
|
108
|
+
end
|
109
|
+
v = traverse(record, @value_name)
|
110
|
+
sk = @key_prefix + k + @key_suffix
|
111
|
+
|
112
|
+
@redis.sadd sk, v
|
113
|
+
if @key_expire > 0
|
114
|
+
@redis.expire sk, @key_expire
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def operation_for_list(record)
|
119
|
+
if @fixed_key_value
|
120
|
+
k = @fixed_key_value
|
121
|
+
else
|
122
|
+
k = traverse(record, @key_name).to_s
|
123
|
+
end
|
124
|
+
v = traverse(record, @value_name)
|
125
|
+
sk = @key_prefix + k + @key_suffix
|
126
|
+
|
127
|
+
if @order == 'asc'
|
128
|
+
@redis.rpush sk, v
|
129
|
+
else
|
130
|
+
@redis.lpush sk, v
|
131
|
+
end
|
132
|
+
if @key_expire > 0
|
133
|
+
@redis.expire sk, @key_expire
|
134
|
+
end
|
135
|
+
if @value_length > 0
|
136
|
+
script = generate_ltrim_script(sk, @value_length, @order)
|
137
|
+
@redis.eval script
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
def operation_for_string(record)
|
142
|
+
if @fixed_key_value
|
143
|
+
k = @fixed_key_value
|
144
|
+
else
|
145
|
+
k = traverse(record, @key_name).to_s
|
146
|
+
end
|
147
|
+
v = traverse(record, @value_name)
|
148
|
+
sk = @key_prefix + k + @key_suffix
|
149
|
+
|
150
|
+
@redis.set sk, v
|
151
|
+
if @key_expire > 0
|
152
|
+
@redis.expire sk, @key_expire
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def generate_zremrangebyrank_script(key, maxlen, order)
|
157
|
+
script = "local key = '" + key.to_s + "'\n"
|
158
|
+
script += "local maxlen = " + maxlen.to_s + "\n"
|
159
|
+
script += "local order ='" + order.to_s + "'\n"
|
160
|
+
script += "local len = tonumber(redis.call('ZCOUNT', key, '-inf', '+inf'))\n"
|
161
|
+
script += "if len > maxlen then\n"
|
162
|
+
script += " if order == 'asc' then\n"
|
163
|
+
script += " local l = len - maxlen\n"
|
164
|
+
script += " if l >= 0 then\n"
|
165
|
+
script += " return redis.call('ZREMRANGEBYRANK', key, 0, l)\n"
|
166
|
+
script += " end\n"
|
167
|
+
script += " else\n"
|
168
|
+
script += " return redis.call('ZREMRANGEBYRANK', key, maxlen, -1)\n"
|
169
|
+
script += " end\n"
|
170
|
+
script += "end\n"
|
171
|
+
return script
|
172
|
+
end
|
173
|
+
|
174
|
+
def generate_ltrim_script(key, maxlen, order)
|
175
|
+
script = "local key = '" + key.to_s + "'\n"
|
176
|
+
script += "local maxlen = " + maxlen.to_s + "\n"
|
177
|
+
script += "local order ='" + order.to_s + "'\n"
|
178
|
+
script += "local len = tonumber(redis.call('LLEN', key))\n"
|
179
|
+
script += "if len > maxlen then\n"
|
180
|
+
script += " if order == 'asc' then\n"
|
181
|
+
script += " local l = len - maxlen\n"
|
182
|
+
script += " return redis.call('LTRIM', key, l, -1)\n"
|
183
|
+
script += " else\n"
|
184
|
+
script += " return redis.call('LTRIM', key, 0, maxlen - 1)\n"
|
185
|
+
script += " end\n"
|
186
|
+
script += "end\n"
|
187
|
+
return script
|
188
|
+
end
|
189
|
+
|
190
|
+
def traverse(data, key)
|
191
|
+
val = data
|
192
|
+
key.split('.').each{ |k|
|
193
|
+
if val.has_key?(k)
|
194
|
+
val = val[k]
|
195
|
+
else
|
196
|
+
return nil
|
197
|
+
end
|
198
|
+
}
|
199
|
+
return val
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
metadata
ADDED
@@ -0,0 +1,91 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-redisstore
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- moaikids
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-08-31 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ! '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: shoulda
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ! '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: fluentd
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ! '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description: Redis(zset/set/list/string) output plugin for Fluentd...
|
56
|
+
email:
|
57
|
+
executables: []
|
58
|
+
extensions: []
|
59
|
+
extra_rdoc_files: []
|
60
|
+
files:
|
61
|
+
- AUTHORS
|
62
|
+
- Gemfile
|
63
|
+
- README.rdoc
|
64
|
+
- Rakefile
|
65
|
+
- fluent-plugin-redisstore.gemspec
|
66
|
+
- lib/fluent/plugin/out_redisstore.rb
|
67
|
+
homepage: https://github.com/moaikids/fluent-plugin-redisstore
|
68
|
+
licenses:
|
69
|
+
- Apache License Version 2.0
|
70
|
+
metadata: {}
|
71
|
+
post_install_message:
|
72
|
+
rdoc_options: []
|
73
|
+
require_paths:
|
74
|
+
- lib
|
75
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ! '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - ! '>='
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '0'
|
85
|
+
requirements: []
|
86
|
+
rubyforge_project:
|
87
|
+
rubygems_version: 2.0.7
|
88
|
+
signing_key:
|
89
|
+
specification_version: 4
|
90
|
+
summary: Redis(zset/set/list/string) output plugin for Fluentd
|
91
|
+
test_files: []
|