fluent-plugin-redis 0.2.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -1
- data/CHANGELOG.md +7 -0
- data/README.rdoc +25 -1
- data/Rakefile +1 -1
- data/fluent-plugin-redis.gemspec +3 -3
- data/lib/fluent/plugin/out_redis.rb +62 -29
- data/test/plugin/test_out_redis.rb +117 -20
- metadata +12 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 76ed9616d363f33920c8a1dd8e4e5596f2a35902
|
4
|
+
data.tar.gz: 5784039d3c0b6f7ee7d4c7e28a054d54faf91220
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c31c34bbd4d1f441b463a256fbee00a34bee3886fbf24619ad7940685e16965db707864831a372195f63476bd6956e4294d26dde7dea6cc4a549f0dcfdee8951
|
7
|
+
data.tar.gz: e76f25044068976638e066fd2dfc55622cb02bdfb2a1fff5048cd1d8909c8ade7e1250fe942d00d414720d47fc7332fcbc2da28618974a4e0b9ffe600fa3c5c1
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 0.3.0
|
2
|
+
|
3
|
+
* Migrate v0.14 API based plugin
|
4
|
+
* Allow duplicate insert key to support update values functionality
|
5
|
+
* Use v0.14's built-in placeholder functionality
|
6
|
+
* Enabled to specify more flexible tag and time format for identifier
|
7
|
+
|
1
8
|
## 0.2.2
|
2
9
|
|
3
10
|
* Use redis-rb 3.2.x
|
data/README.rdoc
CHANGED
@@ -2,6 +2,16 @@
|
|
2
2
|
|
3
3
|
fluent-plugin-redis is a fluent plugin to output to redis.
|
4
4
|
|
5
|
+
== Requirements
|
6
|
+
|
7
|
+
|fluent-plugin-redis| fluentd | ruby |
|
8
|
+
|-------------------|------------------|--------|
|
9
|
+
| >= 0.3.0 | >= 0.14.8 | >= 2.1 |
|
10
|
+
| == 0.2.3 | ~> 0.12.0 * | >= 1.9 |
|
11
|
+
| < 0.2.3 | >= 0.10.0, < 2 * | >= 1.9 |
|
12
|
+
|
13
|
+
* May not support all future fluentd features
|
14
|
+
|
5
15
|
== Installation
|
6
16
|
|
7
17
|
What you have to do is only installing like this:
|
@@ -13,7 +23,7 @@ Then fluent automatically loads the plugin installed.
|
|
13
23
|
== Configuration
|
14
24
|
|
15
25
|
<match redis.**>
|
16
|
-
type redis
|
26
|
+
@type redis
|
17
27
|
|
18
28
|
host localhost
|
19
29
|
port 6379
|
@@ -22,9 +32,23 @@ Then fluent automatically loads the plugin installed.
|
|
22
32
|
db_number 0 # 0 is default
|
23
33
|
# If requirepass is set, please specify this.
|
24
34
|
# password hogefuga
|
35
|
+
# Users can set '${tag}' or ${tag[0]}.${tag[1]} or ...?
|
36
|
+
# insert_key_prefix '${tag}'
|
37
|
+
# Users can set strftime format.
|
38
|
+
# "%s" will be replaced into unixtime.
|
39
|
+
# "%Y%m%d.%H%M%S" will be replaced like as 20161202.112335.
|
40
|
+
# strftime_format "%s"
|
41
|
+
# Allow insert key duplicate. It will work as update values.
|
42
|
+
# allow_duplicate_key true
|
25
43
|
</match>
|
26
44
|
|
27
45
|
|
46
|
+
=== Notice
|
47
|
+
|
48
|
+
<em>insert_key_prefix</em>, <em>strftime_format</em>, and <em>allow_duplicate_key</em> are newly added config parameters.
|
49
|
+
|
50
|
+
They can use v0.3.0 or later. To use this parameters, users must update Fluentd to v0.14 or later and this plugin to v0.3.0 or later.
|
51
|
+
|
28
52
|
== Contributing to fluent-plugin-redis
|
29
53
|
|
30
54
|
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
|
data/Rakefile
CHANGED
data/fluent-plugin-redis.gemspec
CHANGED
@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-redis"
|
6
|
-
s.version = "0.
|
6
|
+
s.version = "0.3.0"
|
7
7
|
s.platform = Gem::Platform::RUBY
|
8
8
|
s.authors = ["Yuki Nishijima", "Hiroshi Hatake", "Kenji Okimoto"]
|
9
|
-
s.date = %q{2016-12-
|
9
|
+
s.date = %q{2016-12-06}
|
10
10
|
s.email = ["mail@yukinishijima.net", "fluent@clear-code.com"]
|
11
11
|
s.homepage = "http://github.com/yuki24/fluent-plugin-redis"
|
12
12
|
s.summary = "Redis output plugin for Fluent"
|
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
17
|
s.require_paths = ["lib"]
|
18
18
|
|
19
|
-
s.add_dependency %q<fluentd>, ["
|
19
|
+
s.add_dependency %q<fluentd>, [">= 0.14.8", "< 2"]
|
20
20
|
s.add_dependency %q<redis>, ["~> 3.3.0"]
|
21
21
|
s.add_development_dependency %q<rake>, [">= 11.3.0"]
|
22
22
|
s.add_development_dependency %q<bundler>
|
@@ -1,40 +1,53 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'redis'
|
2
|
+
require 'msgpack'
|
3
|
+
require 'fluent/msgpack_factory'
|
4
|
+
require 'fluent/plugin/output'
|
5
|
+
|
6
|
+
module Fluent::Plugin
|
7
|
+
class RedisOutput < Output
|
8
|
+
include Fluent::MessagePackFactory::Mixin
|
9
|
+
|
3
10
|
Fluent::Plugin.register_output('redis', self)
|
4
|
-
attr_reader :redis
|
5
11
|
|
6
|
-
|
7
|
-
config_param :port, :integer, :default => 6379
|
8
|
-
config_param :db_number, :integer, :default => 0
|
9
|
-
config_param :password, :string, :default => nil, :secret => true
|
12
|
+
helpers :compat_parameters, :inject
|
10
13
|
|
11
|
-
|
12
|
-
unless method_defined?(:log)
|
13
|
-
define_method("log") { $log }
|
14
|
-
end
|
14
|
+
DEFAULT_BUFFER_TYPE = "memory"
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
16
|
+
attr_reader :redis
|
17
|
+
|
18
|
+
config_param :host, :string, default: 'localhost'
|
19
|
+
config_param :port, :integer, default: 6379
|
20
|
+
config_param :db_number, :integer, default: 0
|
21
|
+
config_param :password, :string, default: nil, secret: true
|
22
|
+
config_param :insert_key_prefix, :string, default: "${tag}"
|
23
|
+
config_param :strftime_format, :string, default: "%s"
|
24
|
+
config_param :allow_duplicate_key, :bool, default: false
|
25
|
+
|
26
|
+
config_section :buffer do
|
27
|
+
config_set_default :@type, DEFAULT_BUFFER_TYPE
|
28
|
+
config_set_default :chunk_keys, ['tag', 'time']
|
29
|
+
config_set_default :timekey, 60
|
20
30
|
end
|
21
31
|
|
22
32
|
def configure(conf)
|
33
|
+
compat_parameters_convert(conf, :buffer, :inject)
|
23
34
|
super
|
24
35
|
|
25
36
|
if conf.has_key?('namespace')
|
26
37
|
log.warn "namespace option has been removed from fluent-plugin-redis 0.1.3. Please add or remove the namespace '#{conf['namespace']}' manually."
|
27
38
|
end
|
39
|
+
raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
|
40
|
+
raise Fluent::ConfigError, "'time' in chunk_keys is required." if not @chunk_key_time
|
28
41
|
end
|
29
42
|
|
30
43
|
def start
|
31
44
|
super
|
32
45
|
|
33
46
|
options = {
|
34
|
-
:
|
35
|
-
:
|
36
|
-
:
|
37
|
-
:
|
47
|
+
host: @host,
|
48
|
+
port: @port,
|
49
|
+
thread_safe: true,
|
50
|
+
db: @db_number
|
38
51
|
}
|
39
52
|
options[:password] = @password if @password
|
40
53
|
|
@@ -47,22 +60,42 @@ module Fluent
|
|
47
60
|
end
|
48
61
|
|
49
62
|
def format(tag, time, record)
|
50
|
-
|
51
|
-
[
|
63
|
+
record = inject_values_to_record(tag, time, record)
|
64
|
+
[tag, time, record].to_msgpack
|
65
|
+
end
|
66
|
+
|
67
|
+
def formatted_to_msgpack_binary
|
68
|
+
true
|
52
69
|
end
|
53
70
|
|
54
71
|
def write(chunk)
|
72
|
+
tag, time = expand_placeholders(chunk.metadata)
|
55
73
|
@redis.pipelined {
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
74
|
+
unless @allow_duplicate_key
|
75
|
+
chunk.open { |io|
|
76
|
+
begin
|
77
|
+
msgpack_unpacker(io).each.with_index { |record, index|
|
78
|
+
identifier = [tag, time].join(".")
|
79
|
+
@redis.mapped_hmset "#{identifier}.#{index}", record[2]
|
80
|
+
}
|
81
|
+
rescue EOFError
|
82
|
+
# EOFError always occured when reached end of chunk.
|
83
|
+
end
|
84
|
+
}
|
85
|
+
else
|
86
|
+
chunk.each do |_tag, _time, record|
|
87
|
+
@redis.mapped_hmset "#{tag}", record
|
63
88
|
end
|
64
|
-
|
89
|
+
end
|
65
90
|
}
|
66
91
|
end
|
92
|
+
|
93
|
+
private
|
94
|
+
|
95
|
+
def expand_placeholders(metadata)
|
96
|
+
tag = extract_placeholders(@insert_key_prefix, metadata)
|
97
|
+
time = extract_placeholders(@strftime_format, metadata)
|
98
|
+
return tag, time
|
99
|
+
end
|
67
100
|
end
|
68
101
|
end
|
@@ -1,20 +1,31 @@
|
|
1
1
|
require 'fluent/test'
|
2
|
+
require 'fluent/test/helpers'
|
3
|
+
require 'fluent/test/driver/output'
|
2
4
|
require 'fluent/plugin/out_redis'
|
5
|
+
require 'fluent/time' # Fluent::TimeFormatter
|
3
6
|
|
4
7
|
class FileOutputTest < Test::Unit::TestCase
|
8
|
+
include Fluent::Test::Helpers
|
9
|
+
|
10
|
+
CONFIG = %[
|
11
|
+
host localhost
|
12
|
+
port 6379
|
13
|
+
db_number 1
|
14
|
+
]
|
15
|
+
|
5
16
|
def setup
|
6
17
|
Fluent::Test.setup
|
7
18
|
|
8
|
-
@d = create_driver
|
9
|
-
|
10
|
-
port 6379
|
11
|
-
db_number 1
|
12
|
-
]
|
13
|
-
@time = Time.parse("2011-01-02 13:14:15 UTC").to_i
|
19
|
+
@d = create_driver
|
20
|
+
@time = event_time("2011-01-02 13:14:15 UTC")
|
14
21
|
end
|
15
22
|
|
16
23
|
def create_driver(conf = CONFIG)
|
17
|
-
Fluent::Test::
|
24
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::RedisOutput).configure(conf)
|
25
|
+
end
|
26
|
+
|
27
|
+
def time_formatter(inject_config)
|
28
|
+
Fluent::TimeFormatter.new(inject_config.time_format, inject_config.localtime, inject_config.timezone)
|
18
29
|
end
|
19
30
|
|
20
31
|
def test_configure
|
@@ -22,13 +33,13 @@ class FileOutputTest < Test::Unit::TestCase
|
|
22
33
|
assert_equal 6379, @d.instance.port
|
23
34
|
assert_equal 1, @d.instance.db_number
|
24
35
|
assert_nil @d.instance.password
|
36
|
+
assert_equal '${tag}', @d.instance.insert_key_prefix
|
37
|
+
assert_equal '%s', @d.instance.strftime_format
|
38
|
+
assert_false @d.instance.allow_duplicate_key
|
25
39
|
end
|
26
40
|
|
27
41
|
def test_configure_with_password
|
28
|
-
d = create_driver %[
|
29
|
-
host localhost
|
30
|
-
port 6379
|
31
|
-
db_number 1
|
42
|
+
d = create_driver CONFIG + %[
|
32
43
|
password testpass
|
33
44
|
]
|
34
45
|
assert_equal 'localhost', d.instance.host
|
@@ -37,18 +48,104 @@ class FileOutputTest < Test::Unit::TestCase
|
|
37
48
|
assert_equal 'testpass', d.instance.password
|
38
49
|
end
|
39
50
|
|
51
|
+
def test_configure_without_tag_chunk_key
|
52
|
+
config = config_element('ROOT', '', {
|
53
|
+
"host" => "localhost",
|
54
|
+
"port" => 6379,
|
55
|
+
"db_number" => 1,
|
56
|
+
}, [
|
57
|
+
config_element('buffer', 'time', {
|
58
|
+
'chunk_keys' => 'time',
|
59
|
+
})
|
60
|
+
])
|
61
|
+
assert_raise Fluent::ConfigError do
|
62
|
+
create_driver(config)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
40
66
|
def test_format
|
41
|
-
@d.
|
42
|
-
|
43
|
-
|
67
|
+
@d.run(default_tag: 'test') do
|
68
|
+
@d.feed(@time, {"a"=>1})
|
69
|
+
end
|
70
|
+
assert_equal [["test", @time, {"a"=>1}].to_msgpack], @d.formatted
|
71
|
+
end
|
72
|
+
|
73
|
+
class InjectTest < self
|
74
|
+
def test_format_inject_tag_keys
|
75
|
+
d = create_driver CONFIG + %[
|
76
|
+
include_tag_key true
|
77
|
+
]
|
78
|
+
d.run(default_tag: 'test') do
|
79
|
+
d.feed(@time, {"a"=>1})
|
80
|
+
end
|
81
|
+
assert_equal [["test", @time, {"a"=>1, "tag" => "test"}].to_msgpack], d.formatted
|
82
|
+
end
|
83
|
+
|
84
|
+
def test_format_inject_time_keys
|
85
|
+
d = create_driver CONFIG + %[
|
86
|
+
include_time_key true
|
87
|
+
]
|
88
|
+
timef = time_formatter(d.instance.inject_config)
|
89
|
+
d.run(default_tag: 'test') do
|
90
|
+
d.feed(@time, {"a"=>1})
|
91
|
+
end
|
92
|
+
assert_equal [["test", @time, {"a"=>1, "time" => timef.call(@time)}].to_msgpack], d.formatted
|
93
|
+
end
|
44
94
|
end
|
45
95
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
96
|
+
class WriteTest < self
|
97
|
+
def test_write
|
98
|
+
d = create_driver
|
99
|
+
time = event_time("2011-01-02 13:14:00 UTC")
|
100
|
+
d.run(default_tag: 'test') do
|
101
|
+
d.feed(time, {"a"=>2})
|
102
|
+
d.feed(time, {"a"=>3})
|
103
|
+
end
|
104
|
+
|
105
|
+
assert_equal "2", d.instance.redis.hget("test.#{time}.0", "a")
|
106
|
+
assert_equal "3", d.instance.redis.hget("test.#{time}.1", "a")
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_write_with_insert_key_prefix
|
110
|
+
d = create_driver CONFIG + %[
|
111
|
+
insert_key_prefix "${tag[1]}.${tag[2]}"
|
112
|
+
]
|
113
|
+
time = event_time("2011-01-02 13:14:00 UTC")
|
114
|
+
d.run(default_tag: 'prefix.insert.test') do
|
115
|
+
d.feed(time, {"a"=>2})
|
116
|
+
d.feed(time, {"a"=>3})
|
117
|
+
end
|
118
|
+
|
119
|
+
assert_equal "2", d.instance.redis.hget("insert.test.#{time}.0", "a")
|
120
|
+
assert_equal "3", d.instance.redis.hget("insert.test.#{time}.1", "a")
|
121
|
+
end
|
122
|
+
|
123
|
+
def test_write_with_custom_strftime_format
|
124
|
+
d = create_driver CONFIG + %[
|
125
|
+
strftime_format "%Y%m%d.%H%M%S"
|
126
|
+
]
|
127
|
+
time = event_time("2011-01-02 13:14:00 UTC")
|
128
|
+
strtime = Time.at(time.to_r).strftime("%Y%m%d.%H%M%S")
|
129
|
+
d.run(default_tag: 'test') do
|
130
|
+
d.feed(time, {"a"=>4})
|
131
|
+
d.feed(time, {"a"=>5})
|
132
|
+
end
|
133
|
+
|
134
|
+
assert_equal "4", d.instance.redis.hget("test.#{strtime}.0", "a")
|
135
|
+
assert_equal "5", d.instance.redis.hget("test.#{strtime}.1", "a")
|
136
|
+
end
|
137
|
+
|
138
|
+
def test_write_with_allow_duplicate
|
139
|
+
d = create_driver CONFIG + %[
|
140
|
+
allow_duplicate_key true
|
141
|
+
]
|
142
|
+
time = event_time("2011-01-02 13:14:00 UTC")
|
143
|
+
d.run(default_tag: 'test.duplicate') do
|
144
|
+
d.feed(time, {"a"=>6})
|
145
|
+
d.feed(time, {"a"=>7})
|
146
|
+
end
|
50
147
|
|
51
|
-
|
52
|
-
|
148
|
+
assert_equal "7", d.instance.redis.hget("test.duplicate", "a")
|
149
|
+
end
|
53
150
|
end
|
54
151
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-redis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yuki Nishijima
|
@@ -10,22 +10,28 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2016-12-
|
13
|
+
date: 2016-12-06 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
17
17
|
requirement: !ruby/object:Gem::Requirement
|
18
18
|
requirements:
|
19
|
-
- - "
|
19
|
+
- - ">="
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 0.14.8
|
22
|
+
- - "<"
|
20
23
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
24
|
+
version: '2'
|
22
25
|
type: :runtime
|
23
26
|
prerelease: false
|
24
27
|
version_requirements: !ruby/object:Gem::Requirement
|
25
28
|
requirements:
|
26
|
-
- - "
|
29
|
+
- - ">="
|
30
|
+
- !ruby/object:Gem::Version
|
31
|
+
version: 0.14.8
|
32
|
+
- - "<"
|
27
33
|
- !ruby/object:Gem::Version
|
28
|
-
version:
|
34
|
+
version: '2'
|
29
35
|
- !ruby/object:Gem::Dependency
|
30
36
|
name: redis
|
31
37
|
requirement: !ruby/object:Gem::Requirement
|