fluent-plugin-redis 0.2.3 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f837fbede1a7e35897c7f8dd4b6f525f026c4f98
4
- data.tar.gz: e382edd5215f8fcd18741da1dfff0586f889833d
3
+ metadata.gz: 76ed9616d363f33920c8a1dd8e4e5596f2a35902
4
+ data.tar.gz: 5784039d3c0b6f7ee7d4c7e28a054d54faf91220
5
5
  SHA512:
6
- metadata.gz: 43b836f1368931c4ed2076a739fac49dc847296c7dd94d321f3ae5e91658f14da8f2aa416288eb48a6a7f0cdbe7777ea17b38e1dbc4ad86d917ba6d73c7af8ae
7
- data.tar.gz: a1496462b70612c6eb5c35ab534b9c327603cc03b5e7507a05855fd0117904872fc9f562f3e565369b1a59e530b64c79e5ce58bd17eedd2fe76a498e0f27bf3a
6
+ metadata.gz: c31c34bbd4d1f441b463a256fbee00a34bee3886fbf24619ad7940685e16965db707864831a372195f63476bd6956e4294d26dde7dea6cc4a549f0dcfdee8951
7
+ data.tar.gz: e76f25044068976638e066fd2dfc55622cb02bdfb2a1fff5048cd1d8909c8ade7e1250fe942d00d414720d47fc7332fcbc2da28618974a4e0b9ffe600fa3c5c1
data/.travis.yml CHANGED
@@ -6,6 +6,7 @@ rvm:
6
6
  - 2.2
7
7
  - 2.3.3
8
8
  gemfile:
9
- - gemfiles/fluentd_v0.12.gemfile
9
+ - gemfiles/fluentd_v0.14.gemfile
10
+ - Gemfile
10
11
  before_install:
11
12
  - gem update bundler
data/CHANGELOG.md CHANGED
@@ -1,3 +1,10 @@
1
+ ## 0.3.0
2
+
3
+ * Migrate v0.14 API based plugin
4
+ * Allow duplicate insert key to support update values functionality
5
+ * Use v0.14's built-in placeholder functionality
6
+ * Enabled to specify more flexible tag and time format for identifier
7
+
1
8
  ## 0.2.2
2
9
 
3
10
  * Use redis-rb 3.2.x
data/README.rdoc CHANGED
@@ -2,6 +2,16 @@
2
2
 
3
3
  fluent-plugin-redis is a fluent plugin to output to redis.
4
4
 
5
+ == Requirements
6
+
7
+ |fluent-plugin-redis| fluentd | ruby |
8
+ |-------------------|------------------|--------|
9
+ | >= 0.3.0 | >= 0.14.8 | >= 2.1 |
10
+ | == 0.2.3 | ~> 0.12.0 * | >= 1.9 |
11
+ | < 0.2.3 | >= 0.10.0, < 2 * | >= 1.9 |
12
+
13
+ * May not support all future fluentd features
14
+
5
15
  == Installation
6
16
 
7
17
  What you have to do is only installing like this:
@@ -13,7 +23,7 @@ Then fluent automatically loads the plugin installed.
13
23
  == Configuration
14
24
 
15
25
  <match redis.**>
16
- type redis
26
+ @type redis
17
27
 
18
28
  host localhost
19
29
  port 6379
@@ -22,9 +32,23 @@ Then fluent automatically loads the plugin installed.
22
32
  db_number 0 # 0 is default
23
33
  # If requirepass is set, please specify this.
24
34
  # password hogefuga
35
+ # Users can set '${tag}' or ${tag[0]}.${tag[1]} or ...?
36
+ # insert_key_prefix '${tag}'
37
+ # Users can set strftime format.
38
+ # "%s" will be replaced into unixtime.
39
+ # "%Y%m%d.%H%M%S" will be replaced like as 20161202.112335.
40
+ # strftime_format "%s"
41
+ # Allow insert key duplicate. It will work as update values.
42
+ # allow_duplicate_key true
25
43
  </match>
26
44
 
27
45
 
46
+ === Notice
47
+
48
+ <em>insert_key_prefix</em>, <em>strftime_format</em>, and <em>allow_duplicate_key</em> are newly added config parameters.
49
+
50
+ They can use v0.3.0 or later. To use this parameters, users must update Fluentd to v0.14 or later and this plugin to v0.3.0 or later.
51
+
28
52
  == Contributing to fluent-plugin-redis
29
53
 
30
54
  * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
data/Rakefile CHANGED
@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |test|
7
7
  test.verbose = true
8
8
  end
9
9
 
10
- task :default => :test
10
+ task default: :test
@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-redis"
6
- s.version = "0.2.3"
6
+ s.version = "0.3.0"
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.authors = ["Yuki Nishijima", "Hiroshi Hatake", "Kenji Okimoto"]
9
- s.date = %q{2016-12-01}
9
+ s.date = %q{2016-12-06}
10
10
  s.email = ["mail@yukinishijima.net", "fluent@clear-code.com"]
11
11
  s.homepage = "http://github.com/yuki24/fluent-plugin-redis"
12
12
  s.summary = "Redis output plugin for Fluent"
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
16
16
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
17
  s.require_paths = ["lib"]
18
18
 
19
- s.add_dependency %q<fluentd>, ["~> 0.12.0"]
19
+ s.add_dependency %q<fluentd>, [">= 0.14.8", "< 2"]
20
20
  s.add_dependency %q<redis>, ["~> 3.3.0"]
21
21
  s.add_development_dependency %q<rake>, [">= 11.3.0"]
22
22
  s.add_development_dependency %q<bundler>
@@ -1,40 +1,53 @@
1
- module Fluent
2
- class RedisOutput < BufferedOutput
1
+ require 'redis'
2
+ require 'msgpack'
3
+ require 'fluent/msgpack_factory'
4
+ require 'fluent/plugin/output'
5
+
6
+ module Fluent::Plugin
7
+ class RedisOutput < Output
8
+ include Fluent::MessagePackFactory::Mixin
9
+
3
10
  Fluent::Plugin.register_output('redis', self)
4
- attr_reader :redis
5
11
 
6
- config_param :host, :string, :default => 'localhost'
7
- config_param :port, :integer, :default => 6379
8
- config_param :db_number, :integer, :default => 0
9
- config_param :password, :string, :default => nil, :secret => true
12
+ helpers :compat_parameters, :inject
10
13
 
11
- # To support log_level option implemented by Fluentd v0.10.43
12
- unless method_defined?(:log)
13
- define_method("log") { $log }
14
- end
14
+ DEFAULT_BUFFER_TYPE = "memory"
15
15
 
16
- def initialize
17
- super
18
- require 'redis'
19
- require 'msgpack'
16
+ attr_reader :redis
17
+
18
+ config_param :host, :string, default: 'localhost'
19
+ config_param :port, :integer, default: 6379
20
+ config_param :db_number, :integer, default: 0
21
+ config_param :password, :string, default: nil, secret: true
22
+ config_param :insert_key_prefix, :string, default: "${tag}"
23
+ config_param :strftime_format, :string, default: "%s"
24
+ config_param :allow_duplicate_key, :bool, default: false
25
+
26
+ config_section :buffer do
27
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
28
+ config_set_default :chunk_keys, ['tag', 'time']
29
+ config_set_default :timekey, 60
20
30
  end
21
31
 
22
32
  def configure(conf)
33
+ compat_parameters_convert(conf, :buffer, :inject)
23
34
  super
24
35
 
25
36
  if conf.has_key?('namespace')
26
37
  log.warn "namespace option has been removed from fluent-plugin-redis 0.1.3. Please add or remove the namespace '#{conf['namespace']}' manually."
27
38
  end
39
+ raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
40
+ raise Fluent::ConfigError, "'time' in chunk_keys is required." if not @chunk_key_time
28
41
  end
29
42
 
30
43
  def start
31
44
  super
32
45
 
33
46
  options = {
34
- :host => @host,
35
- :port => @port,
36
- :thread_safe => true,
37
- :db => @db_number
47
+ host: @host,
48
+ port: @port,
49
+ thread_safe: true,
50
+ db: @db_number
38
51
  }
39
52
  options[:password] = @password if @password
40
53
 
@@ -47,22 +60,42 @@ module Fluent
47
60
  end
48
61
 
49
62
  def format(tag, time, record)
50
- identifier = [tag, time].join(".")
51
- [identifier, record].to_msgpack
63
+ record = inject_values_to_record(tag, time, record)
64
+ [tag, time, record].to_msgpack
65
+ end
66
+
67
+ def formatted_to_msgpack_binary
68
+ true
52
69
  end
53
70
 
54
71
  def write(chunk)
72
+ tag, time = expand_placeholders(chunk.metadata)
55
73
  @redis.pipelined {
56
- chunk.open { |io|
57
- begin
58
- MessagePack::Unpacker.new(io).each.each_with_index { |record, index|
59
- @redis.mapped_hmset "#{record[0]}.#{index}", record[1]
60
- }
61
- rescue EOFError
62
- # EOFError always occured when reached end of chunk.
74
+ unless @allow_duplicate_key
75
+ chunk.open { |io|
76
+ begin
77
+ msgpack_unpacker(io).each.with_index { |record, index|
78
+ identifier = [tag, time].join(".")
79
+ @redis.mapped_hmset "#{identifier}.#{index}", record[2]
80
+ }
81
+ rescue EOFError
82
+ # EOFError always occured when reached end of chunk.
83
+ end
84
+ }
85
+ else
86
+ chunk.each do |_tag, _time, record|
87
+ @redis.mapped_hmset "#{tag}", record
63
88
  end
64
- }
89
+ end
65
90
  }
66
91
  end
92
+
93
+ private
94
+
95
+ def expand_placeholders(metadata)
96
+ tag = extract_placeholders(@insert_key_prefix, metadata)
97
+ time = extract_placeholders(@strftime_format, metadata)
98
+ return tag, time
99
+ end
67
100
  end
68
101
  end
@@ -1,20 +1,31 @@
1
1
  require 'fluent/test'
2
+ require 'fluent/test/helpers'
3
+ require 'fluent/test/driver/output'
2
4
  require 'fluent/plugin/out_redis'
5
+ require 'fluent/time' # Fluent::TimeFormatter
3
6
 
4
7
  class FileOutputTest < Test::Unit::TestCase
8
+ include Fluent::Test::Helpers
9
+
10
+ CONFIG = %[
11
+ host localhost
12
+ port 6379
13
+ db_number 1
14
+ ]
15
+
5
16
  def setup
6
17
  Fluent::Test.setup
7
18
 
8
- @d = create_driver %[
9
- host localhost
10
- port 6379
11
- db_number 1
12
- ]
13
- @time = Time.parse("2011-01-02 13:14:15 UTC").to_i
19
+ @d = create_driver
20
+ @time = event_time("2011-01-02 13:14:15 UTC")
14
21
  end
15
22
 
16
23
  def create_driver(conf = CONFIG)
17
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisOutput).configure(conf)
24
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::RedisOutput).configure(conf)
25
+ end
26
+
27
+ def time_formatter(inject_config)
28
+ Fluent::TimeFormatter.new(inject_config.time_format, inject_config.localtime, inject_config.timezone)
18
29
  end
19
30
 
20
31
  def test_configure
@@ -22,13 +33,13 @@ class FileOutputTest < Test::Unit::TestCase
22
33
  assert_equal 6379, @d.instance.port
23
34
  assert_equal 1, @d.instance.db_number
24
35
  assert_nil @d.instance.password
36
+ assert_equal '${tag}', @d.instance.insert_key_prefix
37
+ assert_equal '%s', @d.instance.strftime_format
38
+ assert_false @d.instance.allow_duplicate_key
25
39
  end
26
40
 
27
41
  def test_configure_with_password
28
- d = create_driver %[
29
- host localhost
30
- port 6379
31
- db_number 1
42
+ d = create_driver CONFIG + %[
32
43
  password testpass
33
44
  ]
34
45
  assert_equal 'localhost', d.instance.host
@@ -37,18 +48,104 @@ class FileOutputTest < Test::Unit::TestCase
37
48
  assert_equal 'testpass', d.instance.password
38
49
  end
39
50
 
51
+ def test_configure_without_tag_chunk_key
52
+ config = config_element('ROOT', '', {
53
+ "host" => "localhost",
54
+ "port" => 6379,
55
+ "db_number" => 1,
56
+ }, [
57
+ config_element('buffer', 'time', {
58
+ 'chunk_keys' => 'time',
59
+ })
60
+ ])
61
+ assert_raise Fluent::ConfigError do
62
+ create_driver(config)
63
+ end
64
+ end
65
+
40
66
  def test_format
41
- @d.emit({"a"=>1}, @time)
42
- @d.expect_format(["test.#{@time}", {"a"=>1}].to_msgpack)
43
- @d.run
67
+ @d.run(default_tag: 'test') do
68
+ @d.feed(@time, {"a"=>1})
69
+ end
70
+ assert_equal [["test", @time, {"a"=>1}].to_msgpack], @d.formatted
71
+ end
72
+
73
+ class InjectTest < self
74
+ def test_format_inject_tag_keys
75
+ d = create_driver CONFIG + %[
76
+ include_tag_key true
77
+ ]
78
+ d.run(default_tag: 'test') do
79
+ d.feed(@time, {"a"=>1})
80
+ end
81
+ assert_equal [["test", @time, {"a"=>1, "tag" => "test"}].to_msgpack], d.formatted
82
+ end
83
+
84
+ def test_format_inject_time_keys
85
+ d = create_driver CONFIG + %[
86
+ include_time_key true
87
+ ]
88
+ timef = time_formatter(d.instance.inject_config)
89
+ d.run(default_tag: 'test') do
90
+ d.feed(@time, {"a"=>1})
91
+ end
92
+ assert_equal [["test", @time, {"a"=>1, "time" => timef.call(@time)}].to_msgpack], d.formatted
93
+ end
44
94
  end
45
95
 
46
- def test_write
47
- @d.emit({"a"=>2}, @time)
48
- @d.emit({"a"=>3}, @time)
49
- @d.run
96
+ class WriteTest < self
97
+ def test_write
98
+ d = create_driver
99
+ time = event_time("2011-01-02 13:14:00 UTC")
100
+ d.run(default_tag: 'test') do
101
+ d.feed(time, {"a"=>2})
102
+ d.feed(time, {"a"=>3})
103
+ end
104
+
105
+ assert_equal "2", d.instance.redis.hget("test.#{time}.0", "a")
106
+ assert_equal "3", d.instance.redis.hget("test.#{time}.1", "a")
107
+ end
108
+
109
+ def test_write_with_insert_key_prefix
110
+ d = create_driver CONFIG + %[
111
+ insert_key_prefix "${tag[1]}.${tag[2]}"
112
+ ]
113
+ time = event_time("2011-01-02 13:14:00 UTC")
114
+ d.run(default_tag: 'prefix.insert.test') do
115
+ d.feed(time, {"a"=>2})
116
+ d.feed(time, {"a"=>3})
117
+ end
118
+
119
+ assert_equal "2", d.instance.redis.hget("insert.test.#{time}.0", "a")
120
+ assert_equal "3", d.instance.redis.hget("insert.test.#{time}.1", "a")
121
+ end
122
+
123
+ def test_write_with_custom_strftime_format
124
+ d = create_driver CONFIG + %[
125
+ strftime_format "%Y%m%d.%H%M%S"
126
+ ]
127
+ time = event_time("2011-01-02 13:14:00 UTC")
128
+ strtime = Time.at(time.to_r).strftime("%Y%m%d.%H%M%S")
129
+ d.run(default_tag: 'test') do
130
+ d.feed(time, {"a"=>4})
131
+ d.feed(time, {"a"=>5})
132
+ end
133
+
134
+ assert_equal "4", d.instance.redis.hget("test.#{strtime}.0", "a")
135
+ assert_equal "5", d.instance.redis.hget("test.#{strtime}.1", "a")
136
+ end
137
+
138
+ def test_write_with_allow_duplicate
139
+ d = create_driver CONFIG + %[
140
+ allow_duplicate_key true
141
+ ]
142
+ time = event_time("2011-01-02 13:14:00 UTC")
143
+ d.run(default_tag: 'test.duplicate') do
144
+ d.feed(time, {"a"=>6})
145
+ d.feed(time, {"a"=>7})
146
+ end
50
147
 
51
- assert_equal "2", @d.instance.redis.hget("test.#{@time}.0", "a")
52
- assert_equal "3", @d.instance.redis.hget("test.#{@time}.1", "a")
148
+ assert_equal "7", d.instance.redis.hget("test.duplicate", "a")
149
+ end
53
150
  end
54
151
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-redis
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuki Nishijima
@@ -10,22 +10,28 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2016-12-01 00:00:00.000000000 Z
13
+ date: 2016-12-06 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
17
17
  requirement: !ruby/object:Gem::Requirement
18
18
  requirements:
19
- - - "~>"
19
+ - - ">="
20
+ - !ruby/object:Gem::Version
21
+ version: 0.14.8
22
+ - - "<"
20
23
  - !ruby/object:Gem::Version
21
- version: 0.12.0
24
+ version: '2'
22
25
  type: :runtime
23
26
  prerelease: false
24
27
  version_requirements: !ruby/object:Gem::Requirement
25
28
  requirements:
26
- - - "~>"
29
+ - - ">="
30
+ - !ruby/object:Gem::Version
31
+ version: 0.14.8
32
+ - - "<"
27
33
  - !ruby/object:Gem::Version
28
- version: 0.12.0
34
+ version: '2'
29
35
  - !ruby/object:Gem::Dependency
30
36
  name: redis
31
37
  requirement: !ruby/object:Gem::Requirement