fluent-plugin-redis 0.2.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f837fbede1a7e35897c7f8dd4b6f525f026c4f98
4
- data.tar.gz: e382edd5215f8fcd18741da1dfff0586f889833d
3
+ metadata.gz: 76ed9616d363f33920c8a1dd8e4e5596f2a35902
4
+ data.tar.gz: 5784039d3c0b6f7ee7d4c7e28a054d54faf91220
5
5
  SHA512:
6
- metadata.gz: 43b836f1368931c4ed2076a739fac49dc847296c7dd94d321f3ae5e91658f14da8f2aa416288eb48a6a7f0cdbe7777ea17b38e1dbc4ad86d917ba6d73c7af8ae
7
- data.tar.gz: a1496462b70612c6eb5c35ab534b9c327603cc03b5e7507a05855fd0117904872fc9f562f3e565369b1a59e530b64c79e5ce58bd17eedd2fe76a498e0f27bf3a
6
+ metadata.gz: c31c34bbd4d1f441b463a256fbee00a34bee3886fbf24619ad7940685e16965db707864831a372195f63476bd6956e4294d26dde7dea6cc4a549f0dcfdee8951
7
+ data.tar.gz: e76f25044068976638e066fd2dfc55622cb02bdfb2a1fff5048cd1d8909c8ade7e1250fe942d00d414720d47fc7332fcbc2da28618974a4e0b9ffe600fa3c5c1
data/.travis.yml CHANGED
@@ -6,6 +6,7 @@ rvm:
6
6
  - 2.2
7
7
  - 2.3.3
8
8
  gemfile:
9
- - gemfiles/fluentd_v0.12.gemfile
9
+ - gemfiles/fluentd_v0.14.gemfile
10
+ - Gemfile
10
11
  before_install:
11
12
  - gem update bundler
data/CHANGELOG.md CHANGED
@@ -1,3 +1,10 @@
1
+ ## 0.3.0
2
+
3
+ * Migrate v0.14 API based plugin
4
+ * Allow duplicate insert key to support update values functionality
5
+ * Use v0.14's built-in placeholder functionality
6
+ * Enabled to specify more flexible tag and time format for identifier
7
+
1
8
  ## 0.2.2
2
9
 
3
10
  * Use redis-rb 3.2.x
data/README.rdoc CHANGED
@@ -2,6 +2,16 @@
2
2
 
3
3
  fluent-plugin-redis is a fluent plugin to output to redis.
4
4
 
5
+ == Requirements
6
+
7
+ |fluent-plugin-redis| fluentd | ruby |
8
+ |-------------------|------------------|--------|
9
+ | >= 0.3.0 | >= 0.14.8 | >= 2.1 |
10
+ | == 0.2.3 | ~> 0.12.0 * | >= 1.9 |
11
+ | < 0.2.3 | >= 0.10.0, < 2 * | >= 1.9 |
12
+
13
+ * May not support all future fluentd features
14
+
5
15
  == Installation
6
16
 
7
17
  What you have to do is only installing like this:
@@ -13,7 +23,7 @@ Then fluent automatically loads the plugin installed.
13
23
  == Configuration
14
24
 
15
25
  <match redis.**>
16
- type redis
26
+ @type redis
17
27
 
18
28
  host localhost
19
29
  port 6379
@@ -22,9 +32,23 @@ Then fluent automatically loads the plugin installed.
22
32
  db_number 0 # 0 is default
23
33
  # If requirepass is set, please specify this.
24
34
  # password hogefuga
35
+ # Users can set '${tag}' or ${tag[0]}.${tag[1]} or ...?
36
+ # insert_key_prefix '${tag}'
37
+ # Users can set strftime format.
38
+ # "%s" will be replaced into unixtime.
39
+ # "%Y%m%d.%H%M%S" will be replaced like as 20161202.112335.
40
+ # strftime_format "%s"
41
+ # Allow insert key duplicate. It will work as update values.
42
+ # allow_duplicate_key true
25
43
  </match>
26
44
 
27
45
 
46
+ === Notice
47
+
48
+ <em>insert_key_prefix</em>, <em>strftime_format</em>, and <em>allow_duplicate_key</em> are newly added config parameters.
49
+
50
+ They can use v0.3.0 or later. To use this parameters, users must update Fluentd to v0.14 or later and this plugin to v0.3.0 or later.
51
+
28
52
  == Contributing to fluent-plugin-redis
29
53
 
30
54
  * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
data/Rakefile CHANGED
@@ -7,4 +7,4 @@ Rake::TestTask.new(:test) do |test|
7
7
  test.verbose = true
8
8
  end
9
9
 
10
- task :default => :test
10
+ task default: :test
@@ -3,10 +3,10 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-redis"
6
- s.version = "0.2.3"
6
+ s.version = "0.3.0"
7
7
  s.platform = Gem::Platform::RUBY
8
8
  s.authors = ["Yuki Nishijima", "Hiroshi Hatake", "Kenji Okimoto"]
9
- s.date = %q{2016-12-01}
9
+ s.date = %q{2016-12-06}
10
10
  s.email = ["mail@yukinishijima.net", "fluent@clear-code.com"]
11
11
  s.homepage = "http://github.com/yuki24/fluent-plugin-redis"
12
12
  s.summary = "Redis output plugin for Fluent"
@@ -16,7 +16,7 @@ Gem::Specification.new do |s|
16
16
  s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
17
17
  s.require_paths = ["lib"]
18
18
 
19
- s.add_dependency %q<fluentd>, ["~> 0.12.0"]
19
+ s.add_dependency %q<fluentd>, [">= 0.14.8", "< 2"]
20
20
  s.add_dependency %q<redis>, ["~> 3.3.0"]
21
21
  s.add_development_dependency %q<rake>, [">= 11.3.0"]
22
22
  s.add_development_dependency %q<bundler>
@@ -1,40 +1,53 @@
1
- module Fluent
2
- class RedisOutput < BufferedOutput
1
+ require 'redis'
2
+ require 'msgpack'
3
+ require 'fluent/msgpack_factory'
4
+ require 'fluent/plugin/output'
5
+
6
+ module Fluent::Plugin
7
+ class RedisOutput < Output
8
+ include Fluent::MessagePackFactory::Mixin
9
+
3
10
  Fluent::Plugin.register_output('redis', self)
4
- attr_reader :redis
5
11
 
6
- config_param :host, :string, :default => 'localhost'
7
- config_param :port, :integer, :default => 6379
8
- config_param :db_number, :integer, :default => 0
9
- config_param :password, :string, :default => nil, :secret => true
12
+ helpers :compat_parameters, :inject
10
13
 
11
- # To support log_level option implemented by Fluentd v0.10.43
12
- unless method_defined?(:log)
13
- define_method("log") { $log }
14
- end
14
+ DEFAULT_BUFFER_TYPE = "memory"
15
15
 
16
- def initialize
17
- super
18
- require 'redis'
19
- require 'msgpack'
16
+ attr_reader :redis
17
+
18
+ config_param :host, :string, default: 'localhost'
19
+ config_param :port, :integer, default: 6379
20
+ config_param :db_number, :integer, default: 0
21
+ config_param :password, :string, default: nil, secret: true
22
+ config_param :insert_key_prefix, :string, default: "${tag}"
23
+ config_param :strftime_format, :string, default: "%s"
24
+ config_param :allow_duplicate_key, :bool, default: false
25
+
26
+ config_section :buffer do
27
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
28
+ config_set_default :chunk_keys, ['tag', 'time']
29
+ config_set_default :timekey, 60
20
30
  end
21
31
 
22
32
  def configure(conf)
33
+ compat_parameters_convert(conf, :buffer, :inject)
23
34
  super
24
35
 
25
36
  if conf.has_key?('namespace')
26
37
  log.warn "namespace option has been removed from fluent-plugin-redis 0.1.3. Please add or remove the namespace '#{conf['namespace']}' manually."
27
38
  end
39
+ raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @chunk_key_tag
40
+ raise Fluent::ConfigError, "'time' in chunk_keys is required." if not @chunk_key_time
28
41
  end
29
42
 
30
43
  def start
31
44
  super
32
45
 
33
46
  options = {
34
- :host => @host,
35
- :port => @port,
36
- :thread_safe => true,
37
- :db => @db_number
47
+ host: @host,
48
+ port: @port,
49
+ thread_safe: true,
50
+ db: @db_number
38
51
  }
39
52
  options[:password] = @password if @password
40
53
 
@@ -47,22 +60,42 @@ module Fluent
47
60
  end
48
61
 
49
62
  def format(tag, time, record)
50
- identifier = [tag, time].join(".")
51
- [identifier, record].to_msgpack
63
+ record = inject_values_to_record(tag, time, record)
64
+ [tag, time, record].to_msgpack
65
+ end
66
+
67
+ def formatted_to_msgpack_binary
68
+ true
52
69
  end
53
70
 
54
71
  def write(chunk)
72
+ tag, time = expand_placeholders(chunk.metadata)
55
73
  @redis.pipelined {
56
- chunk.open { |io|
57
- begin
58
- MessagePack::Unpacker.new(io).each.each_with_index { |record, index|
59
- @redis.mapped_hmset "#{record[0]}.#{index}", record[1]
60
- }
61
- rescue EOFError
62
- # EOFError always occured when reached end of chunk.
74
+ unless @allow_duplicate_key
75
+ chunk.open { |io|
76
+ begin
77
+ msgpack_unpacker(io).each.with_index { |record, index|
78
+ identifier = [tag, time].join(".")
79
+ @redis.mapped_hmset "#{identifier}.#{index}", record[2]
80
+ }
81
+ rescue EOFError
82
+ # EOFError always occured when reached end of chunk.
83
+ end
84
+ }
85
+ else
86
+ chunk.each do |_tag, _time, record|
87
+ @redis.mapped_hmset "#{tag}", record
63
88
  end
64
- }
89
+ end
65
90
  }
66
91
  end
92
+
93
+ private
94
+
95
+ def expand_placeholders(metadata)
96
+ tag = extract_placeholders(@insert_key_prefix, metadata)
97
+ time = extract_placeholders(@strftime_format, metadata)
98
+ return tag, time
99
+ end
67
100
  end
68
101
  end
@@ -1,20 +1,31 @@
1
1
  require 'fluent/test'
2
+ require 'fluent/test/helpers'
3
+ require 'fluent/test/driver/output'
2
4
  require 'fluent/plugin/out_redis'
5
+ require 'fluent/time' # Fluent::TimeFormatter
3
6
 
4
7
  class FileOutputTest < Test::Unit::TestCase
8
+ include Fluent::Test::Helpers
9
+
10
+ CONFIG = %[
11
+ host localhost
12
+ port 6379
13
+ db_number 1
14
+ ]
15
+
5
16
  def setup
6
17
  Fluent::Test.setup
7
18
 
8
- @d = create_driver %[
9
- host localhost
10
- port 6379
11
- db_number 1
12
- ]
13
- @time = Time.parse("2011-01-02 13:14:15 UTC").to_i
19
+ @d = create_driver
20
+ @time = event_time("2011-01-02 13:14:15 UTC")
14
21
  end
15
22
 
16
23
  def create_driver(conf = CONFIG)
17
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::RedisOutput).configure(conf)
24
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::RedisOutput).configure(conf)
25
+ end
26
+
27
+ def time_formatter(inject_config)
28
+ Fluent::TimeFormatter.new(inject_config.time_format, inject_config.localtime, inject_config.timezone)
18
29
  end
19
30
 
20
31
  def test_configure
@@ -22,13 +33,13 @@ class FileOutputTest < Test::Unit::TestCase
22
33
  assert_equal 6379, @d.instance.port
23
34
  assert_equal 1, @d.instance.db_number
24
35
  assert_nil @d.instance.password
36
+ assert_equal '${tag}', @d.instance.insert_key_prefix
37
+ assert_equal '%s', @d.instance.strftime_format
38
+ assert_false @d.instance.allow_duplicate_key
25
39
  end
26
40
 
27
41
  def test_configure_with_password
28
- d = create_driver %[
29
- host localhost
30
- port 6379
31
- db_number 1
42
+ d = create_driver CONFIG + %[
32
43
  password testpass
33
44
  ]
34
45
  assert_equal 'localhost', d.instance.host
@@ -37,18 +48,104 @@ class FileOutputTest < Test::Unit::TestCase
37
48
  assert_equal 'testpass', d.instance.password
38
49
  end
39
50
 
51
+ def test_configure_without_tag_chunk_key
52
+ config = config_element('ROOT', '', {
53
+ "host" => "localhost",
54
+ "port" => 6379,
55
+ "db_number" => 1,
56
+ }, [
57
+ config_element('buffer', 'time', {
58
+ 'chunk_keys' => 'time',
59
+ })
60
+ ])
61
+ assert_raise Fluent::ConfigError do
62
+ create_driver(config)
63
+ end
64
+ end
65
+
40
66
  def test_format
41
- @d.emit({"a"=>1}, @time)
42
- @d.expect_format(["test.#{@time}", {"a"=>1}].to_msgpack)
43
- @d.run
67
+ @d.run(default_tag: 'test') do
68
+ @d.feed(@time, {"a"=>1})
69
+ end
70
+ assert_equal [["test", @time, {"a"=>1}].to_msgpack], @d.formatted
71
+ end
72
+
73
+ class InjectTest < self
74
+ def test_format_inject_tag_keys
75
+ d = create_driver CONFIG + %[
76
+ include_tag_key true
77
+ ]
78
+ d.run(default_tag: 'test') do
79
+ d.feed(@time, {"a"=>1})
80
+ end
81
+ assert_equal [["test", @time, {"a"=>1, "tag" => "test"}].to_msgpack], d.formatted
82
+ end
83
+
84
+ def test_format_inject_time_keys
85
+ d = create_driver CONFIG + %[
86
+ include_time_key true
87
+ ]
88
+ timef = time_formatter(d.instance.inject_config)
89
+ d.run(default_tag: 'test') do
90
+ d.feed(@time, {"a"=>1})
91
+ end
92
+ assert_equal [["test", @time, {"a"=>1, "time" => timef.call(@time)}].to_msgpack], d.formatted
93
+ end
44
94
  end
45
95
 
46
- def test_write
47
- @d.emit({"a"=>2}, @time)
48
- @d.emit({"a"=>3}, @time)
49
- @d.run
96
+ class WriteTest < self
97
+ def test_write
98
+ d = create_driver
99
+ time = event_time("2011-01-02 13:14:00 UTC")
100
+ d.run(default_tag: 'test') do
101
+ d.feed(time, {"a"=>2})
102
+ d.feed(time, {"a"=>3})
103
+ end
104
+
105
+ assert_equal "2", d.instance.redis.hget("test.#{time}.0", "a")
106
+ assert_equal "3", d.instance.redis.hget("test.#{time}.1", "a")
107
+ end
108
+
109
+ def test_write_with_insert_key_prefix
110
+ d = create_driver CONFIG + %[
111
+ insert_key_prefix "${tag[1]}.${tag[2]}"
112
+ ]
113
+ time = event_time("2011-01-02 13:14:00 UTC")
114
+ d.run(default_tag: 'prefix.insert.test') do
115
+ d.feed(time, {"a"=>2})
116
+ d.feed(time, {"a"=>3})
117
+ end
118
+
119
+ assert_equal "2", d.instance.redis.hget("insert.test.#{time}.0", "a")
120
+ assert_equal "3", d.instance.redis.hget("insert.test.#{time}.1", "a")
121
+ end
122
+
123
+ def test_write_with_custom_strftime_format
124
+ d = create_driver CONFIG + %[
125
+ strftime_format "%Y%m%d.%H%M%S"
126
+ ]
127
+ time = event_time("2011-01-02 13:14:00 UTC")
128
+ strtime = Time.at(time.to_r).strftime("%Y%m%d.%H%M%S")
129
+ d.run(default_tag: 'test') do
130
+ d.feed(time, {"a"=>4})
131
+ d.feed(time, {"a"=>5})
132
+ end
133
+
134
+ assert_equal "4", d.instance.redis.hget("test.#{strtime}.0", "a")
135
+ assert_equal "5", d.instance.redis.hget("test.#{strtime}.1", "a")
136
+ end
137
+
138
+ def test_write_with_allow_duplicate
139
+ d = create_driver CONFIG + %[
140
+ allow_duplicate_key true
141
+ ]
142
+ time = event_time("2011-01-02 13:14:00 UTC")
143
+ d.run(default_tag: 'test.duplicate') do
144
+ d.feed(time, {"a"=>6})
145
+ d.feed(time, {"a"=>7})
146
+ end
50
147
 
51
- assert_equal "2", @d.instance.redis.hget("test.#{@time}.0", "a")
52
- assert_equal "3", @d.instance.redis.hget("test.#{@time}.1", "a")
148
+ assert_equal "7", d.instance.redis.hget("test.duplicate", "a")
149
+ end
53
150
  end
54
151
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-redis
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.3
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yuki Nishijima
@@ -10,22 +10,28 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2016-12-01 00:00:00.000000000 Z
13
+ date: 2016-12-06 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
17
17
  requirement: !ruby/object:Gem::Requirement
18
18
  requirements:
19
- - - "~>"
19
+ - - ">="
20
+ - !ruby/object:Gem::Version
21
+ version: 0.14.8
22
+ - - "<"
20
23
  - !ruby/object:Gem::Version
21
- version: 0.12.0
24
+ version: '2'
22
25
  type: :runtime
23
26
  prerelease: false
24
27
  version_requirements: !ruby/object:Gem::Requirement
25
28
  requirements:
26
- - - "~>"
29
+ - - ">="
30
+ - !ruby/object:Gem::Version
31
+ version: 0.14.8
32
+ - - "<"
27
33
  - !ruby/object:Gem::Version
28
- version: 0.12.0
34
+ version: '2'
29
35
  - !ruby/object:Gem::Dependency
30
36
  name: redis
31
37
  requirement: !ruby/object:Gem::Requirement