fluent-plugin-flume 0.2.0 → 1.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +4 -4
  2. data/VERSION +1 -1
  3. data/lib/fluent/plugin/out_flume.rb +29 -8
  4. data/test/plugin/test_out_flume.rb +54 -47
  5. metadata +5 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d92aa659b736a1e4228653bcf7089cf0844ed412
4
- data.tar.gz: 74f103fa74799fdb9ee89debad3a3ce803c360ae
3
+ metadata.gz: 5c6c30fffafba5fabb5604d70ec69217d014e15b
4
+ data.tar.gz: 39e66e209a62569b43d5a5c9f505dab574918db0
5
5
  SHA512:
6
- metadata.gz: d1ec6c1fdd6ab7cd36acc3bdef637f4186c4464adec65839abeea0116a1e464240dccf2f88c414127cf88ec6dbea340347eac0fc7f6fbd9dee87012820b50ef6
7
- data.tar.gz: fe24084964f0b71d51073c807ede72419dfde1faca46f824b5dbbc7b73875e6c67fba1392905f55cf0254749ae8ae0d1a08d898ce7eb998fb0670f30fe9601b7
6
+ metadata.gz: a1fbfb352b40f8f0d8822a085fc415fc376c834c2e44e2962840140c8cc4cb0b82ad72ccb8281e58a904fa119b3ef2dfdb9508536135fa49b4146dd6f04e0a3c
7
+ data.tar.gz: 59b805d33056fd27735e6c9d75c118c5453e854e40ee3a9dc709c4dd9185f3346e74a3ad2d222ae8e7debf28c60e876536fc4870a392e62c8c1d7f50095b5f8a
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.0
1
+ 1.0.0.rc1
data/lib/fluent/plugin/out_flume.rb CHANGED
@@ -16,21 +16,35 @@
16
16
  # limitations under the License.
17
17
  #
18
18
 
19
- require 'fluent/output'
19
+ require 'fluent/plugin/output'
20
20
 
21
- module Fluent
22
- class FlumeOutput < BufferedOutput
21
+ module Fluent::Plugin
22
+ class FlumeOutput < Output
23
23
  Fluent::Plugin.register_output('flume', self)
24
24
 
25
+ helpers :formatter, :compat_parameters
26
+
27
+ DEFAULT_BUFFER_TYPE = "memory"
28
+ DEFAULT_FORMAT_TYPE = 'json'
29
+
25
30
  config_param :host, :string, :default => 'localhost'
26
31
  config_param :port, :integer, :default => 35863
27
32
  config_param :timeout, :integer, :default => 30
28
33
  config_param :remove_prefix, :string, :default => nil
29
34
  config_param :default_category, :string, :default => 'unknown'
30
35
  desc "The format of the thrift body. (default: json)"
31
- config_param :format, :string, default: 'json'
36
+ config_param :format, :string, default: DEFAULT_FORMAT_TYPE
32
37
  config_param :trim_nl, :bool, default: true
33
38
 
39
+ config_section :buffer do
40
+ config_set_default :@type, DEFAULT_BUFFER_TYPE
41
+ config_set_default :chunk_keys, ['tag']
42
+ end
43
+
44
+ config_section :format do
45
+ config_set_default :@type, DEFAULT_FORMAT_TYPE
46
+ end
47
+
34
48
  unless method_defined?(:log)
35
49
  define_method(:log) { $log }
36
50
  end
@@ -47,11 +61,10 @@ module Fluent
47
61
  def configure(conf)
48
62
  # override default buffer_chunk_limit
49
63
  conf['buffer_chunk_limit'] ||= '1m'
50
-
64
+ compat_parameters_convert(conf, :formatter)
51
65
  super
52
66
 
53
- @formatter = Plugin.new_formatter(@format)
54
- @formatter.configure(conf)
67
+ @formatter = formatter_create
55
68
  end
56
69
 
57
70
  def start
@@ -77,6 +90,14 @@ module Fluent
77
90
  [time, fr].to_msgpack
78
91
  end
79
92
 
93
+ def formatted_to_msgpack_binary
94
+ true
95
+ end
96
+
97
+ def multi_workers_ready?
98
+ true
99
+ end
100
+
80
101
  def write(chunk)
81
102
  socket = Thrift::Socket.new @host, @port, @timeout
82
103
  transport = Thrift::FramedTransport.new socket
@@ -84,7 +105,7 @@ module Fluent
84
105
  protocol = Thrift::CompactProtocol.new transport
85
106
  client = ThriftSourceProtocol::Client.new protocol
86
107
 
87
- tag = chunk.key
108
+ tag = chunk.metadata.tag
88
109
  count = 0
89
110
  header = {}
90
111
  transport.open
data/test/plugin/test_out_flume.rb CHANGED
@@ -1,8 +1,12 @@
1
1
  require 'test/unit'
2
2
  require 'fluent/test'
3
+ require 'fluent/test/helpers'
4
+ require 'fluent/test/driver/output'
3
5
  require 'lib/fluent/plugin/out_flume'
4
6
 
5
7
  class FlumeOutputTest < Test::Unit::TestCase
8
+ include Fluent::Test::Helpers
9
+
6
10
  def setup
7
11
  Fluent::Test.setup
8
12
  end
@@ -12,8 +16,8 @@ class FlumeOutputTest < Test::Unit::TestCase
12
16
  port 35862
13
17
  ]
14
18
 
15
- def create_driver(conf=CONFIG, tag='test')
16
- Fluent::Test::BufferedOutputTestDriver.new(Fluent::FlumeOutput, tag) do
19
+ def create_driver(conf=CONFIG)
20
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::FlumeOutput) do
17
21
  def write(chunk)
18
22
  chunk.read
19
23
  end
@@ -36,79 +40,82 @@ class FlumeOutputTest < Test::Unit::TestCase
36
40
  end
37
41
 
38
42
  def test_format
39
- time = Time.parse("2011-12-21 13:14:15 UTC").to_i
43
+ time = event_time("2011-12-21 13:14:15 UTC")
40
44
 
41
45
  d = create_driver
42
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
43
- d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
44
- d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
45
- d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
46
- d.run
46
+ d.run(default_tag: 'test') do
47
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
48
+ d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
49
+ end
50
+ assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
51
+ assert_equal [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack, d.formatted[1]
47
52
 
48
53
  d = create_driver(CONFIG + %[
49
54
  remove_prefix test
50
- ], 'test.flumeplugin')
51
- assert_equal 'test.flumeplugin', d.tag
55
+ ])
52
56
 
53
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
54
- d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
55
- d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
56
- d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
57
- d.run
57
+ d.run(default_tag: 'test.flumeplugin') do
58
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
59
+ d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
60
+ end
61
+ assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
62
+ assert_equal [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack, d.formatted[1]
58
63
 
59
64
  d = create_driver(CONFIG + %[
60
65
  remove_prefix test
61
- ], 'xxx.test.flumeplugin')
62
- assert_equal 'xxx.test.flumeplugin', d.tag
63
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
64
- d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
65
- d.run
66
+ ])
67
+ d.run(default_tag: 'xxx.test.flumeplugin') do
68
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
69
+ end
66
70
 
67
71
  d = create_driver(CONFIG + %[
68
72
  remove_prefix test
69
- ], 'test')
70
- assert_equal 'test', d.tag
71
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
72
- d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
73
- d.run
73
+ ])
74
+ d.run(default_tag: 'test') do
75
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
76
+ end
77
+ assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
74
78
 
75
79
  d = create_driver(CONFIG + %[
76
80
  remove_prefix test
77
- ], 'test.flumeplugin')
78
- assert_equal 'test.flumeplugin', d.tag
79
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
80
- d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
81
- d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
82
- d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
83
- d.run
81
+ ])
82
+ d.run(default_tag: 'test.flumeplugin') do
83
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
84
+ d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
85
+ end
86
+ assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
87
+ assert_equal [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack, d.formatted[1]
84
88
  end
85
89
 
86
90
  def test_write
87
- time = Time.parse("2011-12-21 13:14:15 UTC").to_i
91
+ time = event_time("2011-12-21 13:14:15 UTC")
88
92
 
89
93
  d = create_driver
90
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
91
- d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
92
- d.run
94
+ d.run(default_tag: 'test') do
95
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
96
+ d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
97
+ end
93
98
 
94
99
  d = create_driver(CONFIG + %[
95
100
  remove_prefix test
96
- ], 'test.flumeplugin')
97
- assert_equal 'test.flumeplugin', d.tag
98
- d.run
101
+ ])
102
+ d.end_if do
103
+ d.events.size >= 0
104
+ end
105
+ d.run(default_tag: 'test.flumeplugin')
99
106
 
100
107
  d = create_driver(CONFIG + %[
101
108
  remove_prefix test
102
- ], 'xxx.test.flumeplugin')
103
- assert_equal 'xxx.test.flumeplugin', d.tag
104
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
105
- d.run
109
+ ])
110
+ d.run(default_tag: 'xxx.test.flumeplugin') do
111
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
112
+ end
106
113
 
107
114
  d = create_driver(CONFIG + %[
108
115
  remove_prefix test
109
- ], 'test')
110
- assert_equal 'test', d.tag
111
- d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
112
- d.run
116
+ ])
117
+ d.run(default_tag: 'test') do
118
+ d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
119
+ end
113
120
  end
114
121
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-flume
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 1.0.0.rc1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Muga Nishizawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-10-25 00:00:00.000000000 Z
11
+ date: 2017-07-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -115,12 +115,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
115
115
  version: '0'
116
116
  required_rubygems_version: !ruby/object:Gem::Requirement
117
117
  requirements:
118
- - - ">="
118
+ - - ">"
119
119
  - !ruby/object:Gem::Version
120
- version: '0'
120
+ version: 1.3.1
121
121
  requirements: []
122
122
  rubyforge_project:
123
- rubygems_version: 2.5.1
123
+ rubygems_version: 2.6.11
124
124
  signing_key:
125
125
  specification_version: 4
126
126
  summary: Flume Input/Output plugin for Fluentd event collector