fluent-plugin-kinesis 2.2.0 → 3.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/.github/PULL_REQUEST_TEMPLATE.md +6 -0
  3. data/.travis.yml +18 -7
  4. data/CHANGELOG.md +9 -2
  5. data/CODE_OF_CONDUCT.md +4 -0
  6. data/CONTRIBUTING.md +61 -0
  7. data/README.md +34 -38
  8. data/benchmark/task.rake +6 -10
  9. data/fluent-plugin-kinesis.gemspec +20 -13
  10. data/gemfiles/{Gemfile.td-agent-2.3.5 → Gemfile.fluentd-0.14.10} +1 -6
  11. data/gemfiles/Gemfile.td-agent-3.1.1 +31 -0
  12. data/gemfiles/Gemfile.td-agent-3.2.0 +31 -0
  13. data/gemfiles/Gemfile.td-agent-3.2.1 +31 -0
  14. data/gemfiles/Gemfile.td-agent-3.3.0 +31 -0
  15. data/lib/fluent/plugin/kinesis.rb +112 -117
  16. data/lib/fluent/plugin/kinesis_helper/aggregator.rb +49 -47
  17. data/lib/fluent/plugin/kinesis_helper/api.rb +139 -137
  18. data/lib/fluent/plugin/kinesis_helper/client.rb +118 -128
  19. data/lib/fluent/plugin/out_kinesis_firehose.rb +31 -29
  20. data/lib/fluent/plugin/out_kinesis_streams.rb +41 -39
  21. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +45 -43
  22. data/lib/fluent_plugin_kinesis/version.rb +1 -1
  23. metadata +57 -24
data/lib/fluent/plugin/out_kinesis_firehose.rb CHANGED
@@ -15,42 +15,44 @@
15
15
  require 'fluent/plugin/kinesis'
16
16
 
17
17
  module Fluent
18
- class KinesisFirehoseOutput < KinesisOutput
19
- Fluent::Plugin.register_output('kinesis_firehose', self)
18
+ module Plugin
19
+ class KinesisFirehoseOutput < KinesisOutput
20
+ Fluent::Plugin.register_output('kinesis_firehose', self)
20
21
 
21
- RequestType = :firehose
22
- BatchRequestLimitCount = 500
23
- BatchRequestLimitSize = 4 * 1024 * 1024
24
- include KinesisHelper::API::BatchRequest
22
+ RequestType = :firehose
23
+ BatchRequestLimitCount = 500
24
+ BatchRequestLimitSize = 4 * 1024 * 1024
25
+ include KinesisHelper::API::BatchRequest
25
26
 
26
- config_param :delivery_stream_name, :string
27
- config_param :append_new_line, :bool, default: true
27
+ config_param :delivery_stream_name, :string
28
+ config_param :append_new_line, :bool, default: true
28
29
 
29
- def configure(conf)
30
- super
31
- if @append_new_line
32
- org_data_formatter = @data_formatter
33
- @data_formatter = ->(tag, time, record) {
34
- org_data_formatter.call(tag, time, record) + "\n"
35
- }
30
+ def configure(conf)
31
+ super
32
+ if @append_new_line
33
+ org_data_formatter = @data_formatter
34
+ @data_formatter = ->(tag, time, record) {
35
+ org_data_formatter.call(tag, time, record).chomp + "\n"
36
+ }
37
+ end
36
38
  end
37
- end
38
39
 
39
- def format(tag, time, record)
40
- format_for_api do
41
- [@data_formatter.call(tag, time, record)]
40
+ def format(tag, time, record)
41
+ format_for_api do
42
+ [@data_formatter.call(tag, time, record)]
43
+ end
42
44
  end
43
- end
44
45
 
45
- def write(chunk)
46
- write_records_batch(chunk) do |batch|
47
- records = batch.map{|(data)|
48
- { data: data }
49
- }
50
- client.put_record_batch(
51
- delivery_stream_name: @delivery_stream_name,
52
- records: records,
53
- )
46
+ def write(chunk)
47
+ write_records_batch(chunk) do |batch|
48
+ records = batch.map{|(data)|
49
+ { data: data }
50
+ }
51
+ client.put_record_batch(
52
+ delivery_stream_name: @delivery_stream_name,
53
+ records: records,
54
+ )
55
+ end
54
56
  end
55
57
  end
56
58
  end
data/lib/fluent/plugin/out_kinesis_streams.rb CHANGED
@@ -15,54 +15,56 @@
15
15
  require 'fluent/plugin/kinesis'
16
16
 
17
17
  module Fluent
18
- class KinesisStreamsOutput < KinesisOutput
19
- Fluent::Plugin.register_output('kinesis_streams', self)
18
+ module Plugin
19
+ class KinesisStreamsOutput < KinesisOutput
20
+ Fluent::Plugin.register_output('kinesis_streams', self)
20
21
 
21
- RequestType = :streams
22
- BatchRequestLimitCount = 500
23
- BatchRequestLimitSize = 5 * 1024 * 1024
24
- include KinesisHelper::API::BatchRequest
22
+ RequestType = :streams
23
+ BatchRequestLimitCount = 500
24
+ BatchRequestLimitSize = 5 * 1024 * 1024
25
+ include KinesisHelper::API::BatchRequest
25
26
 
26
- config_param :stream_name, :string
27
- config_param :partition_key, :string, default: nil
27
+ config_param :stream_name, :string
28
+ config_param :partition_key, :string, default: nil
28
29
 
29
- def configure(conf)
30
- super
31
- @key_formatter = key_formatter_create
32
- end
30
+ def configure(conf)
31
+ super
32
+ @key_formatter = key_formatter_create
33
+ end
33
34
 
34
- def format(tag, time, record)
35
- format_for_api do
36
- data = @data_formatter.call(tag, time, record)
37
- key = @key_formatter.call(record)
38
- [data, key]
35
+ def format(tag, time, record)
36
+ format_for_api do
37
+ data = @data_formatter.call(tag, time, record)
38
+ key = @key_formatter.call(record)
39
+ [data, key]
40
+ end
39
41
  end
40
- end
41
42
 
42
- def write(chunk)
43
- write_records_batch(chunk) do |batch|
44
- records = batch.map{|(data, partition_key)|
45
- { data: data, partition_key: partition_key }
46
- }
47
- client.put_records(
48
- stream_name: @stream_name,
49
- records: records,
50
- )
43
+ def write(chunk)
44
+ write_records_batch(chunk) do |batch|
45
+ records = batch.map{|(data, partition_key)|
46
+ { data: data, partition_key: partition_key }
47
+ }
48
+ client.put_records(
49
+ stream_name: @stream_name,
50
+ records: records,
51
+ )
52
+ end
51
53
  end
52
- end
53
54
 
54
- private
55
+ private
55
56
 
56
- def key_formatter_create
57
- if @partition_key.nil?
58
- ->(record) { SecureRandom.hex(16) }
59
- else
60
- ->(record) {
61
- if !record.key?(@partition_key)
62
- raise KeyNotFoundError.new(@partition_key, record)
63
- end
64
- record[@partition_key]
65
- }
57
+ def key_formatter_create
58
+ if @partition_key.nil?
59
+ ->(record) { SecureRandom.hex(16) }
60
+ else
61
+ ->(record) {
62
+ if !record.key?(@partition_key)
63
+ raise KeyNotFoundError.new(@partition_key, record)
64
+ end
65
+ record[@partition_key]
66
+ }
67
+ end
66
68
  end
67
69
  end
68
70
  end
data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb CHANGED
@@ -16,60 +16,62 @@ require 'fluent/plugin/kinesis'
16
16
  require 'fluent/plugin/kinesis_helper/aggregator'
17
17
 
18
18
  module Fluent
19
- class KinesisStreamsAggregatedOutput < KinesisOutput
20
- Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
21
- include KinesisHelper::Aggregator::Mixin
19
+ module Plugin
20
+ class KinesisStreamsAggregatedOutput < KinesisOutput
21
+ Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
22
+ include KinesisHelper::Aggregator::Mixin
22
23
 
23
- RequestType = :streams_aggregated
24
- BatchRequestLimitCount = 100_000
25
- BatchRequestLimitSize = 1024 * 1024
26
- include KinesisHelper::API::BatchRequest
24
+ RequestType = :streams_aggregated
25
+ BatchRequestLimitCount = 100_000
26
+ BatchRequestLimitSize = 1024 * 1024
27
+ include KinesisHelper::API::BatchRequest
27
28
 
28
- config_param :stream_name, :string
29
- config_param :fixed_partition_key, :string, default: nil
29
+ config_param :stream_name, :string
30
+ config_param :fixed_partition_key, :string, default: nil
30
31
 
31
- def configure(conf)
32
- super
33
- @partition_key_generator = create_partition_key_generator
34
- @batch_request_max_size -= offset
35
- @max_record_size -= offset
36
- end
32
+ def configure(conf)
33
+ super
34
+ @partition_key_generator = create_partition_key_generator
35
+ @batch_request_max_size -= offset
36
+ @max_record_size -= offset
37
+ end
37
38
 
38
- def format(tag, time, record)
39
- format_for_api do
40
- [@data_formatter.call(tag, time, record)]
39
+ def format(tag, time, record)
40
+ format_for_api do
41
+ [@data_formatter.call(tag, time, record)]
42
+ end
41
43
  end
42
- end
43
44
 
44
- def write(chunk)
45
- write_records_batch(chunk) do |batch|
46
- key = @partition_key_generator.call
47
- records = batch.map{|(data)|data}
48
- client.put_records(
49
- stream_name: @stream_name,
50
- records: [{
51
- partition_key: key,
52
- data: aggregator.aggregate(records, key),
53
- }],
54
- )
45
+ def write(chunk)
46
+ write_records_batch(chunk) do |batch|
47
+ key = @partition_key_generator.call
48
+ records = batch.map{|(data)|data}
49
+ client.put_records(
50
+ stream_name: @stream_name,
51
+ records: [{
52
+ partition_key: key,
53
+ data: aggregator.aggregate(records, key),
54
+ }],
55
+ )
56
+ end
55
57
  end
56
- end
57
58
 
58
- def offset
59
- @offset ||= AggregateOffset + @partition_key_generator.call.size*2
60
- end
59
+ def offset
60
+ @offset ||= AggregateOffset + @partition_key_generator.call.size*2
61
+ end
61
62
 
62
- private
63
+ private
63
64
 
64
- def size_of_values(record)
65
- super(record) + RecordOffset
66
- end
65
+ def size_of_values(record)
66
+ super(record) + RecordOffset
67
+ end
67
68
 
68
- def create_partition_key_generator
69
- if @fixed_partition_key.nil?
70
- ->() { SecureRandom.hex(16) }
71
- else
72
- ->() { @fixed_partition_key }
69
+ def create_partition_key_generator
70
+ if @fixed_partition_key.nil?
71
+ ->() { SecureRandom.hex(16) }
72
+ else
73
+ ->() { @fixed_partition_key }
74
+ end
73
75
  end
74
76
  end
75
77
  end
data/lib/fluent_plugin_kinesis/version.rb CHANGED
@@ -13,5 +13,5 @@
13
13
  # language governing permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '2.2.0'
16
+ VERSION = '3.0.0'
17
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-07-13 00:00:00.000000000 Z
11
+ date: 2019-03-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.12.35
19
+ version: 0.14.10
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2'
@@ -26,76 +26,102 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.12.35
29
+ version: 0.14.10
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: aws-sdk
34
+ name: aws-sdk-kinesis
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - ">="
37
+ - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: 2.9.9
40
- - - "<"
39
+ version: '1'
40
+ - - "!="
41
41
  - !ruby/object:Gem::Version
42
- version: '4'
42
+ version: '1.4'
43
+ - - "!="
44
+ - !ruby/object:Gem::Version
45
+ version: '1.5'
43
46
  type: :runtime
44
47
  prerelease: false
45
48
  version_requirements: !ruby/object:Gem::Requirement
46
49
  requirements:
47
- - - ">="
50
+ - - "~>"
48
51
  - !ruby/object:Gem::Version
49
- version: 2.9.9
50
- - - "<"
52
+ version: '1'
53
+ - - "!="
51
54
  - !ruby/object:Gem::Version
52
- version: '4'
55
+ version: '1.4'
56
+ - - "!="
57
+ - !ruby/object:Gem::Version
58
+ version: '1.5'
53
59
  - !ruby/object:Gem::Dependency
54
- name: google-protobuf
60
+ name: aws-sdk-firehose
55
61
  requirement: !ruby/object:Gem::Requirement
56
62
  requirements:
57
63
  - - "~>"
58
64
  - !ruby/object:Gem::Version
59
- version: '3'
60
- - - "<"
65
+ version: '1'
66
+ - - "!="
67
+ - !ruby/object:Gem::Version
68
+ version: '1.5'
69
+ - - "!="
61
70
  - !ruby/object:Gem::Version
62
- version: '3.7'
71
+ version: '1.9'
63
72
  type: :runtime
64
73
  prerelease: false
65
74
  version_requirements: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - "~>"
77
+ - !ruby/object:Gem::Version
78
+ version: '1'
79
+ - - "!="
80
+ - !ruby/object:Gem::Version
81
+ version: '1.5'
82
+ - - "!="
83
+ - !ruby/object:Gem::Version
84
+ version: '1.9'
85
+ - !ruby/object:Gem::Dependency
86
+ name: google-protobuf
87
+ requirement: !ruby/object:Gem::Requirement
66
88
  requirements:
67
89
  - - "~>"
68
90
  - !ruby/object:Gem::Version
69
91
  version: '3'
70
- - - "<"
92
+ type: :runtime
93
+ prerelease: false
94
+ version_requirements: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - "~>"
71
97
  - !ruby/object:Gem::Version
72
- version: '3.7'
98
+ version: '3'
73
99
  - !ruby/object:Gem::Dependency
74
100
  name: bundler
75
101
  requirement: !ruby/object:Gem::Requirement
76
102
  requirements:
77
- - - "~>"
103
+ - - ">="
78
104
  - !ruby/object:Gem::Version
79
105
  version: '1.10'
80
106
  type: :development
81
107
  prerelease: false
82
108
  version_requirements: !ruby/object:Gem::Requirement
83
109
  requirements:
84
- - - "~>"
110
+ - - ">="
85
111
  - !ruby/object:Gem::Version
86
112
  version: '1.10'
87
113
  - !ruby/object:Gem::Dependency
88
114
  name: rake
89
115
  requirement: !ruby/object:Gem::Requirement
90
116
  requirements:
91
- - - "~>"
117
+ - - ">="
92
118
  - !ruby/object:Gem::Version
93
119
  version: '10.0'
94
120
  type: :development
95
121
  prerelease: false
96
122
  version_requirements: !ruby/object:Gem::Requirement
97
123
  requirements:
98
- - - "~>"
124
+ - - ">="
99
125
  - !ruby/object:Gem::Version
100
126
  version: '10.0'
101
127
  - !ruby/object:Gem::Dependency
@@ -230,9 +256,12 @@ executables: []
230
256
  extensions: []
231
257
  extra_rdoc_files: []
232
258
  files:
259
+ - ".github/PULL_REQUEST_TEMPLATE.md"
233
260
  - ".gitignore"
234
261
  - ".travis.yml"
235
262
  - CHANGELOG.md
263
+ - CODE_OF_CONDUCT.md
264
+ - CONTRIBUTING.md
236
265
  - CONTRIBUTORS.txt
237
266
  - Gemfile
238
267
  - LICENSE.txt
@@ -242,7 +271,11 @@ files:
242
271
  - Rakefile
243
272
  - benchmark/task.rake
244
273
  - fluent-plugin-kinesis.gemspec
245
- - gemfiles/Gemfile.td-agent-2.3.5
274
+ - gemfiles/Gemfile.fluentd-0.14.10
275
+ - gemfiles/Gemfile.td-agent-3.1.1
276
+ - gemfiles/Gemfile.td-agent-3.2.0
277
+ - gemfiles/Gemfile.td-agent-3.2.1
278
+ - gemfiles/Gemfile.td-agent-3.3.0
246
279
  - lib/fluent/plugin/kinesis.rb
247
280
  - lib/fluent/plugin/kinesis_helper/aggregator.rb
248
281
  - lib/fluent/plugin/kinesis_helper/api.rb