fluent-plugin-kinesis 2.2.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/.github/PULL_REQUEST_TEMPLATE.md +6 -0
  3. data/.travis.yml +18 -7
  4. data/CHANGELOG.md +9 -2
  5. data/CODE_OF_CONDUCT.md +4 -0
  6. data/CONTRIBUTING.md +61 -0
  7. data/README.md +34 -38
  8. data/benchmark/task.rake +6 -10
  9. data/fluent-plugin-kinesis.gemspec +20 -13
  10. data/gemfiles/{Gemfile.td-agent-2.3.5 → Gemfile.fluentd-0.14.10} +1 -6
  11. data/gemfiles/Gemfile.td-agent-3.1.1 +31 -0
  12. data/gemfiles/Gemfile.td-agent-3.2.0 +31 -0
  13. data/gemfiles/Gemfile.td-agent-3.2.1 +31 -0
  14. data/gemfiles/Gemfile.td-agent-3.3.0 +31 -0
  15. data/lib/fluent/plugin/kinesis.rb +112 -117
  16. data/lib/fluent/plugin/kinesis_helper/aggregator.rb +49 -47
  17. data/lib/fluent/plugin/kinesis_helper/api.rb +139 -137
  18. data/lib/fluent/plugin/kinesis_helper/client.rb +118 -128
  19. data/lib/fluent/plugin/out_kinesis_firehose.rb +31 -29
  20. data/lib/fluent/plugin/out_kinesis_streams.rb +41 -39
  21. data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb +45 -43
  22. data/lib/fluent_plugin_kinesis/version.rb +1 -1
  23. metadata +57 -24
data/lib/fluent/plugin/out_kinesis_firehose.rb CHANGED
@@ -15,42 +15,44 @@
15
15
  require 'fluent/plugin/kinesis'
16
16
 
17
17
  module Fluent
18
- class KinesisFirehoseOutput < KinesisOutput
19
- Fluent::Plugin.register_output('kinesis_firehose', self)
18
+ module Plugin
19
+ class KinesisFirehoseOutput < KinesisOutput
20
+ Fluent::Plugin.register_output('kinesis_firehose', self)
20
21
 
21
- RequestType = :firehose
22
- BatchRequestLimitCount = 500
23
- BatchRequestLimitSize = 4 * 1024 * 1024
24
- include KinesisHelper::API::BatchRequest
22
+ RequestType = :firehose
23
+ BatchRequestLimitCount = 500
24
+ BatchRequestLimitSize = 4 * 1024 * 1024
25
+ include KinesisHelper::API::BatchRequest
25
26
 
26
- config_param :delivery_stream_name, :string
27
- config_param :append_new_line, :bool, default: true
27
+ config_param :delivery_stream_name, :string
28
+ config_param :append_new_line, :bool, default: true
28
29
 
29
- def configure(conf)
30
- super
31
- if @append_new_line
32
- org_data_formatter = @data_formatter
33
- @data_formatter = ->(tag, time, record) {
34
- org_data_formatter.call(tag, time, record) + "\n"
35
- }
30
+ def configure(conf)
31
+ super
32
+ if @append_new_line
33
+ org_data_formatter = @data_formatter
34
+ @data_formatter = ->(tag, time, record) {
35
+ org_data_formatter.call(tag, time, record).chomp + "\n"
36
+ }
37
+ end
36
38
  end
37
- end
38
39
 
39
- def format(tag, time, record)
40
- format_for_api do
41
- [@data_formatter.call(tag, time, record)]
40
+ def format(tag, time, record)
41
+ format_for_api do
42
+ [@data_formatter.call(tag, time, record)]
43
+ end
42
44
  end
43
- end
44
45
 
45
- def write(chunk)
46
- write_records_batch(chunk) do |batch|
47
- records = batch.map{|(data)|
48
- { data: data }
49
- }
50
- client.put_record_batch(
51
- delivery_stream_name: @delivery_stream_name,
52
- records: records,
53
- )
46
+ def write(chunk)
47
+ write_records_batch(chunk) do |batch|
48
+ records = batch.map{|(data)|
49
+ { data: data }
50
+ }
51
+ client.put_record_batch(
52
+ delivery_stream_name: @delivery_stream_name,
53
+ records: records,
54
+ )
55
+ end
54
56
  end
55
57
  end
56
58
  end
data/lib/fluent/plugin/out_kinesis_streams.rb CHANGED
@@ -15,54 +15,56 @@
15
15
  require 'fluent/plugin/kinesis'
16
16
 
17
17
  module Fluent
18
- class KinesisStreamsOutput < KinesisOutput
19
- Fluent::Plugin.register_output('kinesis_streams', self)
18
+ module Plugin
19
+ class KinesisStreamsOutput < KinesisOutput
20
+ Fluent::Plugin.register_output('kinesis_streams', self)
20
21
 
21
- RequestType = :streams
22
- BatchRequestLimitCount = 500
23
- BatchRequestLimitSize = 5 * 1024 * 1024
24
- include KinesisHelper::API::BatchRequest
22
+ RequestType = :streams
23
+ BatchRequestLimitCount = 500
24
+ BatchRequestLimitSize = 5 * 1024 * 1024
25
+ include KinesisHelper::API::BatchRequest
25
26
 
26
- config_param :stream_name, :string
27
- config_param :partition_key, :string, default: nil
27
+ config_param :stream_name, :string
28
+ config_param :partition_key, :string, default: nil
28
29
 
29
- def configure(conf)
30
- super
31
- @key_formatter = key_formatter_create
32
- end
30
+ def configure(conf)
31
+ super
32
+ @key_formatter = key_formatter_create
33
+ end
33
34
 
34
- def format(tag, time, record)
35
- format_for_api do
36
- data = @data_formatter.call(tag, time, record)
37
- key = @key_formatter.call(record)
38
- [data, key]
35
+ def format(tag, time, record)
36
+ format_for_api do
37
+ data = @data_formatter.call(tag, time, record)
38
+ key = @key_formatter.call(record)
39
+ [data, key]
40
+ end
39
41
  end
40
- end
41
42
 
42
- def write(chunk)
43
- write_records_batch(chunk) do |batch|
44
- records = batch.map{|(data, partition_key)|
45
- { data: data, partition_key: partition_key }
46
- }
47
- client.put_records(
48
- stream_name: @stream_name,
49
- records: records,
50
- )
43
+ def write(chunk)
44
+ write_records_batch(chunk) do |batch|
45
+ records = batch.map{|(data, partition_key)|
46
+ { data: data, partition_key: partition_key }
47
+ }
48
+ client.put_records(
49
+ stream_name: @stream_name,
50
+ records: records,
51
+ )
52
+ end
51
53
  end
52
- end
53
54
 
54
- private
55
+ private
55
56
 
56
- def key_formatter_create
57
- if @partition_key.nil?
58
- ->(record) { SecureRandom.hex(16) }
59
- else
60
- ->(record) {
61
- if !record.key?(@partition_key)
62
- raise KeyNotFoundError.new(@partition_key, record)
63
- end
64
- record[@partition_key]
65
- }
57
+ def key_formatter_create
58
+ if @partition_key.nil?
59
+ ->(record) { SecureRandom.hex(16) }
60
+ else
61
+ ->(record) {
62
+ if !record.key?(@partition_key)
63
+ raise KeyNotFoundError.new(@partition_key, record)
64
+ end
65
+ record[@partition_key]
66
+ }
67
+ end
66
68
  end
67
69
  end
68
70
  end
data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb CHANGED
@@ -16,60 +16,62 @@ require 'fluent/plugin/kinesis'
16
16
  require 'fluent/plugin/kinesis_helper/aggregator'
17
17
 
18
18
  module Fluent
19
- class KinesisStreamsAggregatedOutput < KinesisOutput
20
- Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
21
- include KinesisHelper::Aggregator::Mixin
19
+ module Plugin
20
+ class KinesisStreamsAggregatedOutput < KinesisOutput
21
+ Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
22
+ include KinesisHelper::Aggregator::Mixin
22
23
 
23
- RequestType = :streams_aggregated
24
- BatchRequestLimitCount = 100_000
25
- BatchRequestLimitSize = 1024 * 1024
26
- include KinesisHelper::API::BatchRequest
24
+ RequestType = :streams_aggregated
25
+ BatchRequestLimitCount = 100_000
26
+ BatchRequestLimitSize = 1024 * 1024
27
+ include KinesisHelper::API::BatchRequest
27
28
 
28
- config_param :stream_name, :string
29
- config_param :fixed_partition_key, :string, default: nil
29
+ config_param :stream_name, :string
30
+ config_param :fixed_partition_key, :string, default: nil
30
31
 
31
- def configure(conf)
32
- super
33
- @partition_key_generator = create_partition_key_generator
34
- @batch_request_max_size -= offset
35
- @max_record_size -= offset
36
- end
32
+ def configure(conf)
33
+ super
34
+ @partition_key_generator = create_partition_key_generator
35
+ @batch_request_max_size -= offset
36
+ @max_record_size -= offset
37
+ end
37
38
 
38
- def format(tag, time, record)
39
- format_for_api do
40
- [@data_formatter.call(tag, time, record)]
39
+ def format(tag, time, record)
40
+ format_for_api do
41
+ [@data_formatter.call(tag, time, record)]
42
+ end
41
43
  end
42
- end
43
44
 
44
- def write(chunk)
45
- write_records_batch(chunk) do |batch|
46
- key = @partition_key_generator.call
47
- records = batch.map{|(data)|data}
48
- client.put_records(
49
- stream_name: @stream_name,
50
- records: [{
51
- partition_key: key,
52
- data: aggregator.aggregate(records, key),
53
- }],
54
- )
45
+ def write(chunk)
46
+ write_records_batch(chunk) do |batch|
47
+ key = @partition_key_generator.call
48
+ records = batch.map{|(data)|data}
49
+ client.put_records(
50
+ stream_name: @stream_name,
51
+ records: [{
52
+ partition_key: key,
53
+ data: aggregator.aggregate(records, key),
54
+ }],
55
+ )
56
+ end
55
57
  end
56
- end
57
58
 
58
- def offset
59
- @offset ||= AggregateOffset + @partition_key_generator.call.size*2
60
- end
59
+ def offset
60
+ @offset ||= AggregateOffset + @partition_key_generator.call.size*2
61
+ end
61
62
 
62
- private
63
+ private
63
64
 
64
- def size_of_values(record)
65
- super(record) + RecordOffset
66
- end
65
+ def size_of_values(record)
66
+ super(record) + RecordOffset
67
+ end
67
68
 
68
- def create_partition_key_generator
69
- if @fixed_partition_key.nil?
70
- ->() { SecureRandom.hex(16) }
71
- else
72
- ->() { @fixed_partition_key }
69
+ def create_partition_key_generator
70
+ if @fixed_partition_key.nil?
71
+ ->() { SecureRandom.hex(16) }
72
+ else
73
+ ->() { @fixed_partition_key }
74
+ end
73
75
  end
74
76
  end
75
77
  end
data/lib/fluent_plugin_kinesis/version.rb CHANGED
@@ -13,5 +13,5 @@
13
13
  # language governing permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '2.2.0'
16
+ VERSION = '3.0.0'
17
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0
4
+ version: 3.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-07-13 00:00:00.000000000 Z
11
+ date: 2019-03-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -16,7 +16,7 @@ dependencies:
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.12.35
19
+ version: 0.14.10
20
20
  - - "<"
21
21
  - !ruby/object:Gem::Version
22
22
  version: '2'
@@ -26,76 +26,102 @@ dependencies:
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 0.12.35
29
+ version: 0.14.10
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: aws-sdk
34
+ name: aws-sdk-kinesis
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - ">="
37
+ - - "~>"
38
38
  - !ruby/object:Gem::Version
39
- version: 2.9.9
40
- - - "<"
39
+ version: '1'
40
+ - - "!="
41
41
  - !ruby/object:Gem::Version
42
- version: '4'
42
+ version: '1.4'
43
+ - - "!="
44
+ - !ruby/object:Gem::Version
45
+ version: '1.5'
43
46
  type: :runtime
44
47
  prerelease: false
45
48
  version_requirements: !ruby/object:Gem::Requirement
46
49
  requirements:
47
- - - ">="
50
+ - - "~>"
48
51
  - !ruby/object:Gem::Version
49
- version: 2.9.9
50
- - - "<"
52
+ version: '1'
53
+ - - "!="
51
54
  - !ruby/object:Gem::Version
52
- version: '4'
55
+ version: '1.4'
56
+ - - "!="
57
+ - !ruby/object:Gem::Version
58
+ version: '1.5'
53
59
  - !ruby/object:Gem::Dependency
54
- name: google-protobuf
60
+ name: aws-sdk-firehose
55
61
  requirement: !ruby/object:Gem::Requirement
56
62
  requirements:
57
63
  - - "~>"
58
64
  - !ruby/object:Gem::Version
59
- version: '3'
60
- - - "<"
65
+ version: '1'
66
+ - - "!="
67
+ - !ruby/object:Gem::Version
68
+ version: '1.5'
69
+ - - "!="
61
70
  - !ruby/object:Gem::Version
62
- version: '3.7'
71
+ version: '1.9'
63
72
  type: :runtime
64
73
  prerelease: false
65
74
  version_requirements: !ruby/object:Gem::Requirement
75
+ requirements:
76
+ - - "~>"
77
+ - !ruby/object:Gem::Version
78
+ version: '1'
79
+ - - "!="
80
+ - !ruby/object:Gem::Version
81
+ version: '1.5'
82
+ - - "!="
83
+ - !ruby/object:Gem::Version
84
+ version: '1.9'
85
+ - !ruby/object:Gem::Dependency
86
+ name: google-protobuf
87
+ requirement: !ruby/object:Gem::Requirement
66
88
  requirements:
67
89
  - - "~>"
68
90
  - !ruby/object:Gem::Version
69
91
  version: '3'
70
- - - "<"
92
+ type: :runtime
93
+ prerelease: false
94
+ version_requirements: !ruby/object:Gem::Requirement
95
+ requirements:
96
+ - - "~>"
71
97
  - !ruby/object:Gem::Version
72
- version: '3.7'
98
+ version: '3'
73
99
  - !ruby/object:Gem::Dependency
74
100
  name: bundler
75
101
  requirement: !ruby/object:Gem::Requirement
76
102
  requirements:
77
- - - "~>"
103
+ - - ">="
78
104
  - !ruby/object:Gem::Version
79
105
  version: '1.10'
80
106
  type: :development
81
107
  prerelease: false
82
108
  version_requirements: !ruby/object:Gem::Requirement
83
109
  requirements:
84
- - - "~>"
110
+ - - ">="
85
111
  - !ruby/object:Gem::Version
86
112
  version: '1.10'
87
113
  - !ruby/object:Gem::Dependency
88
114
  name: rake
89
115
  requirement: !ruby/object:Gem::Requirement
90
116
  requirements:
91
- - - "~>"
117
+ - - ">="
92
118
  - !ruby/object:Gem::Version
93
119
  version: '10.0'
94
120
  type: :development
95
121
  prerelease: false
96
122
  version_requirements: !ruby/object:Gem::Requirement
97
123
  requirements:
98
- - - "~>"
124
+ - - ">="
99
125
  - !ruby/object:Gem::Version
100
126
  version: '10.0'
101
127
  - !ruby/object:Gem::Dependency
@@ -230,9 +256,12 @@ executables: []
230
256
  extensions: []
231
257
  extra_rdoc_files: []
232
258
  files:
259
+ - ".github/PULL_REQUEST_TEMPLATE.md"
233
260
  - ".gitignore"
234
261
  - ".travis.yml"
235
262
  - CHANGELOG.md
263
+ - CODE_OF_CONDUCT.md
264
+ - CONTRIBUTING.md
236
265
  - CONTRIBUTORS.txt
237
266
  - Gemfile
238
267
  - LICENSE.txt
@@ -242,7 +271,11 @@ files:
242
271
  - Rakefile
243
272
  - benchmark/task.rake
244
273
  - fluent-plugin-kinesis.gemspec
245
- - gemfiles/Gemfile.td-agent-2.3.5
274
+ - gemfiles/Gemfile.fluentd-0.14.10
275
+ - gemfiles/Gemfile.td-agent-3.1.1
276
+ - gemfiles/Gemfile.td-agent-3.2.0
277
+ - gemfiles/Gemfile.td-agent-3.2.1
278
+ - gemfiles/Gemfile.td-agent-3.3.0
246
279
  - lib/fluent/plugin/kinesis.rb
247
280
  - lib/fluent/plugin/kinesis_helper/aggregator.rb
248
281
  - lib/fluent/plugin/kinesis_helper/api.rb