fluent-plugin-kinesis 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +11 -0
- data/CHANGELOG.md +8 -0
- data/Makefile +10 -11
- data/README.md +2 -8
- data/Rakefile +2 -1
- data/benchmark/dummy.conf +0 -0
- data/benchmark/task.rake +72 -0
- data/fluent-plugin-kinesis.gemspec +0 -1
- data/gemfiles/Gemfile.fluentd-0.12 +20 -0
- data/lib/fluent/plugin/kinesis_helper.rb +1 -0
- data/lib/fluent/plugin/kinesis_helper/class_methods.rb +5 -4
- data/lib/fluent/plugin/kinesis_helper/initialize.rb +1 -0
- data/lib/fluent/plugin/out_kinesis_producer.rb +11 -1
- data/lib/fluent_plugin_kinesis/version.rb +1 -1
- metadata +5 -20
- data/benchmark/dummer.conf +0 -13
- data/benchmark/firehose.conf +0 -24
- data/benchmark/producer.conf +0 -28
- data/benchmark/streams.conf +0 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f1802cbf044412c40fee66e52412c78a8f00c58
|
4
|
+
data.tar.gz: 3bb9b8bb9b5a079cbf3fa61f90b6b3e16cd81e77
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: acf1eefc4d2d071cb81cf22dd1969f5fe60276bfecd3616ac193b19106073c3d585bef51d3a5750f7a803a914e5a4cab75ea32d37153380d19b910c40891f35e
|
7
|
+
data.tar.gz: ceed8d7790f50d07256fc7a927ee053d3808d92ee6d444d87c96c30813283a8758b21d13364b476747efd777a1882dc729a0a7d3308777258a3214eb59f6c939
|
data/.travis.yml
CHANGED
@@ -8,6 +8,7 @@ rvm:
|
|
8
8
|
gemfile:
|
9
9
|
- Gemfile
|
10
10
|
- gemfiles/Gemfile.fluentd-0.10.58
|
11
|
+
- gemfiles/Gemfile.fluentd-0.12
|
11
12
|
|
12
13
|
os:
|
13
14
|
- linux
|
@@ -22,5 +23,15 @@ sudo: false
|
|
22
23
|
|
23
24
|
matrix:
|
24
25
|
exclude:
|
26
|
+
- rvm: 2.0.0
|
27
|
+
os: osx
|
28
|
+
- rvm: 2.0.0
|
29
|
+
gemfile: Gemfile
|
30
|
+
- rvm: 2.1
|
31
|
+
os: osx
|
32
|
+
gemfile: gemfiles/Gemfile.fluentd-0.10.58
|
33
|
+
- rvm: 2.1
|
34
|
+
os: osx
|
35
|
+
gemfile: gemfiles/Gemfile.fluentd-0.12
|
25
36
|
- rvm: 2.2
|
26
37
|
os: osx
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## 1.1.0
|
4
|
+
|
5
|
+
- Feature - Derive stream name from fluentd tag for KPL [#67](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/67)
|
6
|
+
- Enhancement - Make http_proxy parameter secret [#64](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/64)
|
7
|
+
- Bug fix - Plugin incompatible with new fluentd release 0.14 [#70](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/70)
|
8
|
+
- Misc - Fix legacy test and reduce travis tests [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74)
|
9
|
+
- Misc - Some test, benchmark improvement [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74), [#75](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/75), [#76](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/76)
|
10
|
+
|
3
11
|
## 1.0.1
|
4
12
|
|
5
13
|
- Bug fix - Instance profile credentials expiring [#58](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/58)
|
data/Makefile
CHANGED
@@ -12,26 +12,25 @@
|
|
12
12
|
# express or implied. See the License for the specific language governing
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
|
15
|
-
.PHONY: install streams
|
15
|
+
.PHONY: test install benchmark benchmark-streams benchmark-producer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
|
16
16
|
|
17
17
|
all:
|
18
18
|
bundle install
|
19
|
-
bundle exec rake
|
19
|
+
bundle exec rake binaries
|
20
|
+
|
21
|
+
test:
|
22
|
+
bundle exec rake test
|
20
23
|
|
21
24
|
install:
|
22
25
|
bundle exec rake install:local
|
23
26
|
|
24
|
-
streams
|
25
|
-
bundle exec fluentd -c benchmark/streams.conf -vv
|
26
|
-
|
27
|
-
firehose:
|
28
|
-
bundle exec fluentd -c benchmark/firehose.conf -vv
|
27
|
+
benchmark: benchmark-streams benchmark-producer
|
29
28
|
|
30
|
-
|
31
|
-
bundle exec
|
29
|
+
benchmark-streams:
|
30
|
+
bundle exec rake benchmark TYPE=streams
|
32
31
|
|
33
|
-
|
34
|
-
bundle exec
|
32
|
+
benchmark-producer:
|
33
|
+
bundle exec rake benchmark TYPE=producer
|
35
34
|
|
36
35
|
hello:
|
37
36
|
echo Hello World | bundle exec fluent-cat --none dummy
|
data/README.md
CHANGED
@@ -527,15 +527,9 @@ To launch `fluentd` process with this plugin for development, follow the steps b
|
|
527
527
|
git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
|
528
528
|
cd aws-fluent-plugin-kinesis
|
529
529
|
make # will install gems and download KPL jar file and extract binaries
|
530
|
-
|
530
|
+
bundle exec fluentd -c /path/to/fluent.conf
|
531
531
|
|
532
|
-
|
533
|
-
|
534
|
-
make hello
|
535
|
-
|
536
|
-
Also, you can test streaming log data by `dummer`:
|
537
|
-
|
538
|
-
make dummer # keep writing to /tmp/dummy.log
|
532
|
+
If you want to run benchmark, use `make benchmark`.
|
539
533
|
|
540
534
|
## Contributing
|
541
535
|
|
data/Rakefile
CHANGED
File without changes
|
data/benchmark/task.rake
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Amazon Software License (the "License").
|
5
|
+
# You may not use this file except in compliance with the License.
|
6
|
+
# A copy of the License is located at
|
7
|
+
#
|
8
|
+
# http://aws.amazon.com/asl/
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is distributed
|
11
|
+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
+
# express or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
|
15
|
+
require_relative '../test/dummy_server'
|
16
|
+
|
17
|
+
task :benchmark do
|
18
|
+
server = DummyServer.start
|
19
|
+
conf = profile_conf(ENV["TYPE"] || 'streams', ENV["RATE"] || 1000, server)
|
20
|
+
pid = spawn("bundle exec fluentd -i '#{conf}' -c benchmark/dummy.conf")
|
21
|
+
sleep 10
|
22
|
+
Process.kill("TERM", pid)
|
23
|
+
Process.wait
|
24
|
+
puts "Results: requets: #{server.requests.size}, raw_records: #{server.raw_records.size}, records: #{server.records.size}"
|
25
|
+
server.shutdown
|
26
|
+
end
|
27
|
+
|
28
|
+
def profile_conf(type, rate, server)
|
29
|
+
additional_conf = case type
|
30
|
+
when 'streams', 'firehose'
|
31
|
+
<<-EOS
|
32
|
+
endpoint https://localhost:#{server.port}
|
33
|
+
ssl_verify_peer false
|
34
|
+
EOS
|
35
|
+
when 'producer'
|
36
|
+
<<-EOS
|
37
|
+
debug true
|
38
|
+
<kinesis_producer>
|
39
|
+
custom_endpoint localhost
|
40
|
+
port #{server.port}
|
41
|
+
verify_certificate false
|
42
|
+
record_max_buffered_time 1000
|
43
|
+
log_level error
|
44
|
+
</kinesis_producer>
|
45
|
+
EOS
|
46
|
+
end
|
47
|
+
|
48
|
+
conf = <<-EOS
|
49
|
+
<source>
|
50
|
+
@type dummy
|
51
|
+
tag dummy
|
52
|
+
rate #{rate}
|
53
|
+
</source>
|
54
|
+
|
55
|
+
<match dummy>
|
56
|
+
@type kinesis_#{type}
|
57
|
+
flush_interval 1
|
58
|
+
buffer_chunk_limit 1m
|
59
|
+
try_flush_interval 0.1
|
60
|
+
queued_chunk_flush_interval 0.01
|
61
|
+
@log_level debug
|
62
|
+
|
63
|
+
num_threads 100
|
64
|
+
|
65
|
+
region ap-northeast-1
|
66
|
+
stream_name fluent-plugin-test
|
67
|
+
|
68
|
+
#{additional_conf}
|
69
|
+
</match>
|
70
|
+
EOS
|
71
|
+
conf
|
72
|
+
end
|
@@ -46,7 +46,6 @@ Gem::Specification.new do |spec|
|
|
46
46
|
spec.add_development_dependency "pry-byebug", ">= 3.3.0"
|
47
47
|
spec.add_development_dependency "pry-stack_explorer", ">= 0.4.9.2"
|
48
48
|
spec.add_development_dependency "net-empty_port", ">= 0.0.2"
|
49
|
-
spec.add_development_dependency "dummer", ">= 0.4.0"
|
50
49
|
spec.add_development_dependency "rubyzip", ">= 1.0.0"
|
51
50
|
spec.add_development_dependency "mocha", ">= 1.1.0"
|
52
51
|
spec.add_development_dependency "webmock", ">= 1.24.2"
|
@@ -0,0 +1,20 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Amazon Software License (the "License").
|
5
|
+
# You may not use this file except in compliance with the License.
|
6
|
+
# A copy of the License is located at
|
7
|
+
#
|
8
|
+
# http://aws.amazon.com/asl/
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is distributed
|
11
|
+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
+
# express or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
|
15
|
+
source 'https://rubygems.org'
|
16
|
+
|
17
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
18
|
+
gemspec path: ".."
|
19
|
+
|
20
|
+
gem "fluentd", "~> 0.12.26"
|
@@ -45,9 +45,10 @@ module Fluent
|
|
45
45
|
|
46
46
|
def config_param_for_producer
|
47
47
|
const_set(:RequestType, :producer)
|
48
|
-
config_param :stream_name,
|
49
|
-
config_param :
|
50
|
-
config_param :
|
48
|
+
config_param :stream_name, :string, default: nil
|
49
|
+
config_param :stream_name_prefix, :string, default: nil
|
50
|
+
config_param :region, :string, default: nil
|
51
|
+
config_param :partition_key, :string, default: nil
|
51
52
|
config_param_for_credentials
|
52
53
|
config_param_for_format
|
53
54
|
config_param_for_debug
|
@@ -69,7 +70,7 @@ module Fluent
|
|
69
70
|
end
|
70
71
|
|
71
72
|
def config_param_for_sdk
|
72
|
-
config_param :http_proxy, :string, default: nil
|
73
|
+
config_param :http_proxy, :string, default: nil, secret: true
|
73
74
|
config_param :endpoint, :string, default: nil
|
74
75
|
config_param :ssl_verify_peer, :bool, default: true
|
75
76
|
end
|
@@ -32,6 +32,7 @@ module Fluent
|
|
32
32
|
include API
|
33
33
|
when kpl?
|
34
34
|
require 'fluent/plugin/kinesis_helper/kpl'
|
35
|
+
require 'fluent/version'
|
35
36
|
if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.12.20')
|
36
37
|
# Backport from https://github.com/fluent/fluentd/pull/757
|
37
38
|
require 'fluent/plugin/patched_detach_process_impl'
|
@@ -20,6 +20,16 @@ module Fluent
|
|
20
20
|
Fluent::Plugin.register_output('kinesis_producer', self)
|
21
21
|
config_param_for_producer
|
22
22
|
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
unless @stream_name or @stream_name_prefix
|
26
|
+
raise Fluent::ConfigError, "'stream_name' or 'stream_name_prefix' is required"
|
27
|
+
end
|
28
|
+
if @stream_name and @stream_name_prefix
|
29
|
+
raise Fluent::ConfigError, "Only one of 'stream_name' or 'stream_name_prefix' is allowed"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
23
33
|
def write(chunk)
|
24
34
|
records = convert_to_records(chunk)
|
25
35
|
wait_futures(write_chunk_to_kpl(records))
|
@@ -31,7 +41,7 @@ module Fluent
|
|
31
41
|
{
|
32
42
|
data: data_format(tag, time, record),
|
33
43
|
partition_key: key(record),
|
34
|
-
stream_name: @stream_name,
|
44
|
+
stream_name: @stream_name ? @stream_name : @stream_name_prefix + tag,
|
35
45
|
}
|
36
46
|
end
|
37
47
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kinesis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amazon Web Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -198,20 +198,6 @@ dependencies:
|
|
198
198
|
- - ">="
|
199
199
|
- !ruby/object:Gem::Version
|
200
200
|
version: 0.0.2
|
201
|
-
- !ruby/object:Gem::Dependency
|
202
|
-
name: dummer
|
203
|
-
requirement: !ruby/object:Gem::Requirement
|
204
|
-
requirements:
|
205
|
-
- - ">="
|
206
|
-
- !ruby/object:Gem::Version
|
207
|
-
version: 0.4.0
|
208
|
-
type: :development
|
209
|
-
prerelease: false
|
210
|
-
version_requirements: !ruby/object:Gem::Requirement
|
211
|
-
requirements:
|
212
|
-
- - ">="
|
213
|
-
- !ruby/object:Gem::Version
|
214
|
-
version: 0.4.0
|
215
201
|
- !ruby/object:Gem::Dependency
|
216
202
|
name: rubyzip
|
217
203
|
requirement: !ruby/object:Gem::Requirement
|
@@ -288,12 +274,11 @@ files:
|
|
288
274
|
- amazon-kinesis-producer-native-binaries/linux/kinesis_producer
|
289
275
|
- amazon-kinesis-producer-native-binaries/osx/kinesis_producer
|
290
276
|
- amazon-kinesis-producer-native-binaries/windows/kinesis_producer.exe
|
291
|
-
- benchmark/
|
292
|
-
- benchmark/
|
293
|
-
- benchmark/producer.conf
|
294
|
-
- benchmark/streams.conf
|
277
|
+
- benchmark/dummy.conf
|
278
|
+
- benchmark/task.rake
|
295
279
|
- fluent-plugin-kinesis.gemspec
|
296
280
|
- gemfiles/Gemfile.fluentd-0.10.58
|
281
|
+
- gemfiles/Gemfile.fluentd-0.12
|
297
282
|
- lib/fluent/plugin/kinesis_helper.rb
|
298
283
|
- lib/fluent/plugin/kinesis_helper/api.rb
|
299
284
|
- lib/fluent/plugin/kinesis_helper/class_methods.rb
|
data/benchmark/dummer.conf
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
configure 'sample' do
|
2
|
-
output "/tmp/dummy.log"
|
3
|
-
rate 5000
|
4
|
-
delimiter "\t"
|
5
|
-
labeled true
|
6
|
-
field :id, type: :integer, countup: true, format: "%04d"
|
7
|
-
field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
|
8
|
-
field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
|
9
|
-
field :method, type: :string, any: %w[GET POST PUT]
|
10
|
-
field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
|
11
|
-
field :reqtime, type: :float, range: 0.1..5.0
|
12
|
-
field :foobar, type: :string, length: 8
|
13
|
-
end
|
data/benchmark/firehose.conf
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
<source>
|
2
|
-
@type tail
|
3
|
-
path /tmp/dummy.log
|
4
|
-
format none
|
5
|
-
tag dummy
|
6
|
-
</source>
|
7
|
-
|
8
|
-
<source>
|
9
|
-
@type forward
|
10
|
-
</source>
|
11
|
-
|
12
|
-
<match dummy>
|
13
|
-
@type kinesis_firehose
|
14
|
-
flush_interval 1
|
15
|
-
buffer_chunk_limit 1m
|
16
|
-
try_flush_interval 0.1
|
17
|
-
queued_chunk_flush_interval 0.01
|
18
|
-
num_threads 15
|
19
|
-
detach_process 5
|
20
|
-
log_level debug
|
21
|
-
|
22
|
-
region us-west-2
|
23
|
-
delivery_stream_name fluent-plugin-test
|
24
|
-
</match>
|
data/benchmark/producer.conf
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
<source>
|
2
|
-
@type tail
|
3
|
-
path /tmp/dummy.log
|
4
|
-
format none
|
5
|
-
tag dummy
|
6
|
-
</source>
|
7
|
-
|
8
|
-
<source>
|
9
|
-
@type forward
|
10
|
-
</source>
|
11
|
-
|
12
|
-
<match dummy>
|
13
|
-
@type kinesis_producer
|
14
|
-
flush_interval 1
|
15
|
-
buffer_chunk_limit 1m
|
16
|
-
try_flush_interval 0.1
|
17
|
-
queued_chunk_flush_interval 0.01
|
18
|
-
num_threads 15
|
19
|
-
detach_process 5
|
20
|
-
log_level debug
|
21
|
-
|
22
|
-
region ap-northeast-1
|
23
|
-
stream_name fluent-plugin-test
|
24
|
-
debug true
|
25
|
-
<kinesis_producer>
|
26
|
-
record_max_buffered_time 1000
|
27
|
-
</kinesis_producer>
|
28
|
-
</match>
|
data/benchmark/streams.conf
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
<source>
|
2
|
-
@type tail
|
3
|
-
path /tmp/dummy.log
|
4
|
-
format none
|
5
|
-
tag dummy
|
6
|
-
</source>
|
7
|
-
|
8
|
-
<source>
|
9
|
-
@type forward
|
10
|
-
</source>
|
11
|
-
|
12
|
-
<match dummy>
|
13
|
-
@type kinesis_streams
|
14
|
-
flush_interval 1
|
15
|
-
buffer_chunk_limit 1m
|
16
|
-
try_flush_interval 0.1
|
17
|
-
queued_chunk_flush_interval 0.01
|
18
|
-
num_threads 15
|
19
|
-
detach_process 5
|
20
|
-
log_level debug
|
21
|
-
|
22
|
-
region ap-northeast-1
|
23
|
-
stream_name fluent-plugin-test
|
24
|
-
</match>
|