fluent-plugin-kinesis 1.0.1 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +11 -0
- data/CHANGELOG.md +8 -0
- data/Makefile +10 -11
- data/README.md +2 -8
- data/Rakefile +2 -1
- data/benchmark/dummy.conf +0 -0
- data/benchmark/task.rake +72 -0
- data/fluent-plugin-kinesis.gemspec +0 -1
- data/gemfiles/Gemfile.fluentd-0.12 +20 -0
- data/lib/fluent/plugin/kinesis_helper.rb +1 -0
- data/lib/fluent/plugin/kinesis_helper/class_methods.rb +5 -4
- data/lib/fluent/plugin/kinesis_helper/initialize.rb +1 -0
- data/lib/fluent/plugin/out_kinesis_producer.rb +11 -1
- data/lib/fluent_plugin_kinesis/version.rb +1 -1
- metadata +5 -20
- data/benchmark/dummer.conf +0 -13
- data/benchmark/firehose.conf +0 -24
- data/benchmark/producer.conf +0 -28
- data/benchmark/streams.conf +0 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9f1802cbf044412c40fee66e52412c78a8f00c58
|
4
|
+
data.tar.gz: 3bb9b8bb9b5a079cbf3fa61f90b6b3e16cd81e77
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: acf1eefc4d2d071cb81cf22dd1969f5fe60276bfecd3616ac193b19106073c3d585bef51d3a5750f7a803a914e5a4cab75ea32d37153380d19b910c40891f35e
|
7
|
+
data.tar.gz: ceed8d7790f50d07256fc7a927ee053d3808d92ee6d444d87c96c30813283a8758b21d13364b476747efd777a1882dc729a0a7d3308777258a3214eb59f6c939
|
data/.travis.yml
CHANGED
@@ -8,6 +8,7 @@ rvm:
|
|
8
8
|
gemfile:
|
9
9
|
- Gemfile
|
10
10
|
- gemfiles/Gemfile.fluentd-0.10.58
|
11
|
+
- gemfiles/Gemfile.fluentd-0.12
|
11
12
|
|
12
13
|
os:
|
13
14
|
- linux
|
@@ -22,5 +23,15 @@ sudo: false
|
|
22
23
|
|
23
24
|
matrix:
|
24
25
|
exclude:
|
26
|
+
- rvm: 2.0.0
|
27
|
+
os: osx
|
28
|
+
- rvm: 2.0.0
|
29
|
+
gemfile: Gemfile
|
30
|
+
- rvm: 2.1
|
31
|
+
os: osx
|
32
|
+
gemfile: gemfiles/Gemfile.fluentd-0.10.58
|
33
|
+
- rvm: 2.1
|
34
|
+
os: osx
|
35
|
+
gemfile: gemfiles/Gemfile.fluentd-0.12
|
25
36
|
- rvm: 2.2
|
26
37
|
os: osx
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,13 @@
|
|
1
1
|
# CHANGELOG
|
2
2
|
|
3
|
+
## 1.1.0
|
4
|
+
|
5
|
+
- Feature - Derive stream name from fluentd tag for KPL [#67](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/67)
|
6
|
+
- Enhancement - Make http_proxy parameter secret [#64](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/64)
|
7
|
+
- Bug fix - Plugin incompatible with new fluentd release 0.14 [#70](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/70)
|
8
|
+
- Misc - Fix legacy test and reduce travis tests [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74)
|
9
|
+
- Misc - Some test, benchmark improvement [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74), [#75](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/75), [#76](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/76)
|
10
|
+
|
3
11
|
## 1.0.1
|
4
12
|
|
5
13
|
- Bug fix - Instance profile credentials expiring [#58](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/58)
|
data/Makefile
CHANGED
@@ -12,26 +12,25 @@
|
|
12
12
|
# express or implied. See the License for the specific language governing
|
13
13
|
# permissions and limitations under the License.
|
14
14
|
|
15
|
-
.PHONY: install streams
|
15
|
+
.PHONY: test install benchmark benchmark-streams benchmark-producer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
|
16
16
|
|
17
17
|
all:
|
18
18
|
bundle install
|
19
|
-
bundle exec rake
|
19
|
+
bundle exec rake binaries
|
20
|
+
|
21
|
+
test:
|
22
|
+
bundle exec rake test
|
20
23
|
|
21
24
|
install:
|
22
25
|
bundle exec rake install:local
|
23
26
|
|
24
|
-
streams
|
25
|
-
bundle exec fluentd -c benchmark/streams.conf -vv
|
26
|
-
|
27
|
-
firehose:
|
28
|
-
bundle exec fluentd -c benchmark/firehose.conf -vv
|
27
|
+
benchmark: benchmark-streams benchmark-producer
|
29
28
|
|
30
|
-
|
31
|
-
bundle exec
|
29
|
+
benchmark-streams:
|
30
|
+
bundle exec rake benchmark TYPE=streams
|
32
31
|
|
33
|
-
|
34
|
-
bundle exec
|
32
|
+
benchmark-producer:
|
33
|
+
bundle exec rake benchmark TYPE=producer
|
35
34
|
|
36
35
|
hello:
|
37
36
|
echo Hello World | bundle exec fluent-cat --none dummy
|
data/README.md
CHANGED
@@ -527,15 +527,9 @@ To launch `fluentd` process with this plugin for development, follow the steps b
|
|
527
527
|
git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
|
528
528
|
cd aws-fluent-plugin-kinesis
|
529
529
|
make # will install gems and download KPL jar file and extract binaries
|
530
|
-
|
530
|
+
bundle exec fluentd -c /path/to/fluent.conf
|
531
531
|
|
532
|
-
|
533
|
-
|
534
|
-
make hello
|
535
|
-
|
536
|
-
Also, you can test streaming log data by `dummer`:
|
537
|
-
|
538
|
-
make dummer # keep writing to /tmp/dummy.log
|
532
|
+
If you want to run benchmark, use `make benchmark`.
|
539
533
|
|
540
534
|
## Contributing
|
541
535
|
|
data/Rakefile
CHANGED
File without changes
|
data/benchmark/task.rake
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Amazon Software License (the "License").
|
5
|
+
# You may not use this file except in compliance with the License.
|
6
|
+
# A copy of the License is located at
|
7
|
+
#
|
8
|
+
# http://aws.amazon.com/asl/
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is distributed
|
11
|
+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
+
# express or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
|
15
|
+
require_relative '../test/dummy_server'
|
16
|
+
|
17
|
+
task :benchmark do
|
18
|
+
server = DummyServer.start
|
19
|
+
conf = profile_conf(ENV["TYPE"] || 'streams', ENV["RATE"] || 1000, server)
|
20
|
+
pid = spawn("bundle exec fluentd -i '#{conf}' -c benchmark/dummy.conf")
|
21
|
+
sleep 10
|
22
|
+
Process.kill("TERM", pid)
|
23
|
+
Process.wait
|
24
|
+
puts "Results: requets: #{server.requests.size}, raw_records: #{server.raw_records.size}, records: #{server.records.size}"
|
25
|
+
server.shutdown
|
26
|
+
end
|
27
|
+
|
28
|
+
def profile_conf(type, rate, server)
|
29
|
+
additional_conf = case type
|
30
|
+
when 'streams', 'firehose'
|
31
|
+
<<-EOS
|
32
|
+
endpoint https://localhost:#{server.port}
|
33
|
+
ssl_verify_peer false
|
34
|
+
EOS
|
35
|
+
when 'producer'
|
36
|
+
<<-EOS
|
37
|
+
debug true
|
38
|
+
<kinesis_producer>
|
39
|
+
custom_endpoint localhost
|
40
|
+
port #{server.port}
|
41
|
+
verify_certificate false
|
42
|
+
record_max_buffered_time 1000
|
43
|
+
log_level error
|
44
|
+
</kinesis_producer>
|
45
|
+
EOS
|
46
|
+
end
|
47
|
+
|
48
|
+
conf = <<-EOS
|
49
|
+
<source>
|
50
|
+
@type dummy
|
51
|
+
tag dummy
|
52
|
+
rate #{rate}
|
53
|
+
</source>
|
54
|
+
|
55
|
+
<match dummy>
|
56
|
+
@type kinesis_#{type}
|
57
|
+
flush_interval 1
|
58
|
+
buffer_chunk_limit 1m
|
59
|
+
try_flush_interval 0.1
|
60
|
+
queued_chunk_flush_interval 0.01
|
61
|
+
@log_level debug
|
62
|
+
|
63
|
+
num_threads 100
|
64
|
+
|
65
|
+
region ap-northeast-1
|
66
|
+
stream_name fluent-plugin-test
|
67
|
+
|
68
|
+
#{additional_conf}
|
69
|
+
</match>
|
70
|
+
EOS
|
71
|
+
conf
|
72
|
+
end
|
@@ -46,7 +46,6 @@ Gem::Specification.new do |spec|
|
|
46
46
|
spec.add_development_dependency "pry-byebug", ">= 3.3.0"
|
47
47
|
spec.add_development_dependency "pry-stack_explorer", ">= 0.4.9.2"
|
48
48
|
spec.add_development_dependency "net-empty_port", ">= 0.0.2"
|
49
|
-
spec.add_development_dependency "dummer", ">= 0.4.0"
|
50
49
|
spec.add_development_dependency "rubyzip", ">= 1.0.0"
|
51
50
|
spec.add_development_dependency "mocha", ">= 1.1.0"
|
52
51
|
spec.add_development_dependency "webmock", ">= 1.24.2"
|
@@ -0,0 +1,20 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Amazon Software License (the "License").
|
5
|
+
# You may not use this file except in compliance with the License.
|
6
|
+
# A copy of the License is located at
|
7
|
+
#
|
8
|
+
# http://aws.amazon.com/asl/
|
9
|
+
#
|
10
|
+
# or in the "license" file accompanying this file. This file is distributed
|
11
|
+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
|
12
|
+
# express or implied. See the License for the specific language governing
|
13
|
+
# permissions and limitations under the License.
|
14
|
+
|
15
|
+
source 'https://rubygems.org'
|
16
|
+
|
17
|
+
# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
|
18
|
+
gemspec path: ".."
|
19
|
+
|
20
|
+
gem "fluentd", "~> 0.12.26"
|
@@ -45,9 +45,10 @@ module Fluent
|
|
45
45
|
|
46
46
|
def config_param_for_producer
|
47
47
|
const_set(:RequestType, :producer)
|
48
|
-
config_param :stream_name,
|
49
|
-
config_param :
|
50
|
-
config_param :
|
48
|
+
config_param :stream_name, :string, default: nil
|
49
|
+
config_param :stream_name_prefix, :string, default: nil
|
50
|
+
config_param :region, :string, default: nil
|
51
|
+
config_param :partition_key, :string, default: nil
|
51
52
|
config_param_for_credentials
|
52
53
|
config_param_for_format
|
53
54
|
config_param_for_debug
|
@@ -69,7 +70,7 @@ module Fluent
|
|
69
70
|
end
|
70
71
|
|
71
72
|
def config_param_for_sdk
|
72
|
-
config_param :http_proxy, :string, default: nil
|
73
|
+
config_param :http_proxy, :string, default: nil, secret: true
|
73
74
|
config_param :endpoint, :string, default: nil
|
74
75
|
config_param :ssl_verify_peer, :bool, default: true
|
75
76
|
end
|
@@ -32,6 +32,7 @@ module Fluent
|
|
32
32
|
include API
|
33
33
|
when kpl?
|
34
34
|
require 'fluent/plugin/kinesis_helper/kpl'
|
35
|
+
require 'fluent/version'
|
35
36
|
if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.12.20')
|
36
37
|
# Backport from https://github.com/fluent/fluentd/pull/757
|
37
38
|
require 'fluent/plugin/patched_detach_process_impl'
|
@@ -20,6 +20,16 @@ module Fluent
|
|
20
20
|
Fluent::Plugin.register_output('kinesis_producer', self)
|
21
21
|
config_param_for_producer
|
22
22
|
|
23
|
+
def configure(conf)
|
24
|
+
super
|
25
|
+
unless @stream_name or @stream_name_prefix
|
26
|
+
raise Fluent::ConfigError, "'stream_name' or 'stream_name_prefix' is required"
|
27
|
+
end
|
28
|
+
if @stream_name and @stream_name_prefix
|
29
|
+
raise Fluent::ConfigError, "Only one of 'stream_name' or 'stream_name_prefix' is allowed"
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
23
33
|
def write(chunk)
|
24
34
|
records = convert_to_records(chunk)
|
25
35
|
wait_futures(write_chunk_to_kpl(records))
|
@@ -31,7 +41,7 @@ module Fluent
|
|
31
41
|
{
|
32
42
|
data: data_format(tag, time, record),
|
33
43
|
partition_key: key(record),
|
34
|
-
stream_name: @stream_name,
|
44
|
+
stream_name: @stream_name ? @stream_name : @stream_name_prefix + tag,
|
35
45
|
}
|
36
46
|
end
|
37
47
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-kinesis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Amazon Web Services
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -198,20 +198,6 @@ dependencies:
|
|
198
198
|
- - ">="
|
199
199
|
- !ruby/object:Gem::Version
|
200
200
|
version: 0.0.2
|
201
|
-
- !ruby/object:Gem::Dependency
|
202
|
-
name: dummer
|
203
|
-
requirement: !ruby/object:Gem::Requirement
|
204
|
-
requirements:
|
205
|
-
- - ">="
|
206
|
-
- !ruby/object:Gem::Version
|
207
|
-
version: 0.4.0
|
208
|
-
type: :development
|
209
|
-
prerelease: false
|
210
|
-
version_requirements: !ruby/object:Gem::Requirement
|
211
|
-
requirements:
|
212
|
-
- - ">="
|
213
|
-
- !ruby/object:Gem::Version
|
214
|
-
version: 0.4.0
|
215
201
|
- !ruby/object:Gem::Dependency
|
216
202
|
name: rubyzip
|
217
203
|
requirement: !ruby/object:Gem::Requirement
|
@@ -288,12 +274,11 @@ files:
|
|
288
274
|
- amazon-kinesis-producer-native-binaries/linux/kinesis_producer
|
289
275
|
- amazon-kinesis-producer-native-binaries/osx/kinesis_producer
|
290
276
|
- amazon-kinesis-producer-native-binaries/windows/kinesis_producer.exe
|
291
|
-
- benchmark/
|
292
|
-
- benchmark/
|
293
|
-
- benchmark/producer.conf
|
294
|
-
- benchmark/streams.conf
|
277
|
+
- benchmark/dummy.conf
|
278
|
+
- benchmark/task.rake
|
295
279
|
- fluent-plugin-kinesis.gemspec
|
296
280
|
- gemfiles/Gemfile.fluentd-0.10.58
|
281
|
+
- gemfiles/Gemfile.fluentd-0.12
|
297
282
|
- lib/fluent/plugin/kinesis_helper.rb
|
298
283
|
- lib/fluent/plugin/kinesis_helper/api.rb
|
299
284
|
- lib/fluent/plugin/kinesis_helper/class_methods.rb
|
data/benchmark/dummer.conf
DELETED
@@ -1,13 +0,0 @@
|
|
1
|
-
configure 'sample' do
|
2
|
-
output "/tmp/dummy.log"
|
3
|
-
rate 5000
|
4
|
-
delimiter "\t"
|
5
|
-
labeled true
|
6
|
-
field :id, type: :integer, countup: true, format: "%04d"
|
7
|
-
field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
|
8
|
-
field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
|
9
|
-
field :method, type: :string, any: %w[GET POST PUT]
|
10
|
-
field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
|
11
|
-
field :reqtime, type: :float, range: 0.1..5.0
|
12
|
-
field :foobar, type: :string, length: 8
|
13
|
-
end
|
data/benchmark/firehose.conf
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
<source>
|
2
|
-
@type tail
|
3
|
-
path /tmp/dummy.log
|
4
|
-
format none
|
5
|
-
tag dummy
|
6
|
-
</source>
|
7
|
-
|
8
|
-
<source>
|
9
|
-
@type forward
|
10
|
-
</source>
|
11
|
-
|
12
|
-
<match dummy>
|
13
|
-
@type kinesis_firehose
|
14
|
-
flush_interval 1
|
15
|
-
buffer_chunk_limit 1m
|
16
|
-
try_flush_interval 0.1
|
17
|
-
queued_chunk_flush_interval 0.01
|
18
|
-
num_threads 15
|
19
|
-
detach_process 5
|
20
|
-
log_level debug
|
21
|
-
|
22
|
-
region us-west-2
|
23
|
-
delivery_stream_name fluent-plugin-test
|
24
|
-
</match>
|
data/benchmark/producer.conf
DELETED
@@ -1,28 +0,0 @@
|
|
1
|
-
<source>
|
2
|
-
@type tail
|
3
|
-
path /tmp/dummy.log
|
4
|
-
format none
|
5
|
-
tag dummy
|
6
|
-
</source>
|
7
|
-
|
8
|
-
<source>
|
9
|
-
@type forward
|
10
|
-
</source>
|
11
|
-
|
12
|
-
<match dummy>
|
13
|
-
@type kinesis_producer
|
14
|
-
flush_interval 1
|
15
|
-
buffer_chunk_limit 1m
|
16
|
-
try_flush_interval 0.1
|
17
|
-
queued_chunk_flush_interval 0.01
|
18
|
-
num_threads 15
|
19
|
-
detach_process 5
|
20
|
-
log_level debug
|
21
|
-
|
22
|
-
region ap-northeast-1
|
23
|
-
stream_name fluent-plugin-test
|
24
|
-
debug true
|
25
|
-
<kinesis_producer>
|
26
|
-
record_max_buffered_time 1000
|
27
|
-
</kinesis_producer>
|
28
|
-
</match>
|
data/benchmark/streams.conf
DELETED
@@ -1,24 +0,0 @@
|
|
1
|
-
<source>
|
2
|
-
@type tail
|
3
|
-
path /tmp/dummy.log
|
4
|
-
format none
|
5
|
-
tag dummy
|
6
|
-
</source>
|
7
|
-
|
8
|
-
<source>
|
9
|
-
@type forward
|
10
|
-
</source>
|
11
|
-
|
12
|
-
<match dummy>
|
13
|
-
@type kinesis_streams
|
14
|
-
flush_interval 1
|
15
|
-
buffer_chunk_limit 1m
|
16
|
-
try_flush_interval 0.1
|
17
|
-
queued_chunk_flush_interval 0.01
|
18
|
-
num_threads 15
|
19
|
-
detach_process 5
|
20
|
-
log_level debug
|
21
|
-
|
22
|
-
region ap-northeast-1
|
23
|
-
stream_name fluent-plugin-test
|
24
|
-
</match>
|