fluent-plugin-kinesis 1.0.1 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 46273afe8f8e8fa0f70931b1f2bbf1893df30256
4
- data.tar.gz: 55beac91c0b19714355e901ed256dbf16210a930
3
+ metadata.gz: 9f1802cbf044412c40fee66e52412c78a8f00c58
4
+ data.tar.gz: 3bb9b8bb9b5a079cbf3fa61f90b6b3e16cd81e77
5
5
  SHA512:
6
- metadata.gz: 57f4784a5c00763fb358de48d74d161815a90a466f4e8afe895b91485bae1db6a6eba84b994def1ddf6a90f9a60742ac63b4abd91d1fa9eda16d360bb5c0db02
7
- data.tar.gz: a5b6e7769c7ddce92bd9ee27e603c7bfbe41ac3a53d032b12b5edf30522a1eb98f70e54a75dee6ec33dd8a197f8af8effd3d373a0c98c9bb709e67ecd0d947a0
6
+ metadata.gz: acf1eefc4d2d071cb81cf22dd1969f5fe60276bfecd3616ac193b19106073c3d585bef51d3a5750f7a803a914e5a4cab75ea32d37153380d19b910c40891f35e
7
+ data.tar.gz: ceed8d7790f50d07256fc7a927ee053d3808d92ee6d444d87c96c30813283a8758b21d13364b476747efd777a1882dc729a0a7d3308777258a3214eb59f6c939
data/.travis.yml CHANGED
@@ -8,6 +8,7 @@ rvm:
8
8
  gemfile:
9
9
  - Gemfile
10
10
  - gemfiles/Gemfile.fluentd-0.10.58
11
+ - gemfiles/Gemfile.fluentd-0.12
11
12
 
12
13
  os:
13
14
  - linux
@@ -22,5 +23,15 @@ sudo: false
22
23
 
23
24
  matrix:
24
25
  exclude:
26
+ - rvm: 2.0.0
27
+ os: osx
28
+ - rvm: 2.0.0
29
+ gemfile: Gemfile
30
+ - rvm: 2.1
31
+ os: osx
32
+ gemfile: gemfiles/Gemfile.fluentd-0.10.58
33
+ - rvm: 2.1
34
+ os: osx
35
+ gemfile: gemfiles/Gemfile.fluentd-0.12
25
36
  - rvm: 2.2
26
37
  os: osx
data/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # CHANGELOG
2
2
 
3
+ ## 1.1.0
4
+
5
+ - Feature - Derive stream name from fluentd tag for KPL [#67](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/67)
6
+ - Enhancement - Make http_proxy parameter secret [#64](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/64)
7
+ - Bug fix - Plugin incompatible with new fluentd release 0.14 [#70](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/70)
8
+ - Misc - Fix legacy test and reduce travis tests [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74)
9
+ - Misc - Some test, benchmark improvement [#74](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/74), [#75](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/75), [#76](https://github.com/awslabs/aws-fluent-plugin-kinesis/pull/76)
10
+
3
11
  ## 1.0.1
4
12
 
5
13
  - Bug fix - Instance profile credentials expiring [#58](https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/58)
data/Makefile CHANGED
@@ -12,26 +12,25 @@
12
12
  # express or implied. See the License for the specific language governing
13
13
  # permissions and limitations under the License.
14
14
 
15
- .PHONY: install streams firehose producer dummer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
15
+ .PHONY: test install benchmark benchmark-streams benchmark-producer hello $(wildcard test/test_*.rb) $(wildcard test/**/test_*.rb)
16
16
 
17
17
  all:
18
18
  bundle install
19
- bundle exec rake
19
+ bundle exec rake binaries
20
+
21
+ test:
22
+ bundle exec rake test
20
23
 
21
24
  install:
22
25
  bundle exec rake install:local
23
26
 
24
- streams:
25
- bundle exec fluentd -c benchmark/streams.conf -vv
26
-
27
- firehose:
28
- bundle exec fluentd -c benchmark/firehose.conf -vv
27
+ benchmark: benchmark-streams benchmark-producer
29
28
 
30
- producer:
31
- bundle exec fluentd -c benchmark/producer.conf -vv
29
+ benchmark-streams:
30
+ bundle exec rake benchmark TYPE=streams
32
31
 
33
- dummer:
34
- bundle exec dummer -c benchmark/dummer.conf
32
+ benchmark-producer:
33
+ bundle exec rake benchmark TYPE=producer
35
34
 
36
35
  hello:
37
36
  echo Hello World | bundle exec fluent-cat --none dummy
data/README.md CHANGED
@@ -527,15 +527,9 @@ To launch `fluentd` process with this plugin for development, follow the steps b
527
527
  git clone https://github.com/awslabs/aws-fluent-plugin-kinesis.git
528
528
  cd aws-fluent-plugin-kinesis
529
529
  make # will install gems and download KPL jar file and extract binaries
530
- make [stream/firehose/producer]
530
+ bundle exec fluentd -c /path/to/fluent.conf
531
531
 
532
- Then, in another terminal, run the command below. It will emit one record.
533
-
534
- make hello
535
-
536
- Also, you can test streaming log data by `dummer`:
537
-
538
- make dummer # keep writing to /tmp/dummy.log
532
+ If you want to run benchmark, use `make benchmark`.
539
533
 
540
534
  ## Contributing
541
535
 
data/Rakefile CHANGED
@@ -24,6 +24,7 @@ Rake::TestTask.new do |test|
24
24
  end
25
25
 
26
26
  load 'kinesis_producer/tasks/binary.rake'
27
-
28
27
  Rake::Task[:build].enhance [:binaries]
29
28
  Rake::Task[:test].enhance [:binaries]
29
+
30
+ load 'benchmark/task.rake'
File without changes
@@ -0,0 +1,72 @@
1
+ #
2
+ # Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Amazon Software License (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/asl/
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+
15
+ require_relative '../test/dummy_server'
16
+
17
+ task :benchmark do
18
+ server = DummyServer.start
19
+ conf = profile_conf(ENV["TYPE"] || 'streams', ENV["RATE"] || 1000, server)
20
+ pid = spawn("bundle exec fluentd -i '#{conf}' -c benchmark/dummy.conf")
21
+ sleep 10
22
+ Process.kill("TERM", pid)
23
+ Process.wait
24
+ puts "Results: requets: #{server.requests.size}, raw_records: #{server.raw_records.size}, records: #{server.records.size}"
25
+ server.shutdown
26
+ end
27
+
28
+ def profile_conf(type, rate, server)
29
+ additional_conf = case type
30
+ when 'streams', 'firehose'
31
+ <<-EOS
32
+ endpoint https://localhost:#{server.port}
33
+ ssl_verify_peer false
34
+ EOS
35
+ when 'producer'
36
+ <<-EOS
37
+ debug true
38
+ <kinesis_producer>
39
+ custom_endpoint localhost
40
+ port #{server.port}
41
+ verify_certificate false
42
+ record_max_buffered_time 1000
43
+ log_level error
44
+ </kinesis_producer>
45
+ EOS
46
+ end
47
+
48
+ conf = <<-EOS
49
+ <source>
50
+ @type dummy
51
+ tag dummy
52
+ rate #{rate}
53
+ </source>
54
+
55
+ <match dummy>
56
+ @type kinesis_#{type}
57
+ flush_interval 1
58
+ buffer_chunk_limit 1m
59
+ try_flush_interval 0.1
60
+ queued_chunk_flush_interval 0.01
61
+ @log_level debug
62
+
63
+ num_threads 100
64
+
65
+ region ap-northeast-1
66
+ stream_name fluent-plugin-test
67
+
68
+ #{additional_conf}
69
+ </match>
70
+ EOS
71
+ conf
72
+ end
@@ -46,7 +46,6 @@ Gem::Specification.new do |spec|
46
46
  spec.add_development_dependency "pry-byebug", ">= 3.3.0"
47
47
  spec.add_development_dependency "pry-stack_explorer", ">= 0.4.9.2"
48
48
  spec.add_development_dependency "net-empty_port", ">= 0.0.2"
49
- spec.add_development_dependency "dummer", ">= 0.4.0"
50
49
  spec.add_development_dependency "rubyzip", ">= 1.0.0"
51
50
  spec.add_development_dependency "mocha", ">= 1.1.0"
52
51
  spec.add_development_dependency "webmock", ">= 1.24.2"
@@ -0,0 +1,20 @@
1
+ #
2
+ # Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Amazon Software License (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/asl/
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+
15
+ source 'https://rubygems.org'
16
+
17
+ # Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
18
+ gemspec path: ".."
19
+
20
+ gem "fluentd", "~> 0.12.26"
@@ -20,6 +20,7 @@ module Fluent
20
20
  module KinesisHelper
21
21
  include Fluent::SetTimeKeyMixin
22
22
  include Fluent::SetTagKeyMixin
23
+ require 'fluent/process'
23
24
  include Fluent::DetachMultiProcessMixin
24
25
 
25
26
  def self.included(klass)
@@ -45,9 +45,10 @@ module Fluent
45
45
 
46
46
  def config_param_for_producer
47
47
  const_set(:RequestType, :producer)
48
- config_param :stream_name, :string
49
- config_param :region, :string, default: nil
50
- config_param :partition_key, :string, default: nil
48
+ config_param :stream_name, :string, default: nil
49
+ config_param :stream_name_prefix, :string, default: nil
50
+ config_param :region, :string, default: nil
51
+ config_param :partition_key, :string, default: nil
51
52
  config_param_for_credentials
52
53
  config_param_for_format
53
54
  config_param_for_debug
@@ -69,7 +70,7 @@ module Fluent
69
70
  end
70
71
 
71
72
  def config_param_for_sdk
72
- config_param :http_proxy, :string, default: nil
73
+ config_param :http_proxy, :string, default: nil, secret: true
73
74
  config_param :endpoint, :string, default: nil
74
75
  config_param :ssl_verify_peer, :bool, default: true
75
76
  end
@@ -32,6 +32,7 @@ module Fluent
32
32
  include API
33
33
  when kpl?
34
34
  require 'fluent/plugin/kinesis_helper/kpl'
35
+ require 'fluent/version'
35
36
  if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.12.20')
36
37
  # Backport from https://github.com/fluent/fluentd/pull/757
37
38
  require 'fluent/plugin/patched_detach_process_impl'
@@ -20,6 +20,16 @@ module Fluent
20
20
  Fluent::Plugin.register_output('kinesis_producer', self)
21
21
  config_param_for_producer
22
22
 
23
+ def configure(conf)
24
+ super
25
+ unless @stream_name or @stream_name_prefix
26
+ raise Fluent::ConfigError, "'stream_name' or 'stream_name_prefix' is required"
27
+ end
28
+ if @stream_name and @stream_name_prefix
29
+ raise Fluent::ConfigError, "Only one of 'stream_name' or 'stream_name_prefix' is allowed"
30
+ end
31
+ end
32
+
23
33
  def write(chunk)
24
34
  records = convert_to_records(chunk)
25
35
  wait_futures(write_chunk_to_kpl(records))
@@ -31,7 +41,7 @@ module Fluent
31
41
  {
32
42
  data: data_format(tag, time, record),
33
43
  partition_key: key(record),
34
- stream_name: @stream_name,
44
+ stream_name: @stream_name ? @stream_name : @stream_name_prefix + tag,
35
45
  }
36
46
  end
37
47
  end
@@ -13,5 +13,5 @@
13
13
  # permissions and limitations under the License.
14
14
 
15
15
  module FluentPluginKinesis
16
- VERSION = '1.0.1'
16
+ VERSION = '1.1.0'
17
17
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-kinesis
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.1
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Amazon Web Services
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-03-29 00:00:00.000000000 Z
11
+ date: 2016-06-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -198,20 +198,6 @@ dependencies:
198
198
  - - ">="
199
199
  - !ruby/object:Gem::Version
200
200
  version: 0.0.2
201
- - !ruby/object:Gem::Dependency
202
- name: dummer
203
- requirement: !ruby/object:Gem::Requirement
204
- requirements:
205
- - - ">="
206
- - !ruby/object:Gem::Version
207
- version: 0.4.0
208
- type: :development
209
- prerelease: false
210
- version_requirements: !ruby/object:Gem::Requirement
211
- requirements:
212
- - - ">="
213
- - !ruby/object:Gem::Version
214
- version: 0.4.0
215
201
  - !ruby/object:Gem::Dependency
216
202
  name: rubyzip
217
203
  requirement: !ruby/object:Gem::Requirement
@@ -288,12 +274,11 @@ files:
288
274
  - amazon-kinesis-producer-native-binaries/linux/kinesis_producer
289
275
  - amazon-kinesis-producer-native-binaries/osx/kinesis_producer
290
276
  - amazon-kinesis-producer-native-binaries/windows/kinesis_producer.exe
291
- - benchmark/dummer.conf
292
- - benchmark/firehose.conf
293
- - benchmark/producer.conf
294
- - benchmark/streams.conf
277
+ - benchmark/dummy.conf
278
+ - benchmark/task.rake
295
279
  - fluent-plugin-kinesis.gemspec
296
280
  - gemfiles/Gemfile.fluentd-0.10.58
281
+ - gemfiles/Gemfile.fluentd-0.12
297
282
  - lib/fluent/plugin/kinesis_helper.rb
298
283
  - lib/fluent/plugin/kinesis_helper/api.rb
299
284
  - lib/fluent/plugin/kinesis_helper/class_methods.rb
@@ -1,13 +0,0 @@
1
- configure 'sample' do
2
- output "/tmp/dummy.log"
3
- rate 5000
4
- delimiter "\t"
5
- labeled true
6
- field :id, type: :integer, countup: true, format: "%04d"
7
- field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
8
- field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
9
- field :method, type: :string, any: %w[GET POST PUT]
10
- field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
11
- field :reqtime, type: :float, range: 0.1..5.0
12
- field :foobar, type: :string, length: 8
13
- end
@@ -1,24 +0,0 @@
1
- <source>
2
- @type tail
3
- path /tmp/dummy.log
4
- format none
5
- tag dummy
6
- </source>
7
-
8
- <source>
9
- @type forward
10
- </source>
11
-
12
- <match dummy>
13
- @type kinesis_firehose
14
- flush_interval 1
15
- buffer_chunk_limit 1m
16
- try_flush_interval 0.1
17
- queued_chunk_flush_interval 0.01
18
- num_threads 15
19
- detach_process 5
20
- log_level debug
21
-
22
- region us-west-2
23
- delivery_stream_name fluent-plugin-test
24
- </match>
@@ -1,28 +0,0 @@
1
- <source>
2
- @type tail
3
- path /tmp/dummy.log
4
- format none
5
- tag dummy
6
- </source>
7
-
8
- <source>
9
- @type forward
10
- </source>
11
-
12
- <match dummy>
13
- @type kinesis_producer
14
- flush_interval 1
15
- buffer_chunk_limit 1m
16
- try_flush_interval 0.1
17
- queued_chunk_flush_interval 0.01
18
- num_threads 15
19
- detach_process 5
20
- log_level debug
21
-
22
- region ap-northeast-1
23
- stream_name fluent-plugin-test
24
- debug true
25
- <kinesis_producer>
26
- record_max_buffered_time 1000
27
- </kinesis_producer>
28
- </match>
@@ -1,24 +0,0 @@
1
- <source>
2
- @type tail
3
- path /tmp/dummy.log
4
- format none
5
- tag dummy
6
- </source>
7
-
8
- <source>
9
- @type forward
10
- </source>
11
-
12
- <match dummy>
13
- @type kinesis_streams
14
- flush_interval 1
15
- buffer_chunk_limit 1m
16
- try_flush_interval 0.1
17
- queued_chunk_flush_interval 0.01
18
- num_threads 15
19
- detach_process 5
20
- log_level debug
21
-
22
- region ap-northeast-1
23
- stream_name fluent-plugin-test
24
- </match>