fluent-plugin-kinesis 0.4.1 → 1.0.0

data/Rakefile

@@ -1,23 +1,29 @@
-# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
 #
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
 
 require "bundler/gem_tasks"
 
 require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-    test.libs << 'lib' << 'test'
-    test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
+
+task default: [:test]
+Rake::TestTask.new do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/**/test_*.rb']
+  test.options = '-v'
 end
 
-task :default => :test
+load 'kinesis_producer/tasks/binary.rake'
+
+Rake::Task[:build].enhance [:binaries]
+Rake::Task[:test].enhance [:binaries]

data/benchmark/dummer.conf

@@ -0,0 +1,13 @@
+configure 'sample' do
+  output "/tmp/dummy.log"
+  rate 5000
+  delimiter "\t"
+  labeled true
+  field :id, type: :integer, countup: true, format: "%04d"
+  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
+  field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
+  field :method, type: :string, any: %w[GET POST PUT]
+  field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
+  field :reqtime, type: :float, range: 0.1..5.0
+  field :foobar, type: :string, length: 8
+end

data/benchmark/firehose.conf

@@ -0,0 +1,24 @@
+<source>
+  @type tail
+  path /tmp/dummy.log
+  format none
+  tag dummy
+</source>
+
+<source>
+  @type forward
+</source>
+
+<match dummy>
+  @type kinesis_firehose
+  flush_interval 1
+  buffer_chunk_limit 1m
+  try_flush_interval 0.1
+  queued_chunk_flush_interval 0.01
+  num_threads 15
+  detach_process 5
+  log_level debug
+
+  region us-west-2
+  delivery_stream_name fluent-plugin-test
+</match>

data/benchmark/producer.conf

@@ -0,0 +1,28 @@
+<source>
+  @type tail
+  path /tmp/dummy.log
+  format none
+  tag dummy
+</source>
+
+<source>
+  @type forward
+</source>
+
+<match dummy>
+  @type kinesis_producer
+  flush_interval 1
+  buffer_chunk_limit 1m
+  try_flush_interval 0.1
+  queued_chunk_flush_interval 0.01
+  num_threads 15
+  detach_process 5
+  log_level debug
+
+  region ap-northeast-1
+  stream_name fluent-plugin-test
+  debug true
+  <kinesis_producer>
+    record_max_buffered_time 1000
+  </kinesis_producer>
+</match>

data/benchmark/streams.conf

@@ -0,0 +1,24 @@
+<source>
+  @type tail
+  path /tmp/dummy.log
+  format none
+  tag dummy
+</source>
+
+<source>
+  @type forward
+</source>
+
+<match dummy>
+  @type kinesis_streams
+  flush_interval 1
+  buffer_chunk_limit 1m
+  try_flush_interval 0.1
+  queued_chunk_flush_interval 0.01
+  num_threads 15
+  detach_process 5
+  log_level debug
+
+  region ap-northeast-1
+  stream_name fluent-plugin-test
+</match>

data/fluent-plugin-kinesis.gemspec

@@ -1,21 +1,22 @@
-# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# coding: utf-8
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+#  http://aws.amazon.com/asl/
 #
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
 
-# coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-
-require "fluent/plugin/version"
+require 'fluent_plugin_kinesis/version'
+require 'kinesis_producer/binary'
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-kinesis"
@@ -23,19 +24,29 @@ Gem::Specification.new do |spec|
   spec.author        = 'Amazon Web Services'
   spec.summary       = %q{Fluentd output plugin that sends events to Amazon Kinesis.}
   spec.homepage      = "https://github.com/awslabs/aws-fluent-plugin-kinesis"
-  spec.license       = "Apache License, Version 2.0"
+  spec.license       = "Amazon Software License"
 
-  spec.files         = `git ls-files`.split($/)
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files         += KinesisProducer::Binary::Files.values
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
-  spec.required_ruby_version = '>= 1.9.3'
+  spec.required_ruby_version = '>= 2.0.0'
 
-  spec.add_development_dependency "bundler", "~> 1.3"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "test-unit-rr", "~> 1.0"
+  spec.add_dependency "fluentd", ">= 0.10.58", "< 2"
+  spec.add_dependency "protobuf", ">= 3.5.5"
+  spec.add_dependency "aws-sdk", "~> 2"
+  spec.add_dependency "concurrent-ruby", "~> 1"
+  spec.add_dependency "os", ">= 0.9.6"
 
-  spec.add_dependency "fluentd", ">= 0.10.53", "< 2"
-  spec.add_dependency "aws-sdk-core", ">= 2.0.12", "< 3.0"
-  spec.add_dependency "msgpack", ">= 0.5.8"
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", ">= 3.0.8"
+  spec.add_development_dependency "test-unit-rr", ">= 1.0.3"
+  spec.add_development_dependency "pry", ">= 0.10.1"
+  spec.add_development_dependency "pry-byebug", ">= 3.3.0"
+  spec.add_development_dependency "pry-stack_explorer", ">= 0.4.9.2"
+  spec.add_development_dependency "net-empty_port", ">= 0.0.2"
+  spec.add_development_dependency "dummer", ">= 0.4.0"
+  spec.add_development_dependency "rubyzip", ">= 1.0.0"
+  spec.add_development_dependency "mocha", ">= 1.1.0"
 end

data/gemfiles/Gemfile.fluentd-0.10.58

@@ -0,0 +1,20 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+gem "fluentd", "0.10.58"

data/lib/fluent/plugin/kinesis_helper.rb

@@ -0,0 +1,30 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis_helper/class_methods'
+require 'fluent/plugin/kinesis_helper/initialize'
+require 'fluent/plugin/kinesis_helper/error'
+
+module Fluent
+  module KinesisHelper
+    include Fluent::SetTimeKeyMixin
+    include Fluent::SetTagKeyMixin
+    include Fluent::DetachMultiProcessMixin
+
+    def self.included(klass)
+      klass.extend ClassMethods
+    end
+    include Initialize
+  end
+end

data/lib/fluent/plugin/kinesis_helper/api.rb

@@ -0,0 +1,164 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'fluent_plugin_kinesis/version'
+
+module Fluent
+  module KinesisHelper
+    module API
+      def configure(conf)
+        super
+        if @batch_request_max_count > self.class::BatchRequestLimitCount
+          raise ConfigError, "batch_request_max_count can't be grater than #{self.class::BatchRequestLimitCount}."
+        end
+        if @batch_request_max_size > self.class::BatchRequestLimitSize
+          raise ConfigError, "batch_request_max_size can't be grater than #{self.class::BatchRequestLimitSize}."
+        end
+        @region = client.config.region if @region.nil?
+      end
+
+      def start
+        detach_multi_process do
+          super
+        end
+      end
+
+      private
+
+      def client_options
+        options = {
+          user_agent_suffix: "fluent-plugin-kinesis/#{request_type}/#{FluentPluginKinesis::VERSION}",
+          credentials: credentials,
+        }
+        options.update(region:          @region)          unless @region.nil?
+        options.update(http_proxy:      @http_proxy)      unless @http_proxy.nil?
+        options.update(endpoint:        @endpoint)        unless @endpoint.nil?
+        options.update(ssl_verify_peer: @ssl_verify_peer) unless @ssl_verify_peer.nil?
+        if @debug
+          options.update(logger: Logger.new(log.out))
+          options.update(log_level: :debug)
+        end
+        options
+      end
+
+      def split_to_batches(records)
+        batch_by_limit(records, @batch_request_max_count, @batch_request_max_size)
+      end
+
+      def batch_by_limit(records, max_count, max_size)
+        result, buf, size = records.inject([[],[],0]){|(result, buf, size), record|
+          if buf.size >= max_count or size >= max_size
+            result << buf
+            buf = []
+            size = 0
+          end
+          buf << record
+          size += size_of_values(record)
+          [result, buf, size]
+        }
+        result << buf
+      end
+
+      def size_of_values(record)
+        record.values_at(:data, :partition_key).compact.map(&:size).inject(:+) || 0
+      end
+
+      def batch_request_with_retry(batch, retry_count=0, backoff: nil)
+        backoff ||= Backoff.new
+        res = batch_request(batch)
+        if failed_count(res) > 0
+          failed_records = collect_failed_records(batch, res)
+          if retry_count < @retries_on_batch_request
+            backoff.reset if @reset_backoff_if_success and any_records_shipped?(res)
+            sleep(backoff.next)
+            log.warn(truncate 'Retrying to request batch. Retry count: %d, Retry records: %d' % [retry_count, failed_records.size])
+            retry_batch = failed_records.map{|r| r[:original] }
+            batch_request_with_retry(retry_batch, retry_count + 1, backoff: backoff)
+          else
+            give_up_retries(failed_records)
+          end
+        end
+      end
+
+      def any_records_shipped?(res)
+        results(res).size > failed_count(res)
+      end
+
+      def collect_failed_records(records, res)
+        failed_records = []
+        results(res).each_with_index do |record, index|
+          next unless record[:error_code]
+          failed_records.push(
+            original:      records[index],
+            error_code:    record[:error_code],
+            error_message: record[:error_message]
+          )
+        end
+        failed_records
+      end
+
+      def failed_count(res)
+        failed_field = case request_type
+                       when :streams;  :failed_record_count
+                       when :firehose; :failed_put_count
+                       end
+        res[failed_field]
+      end
+
+      def results(res)
+        result_field = case request_type
+                       when :streams;  :records
+                       when :firehose; :request_responses
+                       end
+        res[result_field]
+      end
+
+      def give_up_retries(failed_records)
+        failed_records.each {|record|
+          log.error(truncate 'Could not put record, Error: %s/%s, Record: %s' % [
+            record[:error_code],
+            record[:error_message],
+            record[:original]
+          ])
+        }
+      end
+
+      class Backoff
+        def initialize
+          @count = 0
+        end
+
+        def next
+          value = calc(@count)
+          @count += 1
+          value
+        end
+
+        def reset
+          @count = 0
+        end
+
+        private
+
+        def calc(count)
+          (2 ** count) * scaling_factor
+        end
+
+        def scaling_factor
+          0.3 + (0.5-rand) * 0.1
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kinesis_helper/class_methods.rb

@@ -0,0 +1,120 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Fluent
+  module KinesisHelper
+    module ClassMethods
+      def config_param_for_streams
+        const_set(:RequestType, :streams)
+        const_set(:BatchRequestLimitCount, 500)
+        const_set(:BatchRequestLimitSize, 5 * 1024 * 1024)
+        config_param :stream_name,   :string
+        config_param :region,        :string,  default: nil
+        config_param :partition_key, :string,  default: nil
+        config_param_for_sdk
+        config_param_for_credentials
+        config_param_for_format
+        config_param_for_batch_request
+        config_param_for_debug
+      end
+
+      def config_param_for_firehose
+        const_set(:RequestType, :firehose)
+        const_set(:BatchRequestLimitCount, 500)
+        const_set(:BatchRequestLimitSize, 4 * 1024 * 1024)
+        config_param :delivery_stream_name, :string
+        config_param :region,               :string,  default: nil
+        config_param :append_new_line,      :bool,    default: true
+        config_param_for_sdk
+        config_param_for_credentials
+        config_param_for_format
+        config_param_for_batch_request
+        config_param_for_debug
+      end
+
+      def config_param_for_producer
+        const_set(:RequestType, :producer)
+        config_param :stream_name,   :string
+        config_param :region,        :string, default: nil
+        config_param :partition_key, :string, default: nil
+        config_param_for_credentials
+        config_param_for_format
+        config_param_for_debug
+
+        config_section :kinesis_producer, multi: false do
+          require 'kinesis_producer'
+          type_map = {
+            Protobuf::Field::BoolField   => :bool,
+            Protobuf::Field::Uint64Field => :integer,
+            Protobuf::Field::StringField => :string,
+          }
+          KinesisProducer::ConfigurationFields.each do |field|
+            next if field.name == 'region'
+            type = type_map[field.type_class]
+            config_param field.name, type, default: field.default_value
+          end
+          config_param :credentials_refresh_delay, :integer, default: 5000
+        end
+      end
+
+      def config_param_for_sdk
+        config_param :http_proxy,      :string, default: nil
+        config_param :endpoint,        :string, default: nil
+        config_param :ssl_verify_peer, :bool,   default: true
+      end
+
+      def config_param_for_credentials
+        config_param :aws_key_id,  :string, default: nil, secret: true
+        config_param :aws_sec_key, :string, default: nil, secret: true
+        config_section :shared_credentials, multi: false do
+          config_param :profile_name, :string, default: nil
+          config_param :path,         :string, default: nil
+        end
+        config_section :assume_role_credentials, multi: false do
+          config_param :role_arn,    :string,               secret: true
+          config_param :external_id, :string, default: nil, secret: true
+        end
+      end
+
+      def config_param_for_format
+        config_param :formatter,             :string,  default: 'json'
+        config_param :data_key,              :string,  default: nil
+        config_param :log_truncate_max_size, :integer, default: 0
+      end
+
+      def config_param_for_batch_request
+        config_param :retries_on_batch_request, :integer, default: 3
+        config_param :reset_backoff_if_success, :bool,    default: true
+        config_param :batch_request_max_count,  :integer, default: const_get(:BatchRequestLimitCount)
+        config_param :batch_request_max_size,   :integer, default: const_get(:BatchRequestLimitSize)
+      end
+
+      def config_param_for_debug
+        config_param :debug, :bool, default: false
+      end
+
+      def request_type
+        const_get(:RequestType)
+      end
+
+      def api?
+        [:streams, :firehose].include?(request_type)
+      end
+
+      def kpl?
+        [:producer].include?(request_type)
+      end
+    end
+  end
+end