fluent-plugin-kinesis 1.3.0 → 2.0.0

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -1,44 +1,57 @@
 #
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
 #
-#  http://aws.amazon.com/asl/
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
 
-require 'fluent/plugin/kinesis_helper'
+require 'fluent/plugin/kinesis'
 
 module Fluent
-  class KinesisFirehoseOutput < BufferedOutput
-    include KinesisHelper
+  class KinesisFirehoseOutput < KinesisOutput
     Fluent::Plugin.register_output('kinesis_firehose', self)
-    config_param_for_firehose
 
-    def write(chunk)
-      records = convert_to_records(chunk)
-      split_to_batches(records).each do |batch|
-        batch_request_with_retry(batch)
+    RequestType = :firehose
+    BatchRequestLimitCount = 500
+    BatchRequestLimitSize  = 4 * 1024 * 1024
+    include KinesisHelper::API::BatchRequest
+
+    config_param :delivery_stream_name, :string
+    config_param :append_new_line,      :bool, default: true
+
+    def configure(conf)
+      super
+      if @append_new_line
+        org_data_formatter = @data_formatter
+        @data_formatter = ->(tag, time, record) {
+          org_data_formatter.call(tag, time, record) + "\n"
+        }
       end
-      log.debug("Written #{records.size} records")
     end
 
-    private
-
-    def convert_format(tag, time, record)
-      { data: data_format(tag, time, record) }
+    def format(tag, time, record)
+      format_for_api do
+        [@data_formatter.call(tag, time, record)]
+      end
     end
 
-    def batch_request(batch)
-      client.put_record_batch(
-        delivery_stream_name: @delivery_stream_name,
-        records: batch
-      )
+    def write(chunk)
+      write_records_batch(chunk) do |batch|
+        records = batch.map{|(data)|
+          { data: data }
+        }
+        client.put_record_batch(
+          delivery_stream_name: @delivery_stream_name,
+          records: records,
+        )
+      end
     end
   end
 end

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -1,48 +1,69 @@
 #
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
 #
-#  http://aws.amazon.com/asl/
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
 
-require 'fluent/plugin/kinesis_helper'
+require 'fluent/plugin/kinesis'
 
 module Fluent
-  class KinesisStreamsOutput < BufferedOutput
-    include KinesisHelper
+  class KinesisStreamsOutput < KinesisOutput
     Fluent::Plugin.register_output('kinesis_streams', self)
-    config_param_for_streams
+
+    RequestType = :streams
+    BatchRequestLimitCount = 500
+    BatchRequestLimitSize  = 5 * 1024 * 1024
+    include KinesisHelper::API::BatchRequest
+
+    config_param :stream_name,   :string
+    config_param :partition_key, :string,  default: nil
+
+    def configure(conf)
+      super
+      @key_formatter = key_formatter_create
+    end
+
+    def format(tag, time, record)
+      format_for_api do
+        data = @data_formatter.call(tag, time, record)
+        key = @key_formatter.call(record)
+        [data, key]
+      end
+    end
 
     def write(chunk)
-      records = convert_to_records(chunk)
-      split_to_batches(records).each do |batch|
-        next unless batch.size > 0
-        batch_request_with_retry(batch)
+      write_records_batch(chunk) do |batch|
+        records = batch.map{|(data, partition_key)|
+          { data: data, partition_key: partition_key }
+        }
+        client.put_records(
+          stream_name: @stream_name,
+          records: records,
+        )
       end
-      log.debug("Written #{records.size} records")
     end
 
     private
 
-    def convert_format(tag, time, record)
-      {
-        data: data_format(tag, time, record),
-        partition_key: key(record),
-      }
-    end
-
-    def batch_request(batch)
-      client.put_records(
-        stream_name: @stream_name,
-        records: batch,
-      )
+    def key_formatter_create
+      if @partition_key.nil?
+        ->(record) { SecureRandom.hex(16) }
+      else
+        ->(record) {
+          if !record.key?(@partition_key)
+            raise KeyNotFoundError.new(@partition_key, record)
+          end
+          record[@partition_key]
+        }
+      end
     end
   end
 end

data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb

@@ -0,0 +1,76 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis'
+require 'fluent/plugin/kinesis_helper/aggregator'
+
+module Fluent
+  class KinesisStreamsAggregatedOutput < KinesisOutput
+    Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
+    include KinesisHelper::Aggregator::Mixin
+
+    RequestType = :streams_aggregated
+    BatchRequestLimitCount = 100_000
+    BatchRequestLimitSize  = 1024 * 1024
+    include KinesisHelper::API::BatchRequest
+
+    config_param :stream_name, :string
+    config_param :fixed_partition_key, :string, default: nil
+
+    def configure(conf)
+      super
+      @partition_key_generator = create_partition_key_generator
+      @batch_request_max_size -= offset
+      @max_record_size -= offset
+    end
+
+    def format(tag, time, record)
+      format_for_api do
+        [@data_formatter.call(tag, time, record)]
+      end
+    end
+
+    def write(chunk)
+      write_records_batch(chunk) do |batch|
+        key = @partition_key_generator.call
+        records = batch.map{|(data)|data}
+        client.put_records(
+          stream_name: @stream_name,
+          records: [{
+            partition_key: key,
+            data: aggregator.aggregate(records, key),
+          }],
+        )
+      end
+    end
+
+    def offset
+      @offset ||= AggregateOffset + @partition_key_generator.call.size*2
+    end
+
+    private
+
+    def size_of_values(record)
+      super(record) + RecordOffset
+    end
+
+    def create_partition_key_generator
+      if @fixed_partition_key.nil?
+        ->() { SecureRandom.hex(16) }
+      else
+        ->() { @fixed_partition_key }
+      end
+    end
+  end
+end

data/lib/fluent_plugin_kinesis/version.rb

@@ -1,17 +1,17 @@
 #
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
 #
-#  http://aws.amazon.com/asl/
+#     http://www.apache.org/licenses/LICENSE-2.0
 #
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
 
 module FluentPluginKinesis
-  VERSION = '1.3.0'
+  VERSION = '2.0.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kinesis
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-22 00:00:00.000000000 Z
+date: 2017-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.12.35
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,40 +26,26 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.58
+        version: 0.12.35
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
-  name: protobuf
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.5.5
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 3.5.5
-- !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-kinesis
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '1'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
+  name: aws-sdk-firehose
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -73,19 +59,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: os
+  name: google-protobuf
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.6
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.6
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -198,20 +184,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.0.2
-- !ruby/object:Gem::Dependency
-  name: rubyzip
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
@@ -268,46 +240,22 @@ files:
 - LICENSE.txt
 - Makefile
 - NOTICE.txt
-- README-v0.4.md
 - README.md
 - Rakefile
-- amazon-kinesis-producer-native-binaries/linux/kinesis_producer
-- amazon-kinesis-producer-native-binaries/osx/kinesis_producer
-- amazon-kinesis-producer-native-binaries/windows/kinesis_producer.exe
-- benchmark/dummy.conf
 - benchmark/task.rake
 - fluent-plugin-kinesis.gemspec
-- gemfiles/Gemfile.aws-sdk-2.4
-- gemfiles/Gemfile.fluentd-0.10.58
 - gemfiles/Gemfile.fluentd-0.12
-- gemfiles/Gemfile.fluentd-0.14.11
-- gemfiles/Gemfile.ruby-2.0
-- gemfiles/Gemfile.ruby-2.1
-- lib/fluent/plugin/kinesis_helper.rb
+- lib/fluent/plugin/kinesis.rb
+- lib/fluent/plugin/kinesis_helper/aggregator.rb
 - lib/fluent/plugin/kinesis_helper/api.rb
-- lib/fluent/plugin/kinesis_helper/class_methods.rb
 - lib/fluent/plugin/kinesis_helper/client.rb
-- lib/fluent/plugin/kinesis_helper/credentials.rb
-- lib/fluent/plugin/kinesis_helper/error.rb
-- lib/fluent/plugin/kinesis_helper/format.rb
-- lib/fluent/plugin/kinesis_helper/initialize.rb
-- lib/fluent/plugin/kinesis_helper/kpl.rb
-- lib/fluent/plugin/out_kinesis.rb
 - lib/fluent/plugin/out_kinesis_firehose.rb
-- lib/fluent/plugin/out_kinesis_producer.rb
 - lib/fluent/plugin/out_kinesis_streams.rb
-- lib/fluent/plugin/patched_detach_process_impl.rb
+- lib/fluent/plugin/out_kinesis_streams_aggregated.rb
 - lib/fluent_plugin_kinesis/version.rb
-- lib/kinesis_producer.rb
-- lib/kinesis_producer/binary.rb
-- lib/kinesis_producer/daemon.rb
-- lib/kinesis_producer/library.rb
-- lib/kinesis_producer/protobuf/config.pb.rb
-- lib/kinesis_producer/protobuf/messages.pb.rb
-- lib/kinesis_producer/tasks/binary.rake
 homepage: https://github.com/awslabs/aws-fluent-plugin-kinesis
 licenses:
-- Amazon Software License
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -317,7 +265,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.0.0
+      version: '2.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="