fluent-plugin-kinesis-modified 3.1.3

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -0,0 +1,59 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis'
+
+module Fluent
+  module Plugin
+    class KinesisFirehoseOutput < KinesisOutput
+      Fluent::Plugin.register_output('kinesis_firehose', self)
+
+      RequestType = :firehose
+      BatchRequestLimitCount = 500
+      BatchRequestLimitSize  = 4 * 1024 * 1024
+      include KinesisHelper::API::BatchRequest
+
+      config_param :delivery_stream_name, :string
+      config_param :append_new_line,      :bool, default: true
+
+      def configure(conf)
+        super
+        if @append_new_line
+          org_data_formatter = @data_formatter
+          @data_formatter = ->(tag, time, record) {
+            org_data_formatter.call(tag, time, record).chomp + "\n"
+          }
+        end
+      end
+
+      def format(tag, time, record)
+        format_for_api do
+          [@data_formatter.call(tag, time, record)]
+        end
+      end
+
+      def write(chunk)
+        write_records_batch(chunk) do |batch|
+          records = batch.map{|(data)|
+            { data: data }
+          }
+          client.put_record_batch(
+            delivery_stream_name: @delivery_stream_name,
+            records: records,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -0,0 +1,71 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis'
+
+module Fluent
+  module Plugin
+    class KinesisStreamsOutput < KinesisOutput
+      Fluent::Plugin.register_output('kinesis_streams', self)
+
+      RequestType = :streams
+      BatchRequestLimitCount = 500
+      BatchRequestLimitSize  = 5 * 1024 * 1024
+      include KinesisHelper::API::BatchRequest
+
+      config_param :stream_name,   :string
+      config_param :partition_key, :string,  default: nil
+
+      def configure(conf)
+        super
+        @key_formatter = key_formatter_create
+      end
+
+      def format(tag, time, record)
+        format_for_api do
+          data = @data_formatter.call(tag, time, record)
+          key = @key_formatter.call(record)
+          [data, key]
+        end
+      end
+
+      def write(chunk)
+        write_records_batch(chunk) do |batch|
+          records = batch.map{|(data, partition_key)|
+            { data: data, partition_key: partition_key }
+          }
+          client.put_records(
+            stream_name: @stream_name,
+            records: records,
+          )
+        end
+      end
+
+      private
+
+      def key_formatter_create
+        if @partition_key.nil?
+          ->(record) { SecureRandom.hex(16) }
+        else
+          ->(record) {
+            if !record.key?(@partition_key)
+              raise KeyNotFoundError.new(@partition_key, record)
+            end
+            record[@partition_key]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_kinesis_streams_aggregated.rb

@@ -0,0 +1,78 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis'
+require 'fluent/plugin/kinesis_helper/aggregator'
+
+module Fluent
+  module Plugin
+    class KinesisStreamsAggregatedOutput < KinesisOutput
+      Fluent::Plugin.register_output('kinesis_streams_aggregated', self)
+      include KinesisHelper::Aggregator::Mixin
+
+      RequestType = :streams_aggregated
+      BatchRequestLimitCount = 100_000
+      BatchRequestLimitSize  = 1024 * 1024
+      include KinesisHelper::API::BatchRequest
+
+      config_param :stream_name, :string
+      config_param :fixed_partition_key, :string, default: nil
+
+      def configure(conf)
+        super
+        @partition_key_generator = create_partition_key_generator
+        @size_kb_per_record -= offset
+        @max_record_size -= offset
+      end
+
+      def format(tag, time, record)
+        format_for_api do
+          [@data_formatter.call(tag, time, record)]
+        end
+      end
+
+      def write(chunk)
+        write_records_batch(chunk) do |batch|
+          key = @partition_key_generator.call
+          records = batch.map{|(data)|data}
+          client.put_records(
+            stream_name: @stream_name,
+            records: [{
+              partition_key: key,
+              data: aggregator.aggregate(records, key),
+            }],
+          )
+        end
+      end
+
+      def offset
+        @offset ||= AggregateOffset + @partition_key_generator.call.size*2
+      end
+
+      private
+
+      def size_of_values(record)
+        super(record) + RecordOffset
+      end
+
+      def create_partition_key_generator
+        if @fixed_partition_key.nil?
+          ->() { SecureRandom.hex(16) }
+        else
+          ->() { @fixed_partition_key }
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_kinesis_streams_aggregated_modified.rb

@@ -0,0 +1,83 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis'
+require 'fluent/plugin/kinesis_helper/aggregator'
+
+module Fluent
+  module Plugin
+    class KinesisStreamsAggregatedOutput < KinesisOutput
+      Fluent::Plugin.register_output('kinesis_streams_aggregated_modified', self)
+      include KinesisHelper::Aggregator::Mixin
+
+      RequestType = :streams_aggregated
+      BatchRequestLimitCount = 100_000
+      BatchRequestLimitSize  = 1024 * 1024
+      include KinesisHelper::API::BatchRequest
+
+      config_param :stream_name, :string
+      config_param :fixed_partition_key, :string, default: nil
+
+      def configure(conf)
+        super
+        @partition_key_generator = create_partition_key_generator
+        @size_kb_per_record -= offset
+        @max_record_size -= offset
+      end
+
+      def format(tag, time, record)
+        format_for_api do
+          [@data_formatter.call(tag, time, record)]
+        end
+      end
+
+      def write(chunk)
+        write_records_batch2(chunk) do |batches|
+          records_to_send = []
+          batches.each do |batch|
+            key = @partition_key_generator.call
+            records = batch.map{|(data)|data}
+            regularized_agg_record = {
+              partition_key: key,
+              data: aggregator.aggregate(records, key)
+            }
+            records_to_send << regularized_agg_record
+          end
+          client.put_records(
+            stream_name: @stream_name,
+            records: records_to_send
+          )
+        end
+      end
+
+      def offset
+        @offset ||= AggregateOffset + @partition_key_generator.call.size*2
+      end
+
+      private
+
+      def size_of_values(record)
+        super(record) + RecordOffset
+      end
+
+      def create_partition_key_generator
+        if @fixed_partition_key.nil?
+          ->() { SecureRandom.hex(16) }
+        else
+          ->() { @fixed_partition_key }
+        end
+      end
+    end
+  end
+end

data/lib/fluent_plugin_kinesis/version.rb

@@ -0,0 +1,17 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module FluentPluginKinesis
+  VERSION = '3.1.3'
+end

metadata

@@ -0,0 +1,313 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-kinesis-modified
+version: !ruby/object:Gem::Version
+  version: 3.1.3
+platform: ruby
+authors:
+- iHandy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.10
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-kinesis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-firehose
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: google-protobuf
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+- !ruby/object:Gem::Dependency
+  name: pry-stack_explorer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.9.2
+- !ruby/object:Gem::Dependency
+  name: net-empty_port
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.24.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.24.2
+- !ruby/object:Gem::Dependency
+  name: fakefs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/PULL_REQUEST_TEMPLATE.md"
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- CONTRIBUTING.md
+- CONTRIBUTORS.txt
+- Gemfile
+- LICENSE.txt
+- Makefile
+- NOTICE.txt
+- README.md
+- Rakefile
+- benchmark/task.rake
+- fluent-plugin-kinesis.gemspec
+- gemfiles/Gemfile.fluentd-0.14.10
+- gemfiles/Gemfile.td-agent-3.1.1
+- gemfiles/Gemfile.td-agent-3.2.0
+- gemfiles/Gemfile.td-agent-3.2.1
+- gemfiles/Gemfile.td-agent-3.3.0
+- lib/fluent/plugin/kinesis.rb
+- lib/fluent/plugin/kinesis_helper/aggregator.rb
+- lib/fluent/plugin/kinesis_helper/api.rb
+- lib/fluent/plugin/kinesis_helper/client.rb
+- lib/fluent/plugin/out_kinesis_firehose.rb
+- lib/fluent/plugin/out_kinesis_streams.rb
+- lib/fluent/plugin/out_kinesis_streams_aggregated.rb
+- lib/fluent/plugin/out_kinesis_streams_aggregated_modified.rb
+- lib/fluent_plugin_kinesis/version.rb
+homepage: https://github.com/BlackNight95/aws-fluent-plugin-kinesis/tree/feature_modified_aggregation
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.3
+signing_key: 
+specification_version: 4
+summary: Fluentd output plugin that sends events to Amazon Kinesis. Forked from fluent-plugin-kinesis
+  version 3.1.0
+test_files: []