fluent-plugin-kinesis 2.2.0 → 3.0.0

data/gemfiles/Gemfile.td-agent-3.2.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.2.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.2.2"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.87.0"
+gem "aws-sigv4", "1.0.2"
+gem "aws-sdk-core", "3.21.2"
+gem "aws-sdk-kms", "1.5.0"
+gem "aws-sdk-sqs", "1.3.0"
+gem "aws-sdk-s3", "1.13.0"
+gem "fluent-plugin-s3", "1.1.3"

data/gemfiles/Gemfile.td-agent-3.2.1

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.2.1
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.2.6"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.2.1/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.105.0"
+gem "aws-sigv4", "1.0.3"
+gem "aws-sdk-core", "3.30.0"
+gem "aws-sdk-kms", "1.9.0"
+gem "aws-sdk-sqs", "1.7.0"
+gem "aws-sdk-s3", "1.21.0"
+gem "fluent-plugin-s3", "1.1.6"

data/gemfiles/Gemfile.td-agent-3.3.0

@@ -0,0 +1,31 @@
+#
+# Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kinesis.gemspec
+gemspec path: ".."
+
+# Specify related gems for td-agent v3.3.0
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/config/projects/td-agent3.rb#L27
+gem "fluentd", "1.3.3"
+# https://github.com/treasure-data/omnibus-td-agent/blob/v3.3.0/plugin_gems.rb#L16-L23
+gem "jmespath", "1.4.0"
+gem "aws-partitions", "1.127.0"
+gem "aws-sigv4", "1.0.3"
+gem "aws-sdk-core", "3.44.2"
+gem "aws-sdk-kms", "1.13.0"
+gem "aws-sdk-sqs", "1.10.0"
+gem "aws-sdk-s3", "1.30.0"
+gem "fluent-plugin-s3", "1.1.7"

data/lib/fluent/plugin/kinesis.rb

@@ -12,154 +12,149 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
+require 'fluent/plugin/output'
 require 'fluent/plugin/kinesis_helper/client'
 require 'fluent/plugin/kinesis_helper/api'
 require 'zlib'
 
 module Fluent
-  class KinesisOutput < BufferedOutput
-    def self.fluentd_v0_12?
-      @fluentd_v0_12 ||= Gem.loaded_specs['fluentd'].version < Gem::Version.create('0.14')
-    end
-
-    include Fluent::SetTimeKeyMixin
-    include Fluent::SetTagKeyMixin
-
-    include KinesisHelper::Client
-    include KinesisHelper::API
-
-    class SkipRecordError < ::StandardError
-      def initialize(message, record)
-        super message
-        @record_message = if record.is_a? Array
-          record.reverse.map(&:to_s).join(', ')
-        else
-          record.to_s
+  module Plugin
+    class KinesisOutput < Fluent::Plugin::Output
+      include Fluent::MessagePackFactory::Mixin
+      include KinesisHelper::Client
+      include KinesisHelper::API
+
+      class SkipRecordError < ::StandardError
+        def initialize(message, record)
+          super message
+          @record_message = if record.is_a? Array
+                              record.reverse.map(&:to_s).join(', ')
+                            else
+                              record.to_s
+                            end
         end
-      end
 
-      def to_s
-        super + ": " + @record_message
+        def to_s
+          super + ": " + @record_message
+        end
       end
-    end
-    class KeyNotFoundError < SkipRecordError
-      def initialize(key, record)
-        super "Key '#{key}' doesn't exist", record
+      class KeyNotFoundError < SkipRecordError
+        def initialize(key, record)
+          super "Key '#{key}' doesn't exist", record
+        end
       end
-    end
-    class ExceedMaxRecordSizeError < SkipRecordError
-      def initialize(size, record)
-        super "Record size limit exceeded in #{size/1024} KB", record
+      class ExceedMaxRecordSizeError < SkipRecordError
+        def initialize(size, record)
+          super "Record size limit exceeded in #{size/1024} KB", record
+        end
       end
-    end
-    class InvalidRecordError < SkipRecordError
-      def initialize(record)
-        super "Invalid type of record", record
+      class InvalidRecordError < SkipRecordError
+        def initialize(record)
+          super "Invalid type of record", record
+        end
       end
-    end
 
-    config_param :data_key,              :string,  default: nil
-    config_param :log_truncate_max_size, :integer, default: 1024
-    config_param :compression,           :string,  default: nil
-    config_section :format do
-      config_set_default :@type, 'json'
-    end
-    config_section :inject do
-      config_set_default :time_type, 'string'
-      config_set_default :time_format, '%Y-%m-%dT%H:%M:%S.%N%z'
-    end
+      config_param :data_key,              :string,  default: nil
+      config_param :log_truncate_max_size, :integer, default: 1024
+      config_param :compression,           :string,  default: nil
 
-    config_param :debug, :bool, default: false
+      desc "Formatter calls chomp and removes separator from the end of each record. This option is for compatible format with plugin v2. (default: false)"
+      # https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142
+      config_param :chomp_record,          :bool,    default: false
 
-    if fluentd_v0_12?
-      config_param :format, :string,  default: 'json'
-    else
-      helpers :formatter, :inject
-    end
+      config_section :format do
+        config_set_default :@type, 'json'
+      end
+      config_section :inject do
+        config_set_default :time_type, 'string'
+        config_set_default :time_format, '%Y-%m-%dT%H:%M:%S.%N%z'
+      end
 
-    def configure(conf)
-      super
-      @data_formatter = data_formatter_create(conf)
-    end
+      config_param :debug, :bool, default: false
 
-    def multi_workers_ready?
-      true
-    end
+      helpers :formatter, :inject
 
-    private
+      def configure(conf)
+        super
+        @data_formatter = data_formatter_create(conf)
+      end
 
-    def fluentd_v0_12?
-      self.class.fluentd_v0_12?
-    end
+      def multi_workers_ready?
+        true
+      end
+
+      private
 
-    def data_formatter_create(conf)
-      if fluentd_v0_12?
-        formatter = Fluent::Plugin.new_formatter(@format)
-        formatter.configure(conf)
-      else
+      def data_formatter_create(conf)
         formatter = formatter_create
-      end
-      compressor = compressor_create
-      if @data_key.nil?
-        ->(tag, time, record) {
-          unless fluentd_v0_12?
-            record = inject_values_to_record(tag, time, record)
+        compressor = compressor_create
+        if @data_key.nil?
+          if @chomp_record
+            ->(tag, time, record) {
+              record = inject_values_to_record(tag, time, record)
+              # Formatter calls chomp and removes separator from the end of each record.
+              # This option is for compatible format with plugin v2.
+              # https://github.com/awslabs/aws-fluent-plugin-kinesis/issues/142
+              compressor.call(formatter.format(tag, time, record).chomp.b)
+            }
+          else
+            ->(tag, time, record) {
+              record = inject_values_to_record(tag, time, record)
+              compressor.call(formatter.format(tag, time, record).b)
+            }
           end
-          compressor.call(formatter.format(tag, time, record).chomp.b)
-        }
-      else
-        ->(tag, time, record) {
-          raise InvalidRecordError, record unless record.is_a? Hash
-          raise KeyNotFoundError.new(@data_key, record) if record[@data_key].nil?
-          compressor.call(record[@data_key].to_s.b)
-        }
+        else
+          ->(tag, time, record) {
+            raise InvalidRecordError, record unless record.is_a? Hash
+            raise KeyNotFoundError.new(@data_key, record) if record[@data_key].nil?
+            compressor.call(record[@data_key].to_s.b)
+          }
+        end
       end
-    end
 
-    def compressor_create
-      case @compression
-      when "zlib"
-        ->(data) { Zlib::Deflate.deflate(data) }
-      else
-        ->(data) { data }
+      def compressor_create
+        case @compression
+        when "zlib"
+          ->(data) { Zlib::Deflate.deflate(data) }
+        else
+          ->(data) { data }
+        end
       end
-    end
 
-    def format_for_api(&block)
-      converted = block.call
-      size = size_of_values(converted)
-      if size > @max_record_size
-        raise ExceedMaxRecordSizeError.new(size, converted)
+      def format_for_api(&block)
+        converted = block.call
+        size = size_of_values(converted)
+        if size > @max_record_size
+          raise ExceedMaxRecordSizeError.new(size, converted)
+        end
+        converted.to_msgpack
+      rescue SkipRecordError => e
+        log.error(truncate e)
+        ''
       end
-      converted.to_msgpack
-    rescue SkipRecordError => e
-      log.error(truncate e)
-      ''
-    end
 
-    def write_records_batch(chunk, &block)
-      if fluentd_v0_12?
-        unique_id = chunk.unique_id.unpack('H*').first
-      else
+      def write_records_batch(chunk, &block)
         unique_id = chunk.dump_unique_id_hex(chunk.unique_id)
+        chunk.open do |io|
+          records = msgpack_unpacker(io).to_enum
+          split_to_batches(records) do |batch, size|
+            log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
+            batch_request_with_retry(batch, &block)
+            log.debug("Finish writing chunk")
+          end
+        end
       end
-      records = chunk.to_enum(:msgpack_each)
-      split_to_batches(records) do |batch, size|
-        log.debug(sprintf "Write chunk %s / %3d records / %4d KB", unique_id, batch.size, size/1024)
-        batch_request_with_retry(batch, &block)
-        log.debug("Finish writing chunk")
-      end
-    end
 
-    def request_type
-      self.class::RequestType
-    end
+      def request_type
+        self.class::RequestType
+      end
 
-    def truncate(msg)
-      if @log_truncate_max_size == 0 or (msg.to_s.size <= @log_truncate_max_size)
-        msg.to_s
-      else
-        msg.to_s[0...@log_truncate_max_size]
+      def truncate(msg)
+        if @log_truncate_max_size == 0 or (msg.to_s.size <= @log_truncate_max_size)
+          msg.to_s
+        else
+          msg.to_s[0...@log_truncate_max_size]
+        end
       end
     end
   end

data/lib/fluent/plugin/kinesis_helper/aggregator.rb

@@ -34,64 +34,66 @@ Google::Protobuf::DescriptorPool.generated_pool.build do
 end
 
 module Fluent
-  module KinesisHelper
-    class Aggregator
-      AggregatedRecord = Google::Protobuf::DescriptorPool.generated_pool.lookup("AggregatedRecord").msgclass
-      Tag = Google::Protobuf::DescriptorPool.generated_pool.lookup("Tag").msgclass
-      Record = Google::Protobuf::DescriptorPool.generated_pool.lookup("Record").msgclass
+  module Plugin
+    module KinesisHelper
+      class Aggregator
+        AggregatedRecord = Google::Protobuf::DescriptorPool.generated_pool.lookup("AggregatedRecord").msgclass
+        Tag = Google::Protobuf::DescriptorPool.generated_pool.lookup("Tag").msgclass
+        Record = Google::Protobuf::DescriptorPool.generated_pool.lookup("Record").msgclass
 
-      class InvalidEncodingError < ::StandardError; end
+        class InvalidEncodingError < ::StandardError; end
 
-      MagicNumber = ['F3899AC2'].pack('H*')
+        MagicNumber = ['F3899AC2'].pack('H*')
 
-      def aggregate(records, partition_key)
-        message = AggregatedRecord.encode(AggregatedRecord.new(
-          partition_key_table: ['a', partition_key],
-          records: records.map{|data|
-            Record.new(partition_key_index: 1, data: data)
-          },
-        ))
-        [MagicNumber, message, Digest::MD5.digest(message)].pack("A4A*A16")
-      end
-
-      def deaggregate(encoded)
-        unless aggregated?(encoded)
-          raise InvalidEncodingError, "Invalid MagicNumber #{encoded[0..3]}}"
+        def aggregate(records, partition_key)
+          message = AggregatedRecord.encode(AggregatedRecord.new(
+            partition_key_table: ['a', partition_key],
+            records: records.map{|data|
+              Record.new(partition_key_index: 1, data: data)
+            },
+          ))
+          [MagicNumber, message, Digest::MD5.digest(message)].pack("A4A*A16")
         end
-        message, digest = encoded[4..encoded.length-17], encoded[encoded.length-16..-1]
-        if Digest::MD5.digest(message) != digest
-          raise InvalidEncodingError, "Digest mismatch #{digest}"
+
+        def deaggregate(encoded)
+          unless aggregated?(encoded)
+            raise InvalidEncodingError, "Invalid MagicNumber #{encoded[0..3]}}"
+          end
+          message, digest = encoded[4..encoded.length-17], encoded[encoded.length-16..-1]
+          if Digest::MD5.digest(message) != digest
+            raise InvalidEncodingError, "Digest mismatch #{digest}"
+          end
+          decoded = AggregatedRecord.decode(message)
+          records = decoded.records.map(&:data)
+          partition_key = decoded.partition_key_table[1]
+          [records, partition_key]
         end
-        decoded = AggregatedRecord.decode(message)
-        records = decoded.records.map(&:data)
-        partition_key = decoded.partition_key_table[1]
-        [records, partition_key]
-      end
 
-      def aggregated?(encoded)
-        encoded[0..3] == MagicNumber
-      end
+        def aggregated?(encoded)
+          encoded[0..3] == MagicNumber
+        end
 
-      def aggregated_size_offset(partition_key)
-        data = 'd'
-        encoded = aggregate([record(data)], partition_key)
-        finalize(encoded).size - data.size
-      end
+        def aggregated_size_offset(partition_key)
+          data = 'd'
+          encoded = aggregate([record(data)], partition_key)
+          finalize(encoded).size - data.size
+        end
 
-      module Mixin
-        AggregateOffset = 25
-        RecordOffset = 10
+        module Mixin
+          AggregateOffset = 25
+          RecordOffset = 10
 
-        module Params
-          include Fluent::Configurable
-        end
+          module Params
+            include Fluent::Configurable
+          end
 
-        def self.included(mod)
-          mod.include Params
-        end
+          def self.included(mod)
+            mod.include Params
+          end
 
-        def aggregator
-          @aggregator ||= Aggregator.new
+          def aggregator
+            @aggregator ||= Aggregator.new
+          end
         end
       end
     end