fluent-plugin-kinesis 1.3.0 → 2.0.0

data/lib/fluent/plugin/kinesis_helper.rb

@@ -1,36 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-require 'fluent/plugin/kinesis_helper/class_methods'
-require 'fluent/plugin/kinesis_helper/initialize'
-require 'fluent/plugin/kinesis_helper/error'
-
-module Fluent
-  module KinesisHelper
-    include Fluent::SetTimeKeyMixin
-    include Fluent::SetTagKeyMixin
-    # detach_multi_process has been deleted at 0.14.12
-    # https://github.com/fluent/fluentd/commit/fcd8cc18e1f3a95710a80f982b91a1414fadc432
-    require 'fluent/version'
-    if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.14.12')
-      require 'fluent/process'
-      include Fluent::DetachMultiProcessMixin
-    end
-
-    def self.included(klass)
-      klass.extend ClassMethods
-    end
-    include Initialize
-  end
-end

data/lib/fluent/plugin/kinesis_helper/class_methods.rb

@@ -1,123 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-module Fluent
-  module KinesisHelper
-    module ClassMethods
-      def config_param_for_streams
-        const_set(:RequestType, :streams)
-        const_set(:BatchRequestLimitCount, 500)
-        const_set(:BatchRequestLimitSize, 5 * 1024 * 1024)
-        config_param :stream_name,   :string
-        config_param :region,        :string,  default: nil
-        config_param :partition_key, :string,  default: nil
-        config_param_for_sdk
-        config_param_for_credentials
-        config_param_for_format
-        config_param_for_batch_request
-        config_param_for_debug
-      end
-
-      def config_param_for_firehose
-        const_set(:RequestType, :firehose)
-        const_set(:BatchRequestLimitCount, 500)
-        const_set(:BatchRequestLimitSize, 4 * 1024 * 1024)
-        config_param :delivery_stream_name, :string
-        config_param :region,               :string,  default: nil
-        config_param :append_new_line,      :bool,    default: true
-        config_param_for_sdk
-        config_param_for_credentials
-        config_param_for_format
-        config_param_for_batch_request
-        config_param_for_debug
-      end
-
-      def config_param_for_producer
-        const_set(:RequestType, :producer)
-        config_param :stream_name,        :string, default: nil
-        config_param :stream_name_prefix, :string, default: nil
-        config_param :region,             :string, default: nil
-        config_param :partition_key,      :string, default: nil
-        config_param :enable_kpl_logging, :bool,   default: false
-        config_param_for_credentials
-        config_param_for_format
-        config_param_for_debug
-
-        config_section :kinesis_producer, multi: false do
-          require 'kinesis_producer'
-          type_map = {
-            Protobuf::Field::BoolField   => :bool,
-            Protobuf::Field::Uint64Field => :integer,
-            Protobuf::Field::StringField => :string,
-          }
-          KinesisProducer::ConfigurationFields.each do |field|
-            next if field.name == 'region'
-            type = type_map[field.type_class]
-            config_param field.name, type, default: field.default_value
-          end
-          config_param :credentials_refresh_delay, :integer, default: 5000
-        end
-      end
-
-      def config_param_for_sdk
-        config_param :http_proxy,      :string, default: nil, secret: true
-        config_param :endpoint,        :string, default: nil
-        config_param :ssl_verify_peer, :bool,   default: true
-      end
-
-      def config_param_for_credentials
-        config_param :aws_key_id,  :string, default: nil, secret: true
-        config_param :aws_sec_key, :string, default: nil, secret: true
-        config_section :shared_credentials, multi: false do
-          config_param :profile_name, :string, default: nil
-          config_param :path,         :string, default: nil
-        end
-        config_section :assume_role_credentials, multi: false do
-          config_param :role_arn,    :string,               secret: true
-          config_param :external_id, :string, default: nil, secret: true
-        end
-      end
-
-      def config_param_for_format
-        config_param :formatter,             :string,  default: 'json'
-        config_param :data_key,              :string,  default: nil
-        config_param :log_truncate_max_size, :integer, default: 0
-        config_param :reduce_max_size_error_message, :bool, default: false
-      end
-
-      def config_param_for_batch_request
-        config_param :retries_on_batch_request, :integer, default: 3
-        config_param :reset_backoff_if_success, :bool,    default: true
-        config_param :batch_request_max_count,  :integer, default: const_get(:BatchRequestLimitCount)
-        config_param :batch_request_max_size,   :integer, default: const_get(:BatchRequestLimitSize)
-      end
-
-      def config_param_for_debug
-        config_param :debug, :bool, default: false
-      end
-
-      def request_type
-        const_get(:RequestType)
-      end
-
-      def api?
-        [:streams, :firehose].include?(request_type)
-      end
-
-      def kpl?
-        [:producer].include?(request_type)
-      end
-    end
-  end
-end

data/lib/fluent/plugin/kinesis_helper/credentials.rb

@@ -1,51 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-module Fluent
-  module KinesisHelper
-    module Credentials
-      def credentials
-        @provider ||= case
-          when @assume_role_credentials
-            Aws::AssumeRoleCredentials.new(
-              client:            Aws::STS::Client.new(region: @region),
-              role_arn:          @assume_role_credentials.role_arn,
-              external_id:       @assume_role_credentials.external_id,
-              role_session_name: 'aws-fluent-plugin-kinesis',
-              duration_seconds:  60 * 60,
-          )
-          when @shared_credentials
-            Aws::SharedCredentials.new(
-              profile_name: @shared_credentials.profile_name,
-              path:         @shared_credentials.path,
-            )
-          else
-            default_credentials_provider
-          end
-      end
-
-      private
-
-      def default_credentials_provider
-        config_class = Struct.new(:access_key_id, :secret_access_key, :region, :session_token, :profile, :instance_profile_credentials_retries, :instance_profile_credentials_timeout)
-        config = config_class.new(@aws_key_id, @aws_sec_key, @region)
-        provider = Aws::CredentialProviderChain.new(config).resolve
-        if provider.nil?
-          raise Fluent::ConfigError, "You must specify credentials on ~/.aws/credentials, environment variables or instance profile for default credentials"
-        end
-        provider
-      end
-    end
-  end
-end

data/lib/fluent/plugin/kinesis_helper/error.rb

@@ -1,43 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-module Fluent
-  module KinesisHelper
-    class BaseError < ::StandardError; end
-    class SkipRecordError < BaseError; end
-
-    class KeyNotFoundError < SkipRecordError
-      def initialize(key, record)
-        super "Key '#{key}' doesn't exist on #{record}"
-      end
-    end
-
-    class ExceedMaxRecordSizeError < SkipRecordError
-      def initialize(record, reduce_message)
-        if reduce_message == true
-          sampled_record = "#{record.slice(0,1024)}...#{record.slice(-1024,1024)}"
-          super "Record size limit exceeded for #{record.length}-bytes record: #{sampled_record}"
-        else
-          super "Record size limit exceeded for #{record}"
-        end
-      end
-    end
-
-    class InvalidRecordError < SkipRecordError
-      def initialize(record)
-        super "Invalid type of record: #{record}"
-      end
-    end
-  end
-end

data/lib/fluent/plugin/kinesis_helper/format.rb

@@ -1,85 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-require 'fluent/plugin/kinesis_helper/error'
-
-module Fluent
-  module KinesisHelper
-    module Format
-      MaxRecordSize = 1024 * 1024 # 1 MB
-
-      def configure(conf)
-        super
-        @formatter = Fluent::Plugin.new_formatter(@formatter)
-        @formatter.configure(conf)
-      end
-
-      def format(tag, time, record)
-        [tag, time, record].to_msgpack
-      end
-
-      private
-
-      def data_format(tag, time, record)
-        if @data_key and record[@data_key].nil?
-          raise KeyNotFoundError.new(@data_key, record)
-        elsif @data_key
-          record[@data_key].to_s
-        else
-          @formatter.format(tag, time, record).chomp
-        end
-      end
-
-      def key(record)
-        if @partition_key.nil?
-          SecureRandom.hex(16)
-        elsif !record.key?(@partition_key)
-          raise KeyNotFoundError.new(@partition_key, record)
-        else
-          record[@partition_key]
-        end
-      end
-
-      def convert_to_records(chunk)
-        chunk.to_enum(:msgpack_each).map{|tag, time, record|
-          convert_record(tag, time, record)
-        }.compact
-      end
-
-      def convert_record(tag, time, record)
-        unless record.is_a? Hash
-          raise InvalidRecordError, record
-        end
-        converted = convert_format(tag, time, record)
-        converted[:data] += "\n" if @append_new_line
-        if converted[:data].size > MaxRecordSize
-          raise ExceedMaxRecordSizeError.new(converted[:data], @reduce_max_size_error_message)
-        else
-          converted
-        end
-      rescue SkipRecordError => e
-        log.error(truncate e)
-        nil
-      end
-
-      def truncate(msg)
-        if @log_truncate_max_size == 0 or (msg.to_s.size <= @log_truncate_max_size)
-          msg.to_s
-        else
-          msg.to_s[0...@log_truncate_max_size]
-        end
-      end
-    end
-  end
-end

data/lib/fluent/plugin/kinesis_helper/initialize.rb

@@ -1,59 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-module Fluent
-  module KinesisHelper
-    module Initialize
-      def initialize
-        super
-        class << self
-          define_method('log') { $log } unless method_defined?(:log)
-
-          require 'aws-sdk'
-          require 'fluent/plugin/kinesis_helper/format'
-          require 'fluent/plugin/kinesis_helper/client'
-          require 'fluent/plugin/kinesis_helper/credentials'
-          include Format, Client, Credentials
-
-          case
-          when api?
-            require 'fluent/plugin/kinesis_helper/api'
-            include API
-          when kpl?
-            require 'fluent/plugin/kinesis_helper/kpl'
-            require 'fluent/version'
-            if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.12.20')
-              # Backport from https://github.com/fluent/fluentd/pull/757
-              require 'fluent/plugin/patched_detach_process_impl'
-              include PatchedDetachProcessImpl
-            end
-            include KPL
-          end
-
-          def request_type
-            self.class.request_type
-          end
-
-          def api?
-            self.class.api?
-          end
-
-          def kpl?
-            self.class.kpl?
-          end
-        end
-      end
-    end
-  end
-end

data/lib/fluent/plugin/kinesis_helper/kpl.rb

@@ -1,82 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-module Fluent
-  module KinesisHelper
-    module KPL
-      def configure(conf)
-        # To workaround when the kinesis_producer section is not specified
-        if conf.elements.none?{|e|e.name == "kinesis_producer"}
-          conf.add_element("kinesis_producer")
-        end
-        super(conf)
-        if @region.nil?
-          keys = %w(AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION)
-          @region = ENV.values_at(*keys).compact.first
-        end
-      end
-
-      def start
-        detach_multi_process do
-          super
-          client
-        end
-      end
-
-      def shutdown
-        super
-        client.destroy
-      end
-
-      private
-
-      def client_options
-        {
-          credentials: credentials,
-          configuration: build_kpl_configuration,
-          credentials_refresh_delay: @kinesis_producer.credentials_refresh_delay,
-          debug: @debug,
-          logger: log,
-          enable_kpl_logging: @enable_kpl_logging,
-        }
-      end
-
-      def build_kpl_configuration
-        configuration = @kinesis_producer.to_h
-        configuration.update(region: @region) unless @region.nil?
-      end
-
-      def write_chunk_to_kpl(records)
-        records.map do |record|
-          client.put_record(record)
-        end
-      end
-
-      def wait_futures(futures)
-        futures.each do |f|
-          f.wait
-          result = f.value!
-        end
-      end
-
-      def on_detach_process(i)
-        Process.setsid
-      end
-
-      def on_exit_process(i)
-        shutdown
-      end
-    end
-  end
-end