fluent-plugin-kinesis 0.4.1 → 1.0.0

data/lib/fluent/plugin/kinesis_helper/client.rb

@@ -0,0 +1,36 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Fluent
+  module KinesisHelper
+    module Client
+      def client
+        @client ||= client_class.new(client_options)
+      end
+
+      private
+
+      def client_class
+        case request_type
+        when :streams
+          Aws::Kinesis::Client
+        when :firehose
+          Aws::Firehose::Client
+        when :producer
+          KinesisProducer::Library
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kinesis_helper/credentials.rb

@@ -0,0 +1,51 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Fluent
+  module KinesisHelper
+    module Credentials
+      def credentials
+        case
+        when @assume_role_credentials
+          Aws::AssumeRoleCredentials.new(
+            client:            Aws::STS::Client.new(region: @region),
+            role_arn:          @assume_role_credentials.role_arn,
+            external_id:       @assume_role_credentials.external_id,
+            role_session_name: 'aws-fluent-plugin-kinesis',
+            duration_seconds:  60 * 60,
+        )
+        when @shared_credentials
+          Aws::SharedCredentials.new(
+            profile_name: @shared_credentials.profile_name,
+            path:         @shared_credentials.path,
+          )
+        else
+          default_credentials_provider
+        end
+      end
+
+      private
+
+      def default_credentials_provider
+        config_class = Struct.new(:access_key_id, :secret_access_key, :session_token, :profile)
+        config = config_class.new(@aws_key_id, @aws_sec_key)
+        provider = Aws::CredentialProviderChain.new(config).resolve
+        if provider.nil?
+          raise Fluent::ConfigError, "You must specify credentials on ~/.aws/credentials, environment variables or IAM role for default credentials"
+        end
+        provider.credentials
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kinesis_helper/error.rb

@@ -0,0 +1,38 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Fluent
+  module KinesisHelper
+    class BaseError < ::StandardError; end
+    class SkipRecordError < BaseError; end
+
+    class KeyNotFoundError < SkipRecordError
+      def initialize(key, record)
+        super "Key '#{key}' doesn't exist on #{record}"
+      end
+    end
+
+    class ExceedMaxRecordSizeError < SkipRecordError
+      def initialize(record)
+        super "Record size limit exceeded for #{record}"
+      end
+    end
+
+    class InvalidRecordError < SkipRecordError
+      def initialize(record)
+        super "Invalid type of record: #{record}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kinesis_helper/format.rb

@@ -0,0 +1,85 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis_helper/error'
+
+module Fluent
+  module KinesisHelper
+    module Format
+      MaxRecordSize = 1024 * 1024 # 1 MB
+
+      def configure(conf)
+        super
+        @formatter = Fluent::Plugin.new_formatter(@formatter)
+        @formatter.configure(conf)
+      end
+
+      def format(tag, time, record)
+        [tag, time, record].to_msgpack
+      end
+
+      private
+
+      def data_format(tag, time, record)
+        if @data_key and record[@data_key].nil?
+          raise KeyNotFoundError.new(@data_key, record)
+        elsif @data_key
+          record[@data_key].to_s
+        else
+          @formatter.format(tag, time, record).chomp
+        end
+      end
+
+      def key(record)
+        if @partition_key.nil?
+          SecureRandom.hex(16)
+        elsif !record.key?(@partition_key)
+          raise KeyNotFoundError.new(@partition_key, record)
+        else
+          record[@partition_key]
+        end
+      end
+
+      def convert_to_records(chunk)
+        chunk.to_enum(:msgpack_each).map{|tag, time, record|
+          convert_record(tag, time, record)
+        }.compact
+      end
+
+      def convert_record(tag, time, record)
+        unless record.is_a? Hash
+          raise InvalidRecordError, record
+        end
+        converted = convert_format(tag, time, record)
+        converted[:data] += "\n" if @append_new_line
+        if converted[:data].size > MaxRecordSize
+          raise ExceedMaxRecordSizeError, converted[:data]
+        else
+          converted
+        end
+      rescue SkipRecordError => e
+        log.error(truncate e)
+        nil
+      end
+
+      def truncate(msg)
+        if @log_truncate_max_size == 0 or (msg.to_s.size <= @log_truncate_max_size)
+          msg.to_s
+        else
+          msg.to_s[0...@log_truncate_max_size]
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kinesis_helper/initialize.rb

@@ -0,0 +1,58 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Fluent
+  module KinesisHelper
+    module Initialize
+      def initialize
+        super
+        class << self
+          define_method('log') { $log } unless method_defined?(:log)
+
+          require 'aws-sdk'
+          require 'fluent/plugin/kinesis_helper/format'
+          require 'fluent/plugin/kinesis_helper/client'
+          require 'fluent/plugin/kinesis_helper/credentials'
+          include Format, Client, Credentials
+
+          case
+          when api?
+            require 'fluent/plugin/kinesis_helper/api'
+            include API
+          when kpl?
+            require 'fluent/plugin/kinesis_helper/kpl'
+            if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.12.20')
+              # Backport from https://github.com/fluent/fluentd/pull/757
+              require 'fluent/plugin/patched_detach_process_impl'
+              include PatchedDetachProcessImpl
+            end
+            include KPL
+          end
+
+          def request_type
+            self.class.request_type
+          end
+
+          def api?
+            self.class.api?
+          end
+
+          def kpl?
+            self.class.kpl?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/kinesis_helper/kpl.rb

@@ -0,0 +1,81 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Fluent
+  module KinesisHelper
+    module KPL
+      def configure(conf)
+        # To workaround when the kinesis_producer section is not specified
+        if conf.elements.none?{|e|e.name == "kinesis_producer"}
+          conf.add_element("kinesis_producer")
+        end
+        super(conf)
+        if @region.nil?
+          keys = %w(AWS_REGION AMAZON_REGION AWS_DEFAULT_REGION)
+          @region = ENV.values_at(*keys).compact.first
+        end
+      end
+
+      def start
+        detach_multi_process do
+          super
+          client
+        end
+      end
+
+      def shutdown
+        super
+        client.destroy
+      end
+
+      private
+
+      def client_options
+        {
+          credentials: credentials,
+          configuration: build_kpl_configuration,
+          credentials_refresh_delay: @kinesis_producer.credentials_refresh_delay,
+          debug: @debug,
+          logger: log,
+        }
+      end
+
+      def build_kpl_configuration
+        configuration = @kinesis_producer.to_h
+        configuration.update(region: @region) unless @region.nil?
+      end
+
+      def write_chunk_to_kpl(records)
+        records.map do |record|
+          client.put_record(record)
+        end
+      end
+
+      def wait_futures(futures)
+        futures.each do |f|
+          f.wait
+          result = f.value!
+        end
+      end
+
+      def on_detach_process(i)
+        Process.setsid
+      end
+
+      def on_exit_process(i)
+        shutdown
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_kinesis.rb

@@ -1,22 +1,23 @@
-# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-# Licensed under the Apache License, Version 2.0 (the "License"). You
-# may not use this file except in compliance with the License. A copy of
-# the License is located at
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
 #
-# or in the "license" file accompanying this file. This file is
-# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
-# ANY KIND, either express or implied. See the License for the specific
-# language governing permissions and limitations under the License.
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
 
-require 'aws-sdk-core'
+require 'aws-sdk'
 require 'yajl'
 require 'logger'
 require 'securerandom'
-require 'fluent/plugin/version'
 require 'zlib'
+require 'fluent_plugin_kinesis/version'
 
 module FluentPluginKinesis
   class OutputFilter < Fluent::BufferedOutput
@@ -64,6 +65,7 @@ module FluentPluginKinesis
     config_param :http_proxy, :string, default: nil
 
     def configure(conf)
+      log.warn("Deprecated warning: out_kinesis is no longer supported after v1.0.0. Please check out_kinesis_streams out.")
       super
       validate_params
 

data/lib/fluent/plugin/out_kinesis_firehose.rb

@@ -0,0 +1,44 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis_helper'
+
+module Fluent
+  class KinesisFirehoseOutput < BufferedOutput
+    include KinesisHelper
+    Fluent::Plugin.register_output('kinesis_firehose', self)
+    config_param_for_firehose
+
+    def write(chunk)
+      records = convert_to_records(chunk)
+      split_to_batches(records).each do |batch|
+        batch_request_with_retry(batch)
+      end
+      log.debug("Written #{records.size} records")
+    end
+
+    private
+
+    def convert_format(tag, time, record)
+      { data: data_format(tag, time, record) }
+    end
+
+    def batch_request(batch)
+      client.put_record_batch(
+        delivery_stream_name: @delivery_stream_name,
+        records: batch
+      )
+    end
+  end
+end

data/lib/fluent/plugin/out_kinesis_producer.rb

@@ -0,0 +1,38 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis_helper'
+
+module Fluent
+  class KinesisProducerOutput < BufferedOutput
+    include KinesisHelper
+    Fluent::Plugin.register_output('kinesis_producer', self)
+    config_param_for_producer
+
+    def write(chunk)
+      records = convert_to_records(chunk)
+      wait_futures(write_chunk_to_kpl(records))
+    end
+
+    private
+
+    def convert_format(tag, time, record)
+      {
+        data: data_format(tag, time, record),
+        partition_key: key(record),
+        stream_name: @stream_name,
+      }
+    end
+  end
+end