fluent-plugin-kinesis 1.3.0 → 2.0.0

data/lib/fluent/plugin/out_kinesis.rb

@@ -1,323 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-require 'aws-sdk'
-require 'yajl'
-require 'logger'
-require 'securerandom'
-require 'zlib'
-require 'fluent_plugin_kinesis/version'
-
-module FluentPluginKinesis
-  class OutputFilter < Fluent::BufferedOutput
-
-    # detach_multi_process has been deleted at 0.14.12
-    # https://github.com/fluent/fluentd/commit/fcd8cc18e1f3a95710a80f982b91a1414fadc432
-    require 'fluent/version'
-    if Gem::Version.new(Fluent::VERSION) < Gem::Version.new('0.14.12')
-      require 'fluent/process'
-      include Fluent::DetachMultiProcessMixin
-    end
-    include Fluent::SetTimeKeyMixin
-    include Fluent::SetTagKeyMixin
-
-    USER_AGENT_NAME = 'fluent-plugin-kinesis-output-filter'
-    PROC_BASE_STR = 'proc {|record| %s }'
-    PUT_RECORDS_MAX_COUNT = 500
-    PUT_RECORD_MAX_DATA_SIZE = 1024 * 1024
-    PUT_RECORDS_MAX_DATA_SIZE = 1024 * 1024 * 5
-
-    Fluent::Plugin.register_output('kinesis',self)
-
-    config_set_default :include_time_key, true
-    config_set_default :include_tag_key,  true
-
-    config_param :aws_key_id,  :string, default: nil, :secret => true
-    config_param :aws_sec_key, :string, default: nil, :secret => true
-    # The 'region' parameter is optional because
-    # it may be set as an environment variable.
-    config_param :region,      :string, default: nil
-    config_param :ensure_stream_connection, :bool, default: true
-
-    config_param :profile,          :string, :default => nil
-    config_param :credentials_path, :string, :default => nil
-    config_param :role_arn,         :string, :default => nil
-    config_param :external_id,      :string, :default => nil
-
-    config_param :stream_name,            :string
-    config_param :random_partition_key,   :bool,   default: false
-    config_param :partition_key,          :string, default: nil
-    config_param :partition_key_expr,     :string, default: nil
-    config_param :explicit_hash_key,      :string, default: nil
-    config_param :explicit_hash_key_expr, :string, default: nil
-    config_param :order_events,           :bool,   default: false
-    config_param :retries_on_putrecords,  :integer, default: 3
-    config_param :use_yajl,               :bool,   default: false
-    config_param :zlib_compression,       :bool,   default: false
-
-    config_param :debug, :bool, default: false
-
-    config_param :http_proxy, :string, default: nil
-
-    def configure(conf)
-      log.warn("Deprecated warning: out_kinesis is no longer supported after v1.0.0. Please check out_kinesis_streams out.")
-      super
-      validate_params
-
-      if @detach_process or (@num_threads > 1)
-        @parallel_mode = true
-        if @detach_process
-          @use_detach_multi_process_mixin = true
-        end
-      else
-        @parallel_mode = false
-      end
-
-      if @parallel_mode
-        if @order_events
-          log.warn 'You have set "order_events" to true, however this configuration will be ignored due to "detach_process" and/or "num_threads".'
-        end
-        @order_events = false
-      end
-
-      if @partition_key_expr
-        partition_key_proc_str = sprintf(
-          PROC_BASE_STR, @partition_key_expr
-        )
-        @partition_key_proc = eval(partition_key_proc_str)
-      end
-
-      if @explicit_hash_key_expr
-        explicit_hash_key_proc_str = sprintf(
-          PROC_BASE_STR, @explicit_hash_key_expr
-        )
-        @explicit_hash_key_proc = eval(explicit_hash_key_proc_str)
-      end
-
-      @dump_class = @use_yajl ? Yajl : JSON
-    end
-
-    def start
-      detach_multi_process do
-        super
-        load_client
-        if @ensure_stream_connection
-          check_connection_to_stream
-        end
-      end
-    end
-
-    def format(tag, time, record)
-      data = {
-        data: @dump_class.dump(record),
-        partition_key: get_key(:partition_key,record)
-      }
-
-      if @explicit_hash_key or @explicit_hash_key_proc
-        data[:explicit_hash_key] = get_key(:explicit_hash_key,record)
-      end
-
-      data.to_msgpack
-    end
-
-    def write(chunk)
-      data_list = chunk.to_enum(:msgpack_each).map{|record|
-        build_data_to_put(record)
-      }.find_all{|record|
-        unless record_exceeds_max_size?(record[:data])
-          true
-        else
-          log.error sprintf('Record exceeds the %.3f KB(s) per-record size limit and will not be delivered: %s', PUT_RECORD_MAX_DATA_SIZE / 1024.0, record[:data])
-          false
-        end
-      }
-
-      if @order_events
-        put_record_for_order_events(data_list)
-      else
-        records_array = build_records_array_to_put(data_list)
-        records_array.each{|records|
-          put_records_with_retry(records)
-        }
-      end
-    end
-
-    private
-    def validate_params
-      unless @random_partition_key or @partition_key or @partition_key_expr
-        raise Fluent::ConfigError, "'random_partition_key' or 'partition_key' or 'partition_key_expr' is required"
-      end
-    end
-
-    def load_client
-
-      user_agent_suffix = "#{USER_AGENT_NAME}/#{FluentPluginKinesis::VERSION}"
-
-      options = {
-        user_agent_suffix: user_agent_suffix
-      }
-
-      if @region
-        options[:region] = @region
-      end
-
-      if @aws_key_id && @aws_sec_key
-        options.update(
-          access_key_id: @aws_key_id,
-          secret_access_key: @aws_sec_key,
-        )
-      elsif @profile
-        credentials_opts = {:profile_name => @profile}
-        credentials_opts[:path] = @credentials_path if @credentials_path
-        credentials = Aws::SharedCredentials.new(credentials_opts)
-        options[:credentials] = credentials
-      elsif @role_arn
-        credentials = Aws::AssumeRoleCredentials.new(
-          client: Aws::STS::Client.new(options),
-          role_arn: @role_arn,
-          role_session_name: "aws-fluent-plugin-kinesis",
-          external_id: @external_id,
-          duration_seconds: 60 * 60
-        )
-        options[:credentials] = credentials
-      end
-
-      if @debug
-        options.update(
-          logger: Logger.new(log.out),
-          log_level: :debug
-        )
-        # XXX: Add the following options, if necessary
-        # :http_wire_trace => true
-      end
-
-      if @http_proxy
-        options[:http_proxy] = @http_proxy
-      end
-
-      @client = Aws::Kinesis::Client.new(options)
-
-    end
-
-    def check_connection_to_stream
-      @client.describe_stream(stream_name: @stream_name)
-    end
-
-    def get_key(name, record)
-      if @random_partition_key
-        SecureRandom.uuid
-      else
-        key = instance_variable_get("@#{name}")
-        key_proc = instance_variable_get("@#{name}_proc")
-
-        value = key ? record[key] : record
-
-        if key_proc
-          value = key_proc.call(value)
-        end
-
-        value.to_s
-      end
-    end
-
-    def build_data_to_put(data)
-      if @zlib_compression
-        Hash[data.map{|k, v| [k.to_sym, k=="data" ? Zlib::Deflate.deflate(v) : v] }]
-      else
-        Hash[data.map{|k, v| [k.to_sym, v] }]
-      end
-    end
-
-    def put_record_for_order_events(data_list)
-      sequence_number_for_ordering = nil
-      data_list.each do |data_to_put|
-        if sequence_number_for_ordering
-          data_to_put.update(
-            sequence_number_for_ordering: sequence_number_for_ordering
-          )
-        end
-        data_to_put.update(
-          stream_name: @stream_name
-        )
-        result = @client.put_record(data_to_put)
-        sequence_number_for_ordering = result[:sequence_number]
-      end
-    end
-
-    def build_records_array_to_put(data_list)
-      records_array = []
-      records = []
-      records_payload_length = 0
-      data_list.each{|data_to_put|
-        payload = data_to_put[:data]
-        partition_key = data_to_put[:partition_key]
-        if records.length >= PUT_RECORDS_MAX_COUNT or (records_payload_length + payload.length + partition_key.length) >= PUT_RECORDS_MAX_DATA_SIZE
-          records_array.push(records)
-          records = []
-          records_payload_length = 0
-        end
-        records.push(data_to_put)
-        records_payload_length += (payload.length + partition_key.length)
-      }
-      records_array.push(records) unless records.empty?
-      records_array
-    end
-
-    def put_records_with_retry(records,retry_count=0)
-      response = @client.put_records(
-        records: records,
-        stream_name: @stream_name
-      )
-
-      if response[:failed_record_count] && response[:failed_record_count] > 0
-        failed_records = []
-        response[:records].each_with_index{|record,index|
-          if record[:error_code]
-            failed_records.push({body: records[index], error_code: record[:error_code]})
-          end
-        }
-
-        if(retry_count < @retries_on_putrecords)
-          sleep(calculate_sleep_duration(retry_count))
-          retry_count += 1
-          log.warn sprintf('Retrying to put records. Retry count: %d', retry_count)
-          put_records_with_retry(
-            failed_records.map{|record| record[:body]},
-            retry_count
-          )
-        else
-          failed_records.each{|record|
-            log.error sprintf(
-              'Could not put record, Error: %s, Record: %s',
-              record[:error_code],
-              @dump_class.dump(record[:body])
-            )
-          }
-        end
-      end
-    end
-
-    def calculate_sleep_duration(current_retry)
-      Array.new(@retries_on_putrecords){|n| ((2 ** n) * scaling_factor)}[current_retry]
-    end
-
-    def scaling_factor
-      0.5 + Kernel.rand * 0.1
-    end
-
-    def record_exceeds_max_size?(record_string)
-      return record_string.length > PUT_RECORD_MAX_DATA_SIZE
-    end
-  end
-end

data/lib/fluent/plugin/out_kinesis_producer.rb

@@ -1,48 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-require 'fluent/plugin/kinesis_helper'
-
-module Fluent
-  class KinesisProducerOutput < BufferedOutput
-    include KinesisHelper
-    Fluent::Plugin.register_output('kinesis_producer', self)
-    config_param_for_producer
-
-    def configure(conf)
-      super
-      unless @stream_name or @stream_name_prefix
-        raise Fluent::ConfigError, "'stream_name' or 'stream_name_prefix' is required"
-      end
-      if @stream_name and @stream_name_prefix
-        raise Fluent::ConfigError, "Only one of 'stream_name' or 'stream_name_prefix' is allowed"
-      end
-    end
-
-    def write(chunk)
-      records = convert_to_records(chunk)
-      wait_futures(write_chunk_to_kpl(records))
-    end
-
-    private
-
-    def convert_format(tag, time, record)
-      {
-        data: data_format(tag, time, record),
-        partition_key: key(record),
-        stream_name: @stream_name ? @stream_name : @stream_name_prefix + tag,
-      }
-    end
-  end
-end

data/lib/fluent/plugin/patched_detach_process_impl.rb

@@ -1,103 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-module Fluent
-  module PatchedDetachProcessImpl
-    def on_detach_process(i)
-    end
-
-    def on_exit_process(i)
-    end
-
-    private
-
-    def detach_process_impl(num, &block)
-      children = []
-
-      num.times do |i|
-        pid, forward_thread = DetachProcessManager.instance.fork(self)
-
-        if pid
-          # parent process
-          $log.info "detached process", :class=>self.class, :pid=>pid
-          children << [pid, forward_thread]
-          next
-        end
-
-        # child process
-        begin
-          on_detach_process(i)
-
-          block.call
-
-          # disable Engine.stop called by signal handler
-          Engine.define_singleton_method(:stop) do
-            # do nothing
-          end
-          # override signal handlers called by parent process
-          fin = ::Fluent::DetachProcessImpl::FinishWait.new
-          trap :INT do
-            fin.stop
-          end
-          trap :TERM do
-            fin.stop
-          end
-          #forward_thread.join  # TODO this thread won't stop because parent doesn't close pipe
-          fin.wait
-
-          on_exit_process(i)
-          exit! 0
-        ensure
-          $log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid
-          $log.error_backtrace
-        end
-
-        exit! 1
-      end
-
-      # parent process
-      # override shutdown method to kill child processes
-      define_singleton_method(:shutdown) do
-        children.each {|pair|
-          begin
-            pid = pair[0]
-            forward_thread = pair[1]
-            if pid
-              Process.kill(:TERM, pid)
-              forward_thread.join   # wait until child closes pipe
-              Process.waitpid(pid)
-              pair[0] = nil
-            end
-          rescue
-            $log.error "unknown error while shutting down remote child process", :error=>$!.to_s
-            $log.error_backtrace
-          end
-        }
-      end
-
-      # override target.emit and write event stream to the pipe
-      forwarders = children.map {|pair| pair[1].forwarder }
-      if forwarders.length > 1
-        # use roundrobin
-        fwd = DetachProcessManager::MultiForwarder.new(forwarders)
-      else
-        fwd = forwarders[0]
-      end
-      define_singleton_method(:emit) do |tag,es,chain|
-        chain.next
-        fwd.emit(tag, es)
-      end
-    end
-  end
-end