fluent-plugin-kinesis 0.4.1 → 1.0.0

data/lib/fluent/plugin/out_kinesis_streams.rb

@@ -0,0 +1,47 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'fluent/plugin/kinesis_helper'
+
+module Fluent
+  class KinesisStreamsOutput < BufferedOutput
+    include KinesisHelper
+    Fluent::Plugin.register_output('kinesis_streams', self)
+    config_param_for_streams
+
+    def write(chunk)
+      records = convert_to_records(chunk)
+      split_to_batches(records).each do |batch|
+        batch_request_with_retry(batch)
+      end
+      log.debug("Written #{records.size} records")
+    end
+
+    private
+
+    def convert_format(tag, time, record)
+      {
+        data: data_format(tag, time, record),
+        partition_key: key(record),
+      }
+    end
+
+    def batch_request(batch)
+      client.put_records(
+        stream_name: @stream_name,
+        records: batch,
+      )
+    end
+  end
+end

data/lib/fluent/plugin/patched_detach_process_impl.rb

@@ -0,0 +1,103 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Fluent
+  module PatchedDetachProcessImpl
+    def on_detach_process(i)
+    end
+
+    def on_exit_process(i)
+    end
+
+    private
+
+    def detach_process_impl(num, &block)
+      children = []
+
+      num.times do |i|
+        pid, forward_thread = DetachProcessManager.instance.fork(self)
+
+        if pid
+          # parent process
+          $log.info "detached process", :class=>self.class, :pid=>pid
+          children << [pid, forward_thread]
+          next
+        end
+
+        # child process
+        begin
+          on_detach_process(i)
+
+          block.call
+
+          # disable Engine.stop called by signal handler
+          Engine.define_singleton_method(:stop) do
+            # do nothing
+          end
+          # override signal handlers called by parent process
+          fin = ::Fluent::DetachProcessImpl::FinishWait.new
+          trap :INT do
+            fin.stop
+          end
+          trap :TERM do
+            fin.stop
+          end
+          #forward_thread.join  # TODO this thread won't stop because parent doesn't close pipe
+          fin.wait
+
+          on_exit_process(i)
+          exit! 0
+        ensure
+          $log.error "unknown error while shutting down this child process", :error=>$!.to_s, :pid=>Process.pid
+          $log.error_backtrace
+        end
+
+        exit! 1
+      end
+
+      # parent process
+      # override shutdown method to kill child processes
+      define_singleton_method(:shutdown) do
+        children.each {|pair|
+          begin
+            pid = pair[0]
+            forward_thread = pair[1]
+            if pid
+              Process.kill(:TERM, pid)
+              forward_thread.join   # wait until child closes pipe
+              Process.waitpid(pid)
+              pair[0] = nil
+            end
+          rescue
+            $log.error "unknown error while shutting down remote child process", :error=>$!.to_s
+            $log.error_backtrace
+          end
+        }
+      end
+
+      # override target.emit and write event stream to the pipe
+      forwarders = children.map {|pair| pair[1].forwarder }
+      if forwarders.length > 1
+        # use roundrobin
+        fwd = DetachProcessManager::MultiForwarder.new(forwarders)
+      else
+        fwd = forwarders[0]
+      end
+      define_singleton_method(:emit) do |tag,es,chain|
+        chain.next
+        fwd.emit(tag, es)
+      end
+    end
+  end
+end

data/lib/fluent_plugin_kinesis/version.rb

@@ -0,0 +1,17 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module FluentPluginKinesis
+  VERSION = '1.0.0'
+end

data/lib/kinesis_producer.rb

@@ -0,0 +1,24 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'protobuf'
+require 'kinesis_producer/binary'
+require 'kinesis_producer/library'
+require 'kinesis_producer/daemon'
+require 'kinesis_producer/protobuf/config.pb'
+require 'kinesis_producer/protobuf/messages.pb'
+
+module KinesisProducer
+  ConfigurationFields = Protobuf::Configuration.all_fields.reject(&:repeated?)
+end

data/lib/kinesis_producer/binary.rb

@@ -0,0 +1,10 @@
+module KinesisProducer
+  class Binary
+    Dir = 'amazon-kinesis-producer-native-binaries'
+    Files = {
+      'linux'   => File.join(Dir, 'linux',   'kinesis_producer'),
+      'osx'     => File.join(Dir, 'osx',     'kinesis_producer'),
+      'windows' => File.join(Dir, 'windows', 'kinesis_producer.exe'),
+    }
+  end
+end

data/lib/kinesis_producer/daemon.rb

@@ -0,0 +1,238 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'tempfile'
+require 'concurrent'
+
+module KinesisProducer
+  class Daemon
+    FixnumMax = (2 ** (64 - 2)) - 1
+
+    def initialize(binary, handler, options)
+      @binary  = binary
+      @handler = handler
+
+      @configuration             = options[:configuration] || {}
+      @credentials               = options[:credentials]
+      @metrics_credentials       = options[:metrics_credentials]
+      @credentials_refresh_delay = options[:credentials_refresh_delay] || 5000
+      @logger                    = options[:logger]
+      @debug                     = options[:debug]
+
+      @executor = Concurrent::CachedThreadPool.new
+      @shutdown = Concurrent::AtomicBoolean.new(false)
+      @outgoing_messages = Queue.new
+      @incoming_messages = Queue.new
+
+      if debug?
+        @meters = {
+          add_message:     Meter.new,
+          send_message:    Meter.new,
+          receive_message: Meter.new,
+          return_message:  Meter.new,
+        }
+      end
+    end
+
+    def start
+      @executor.post do
+        create_pipes
+        start_child
+      end
+    end
+
+    def destroy
+      @shutdown.make_true
+      if @pid
+        Process.kill("TERM", @pid)
+        Process.waitpid(@pid)
+        sleep 1 # TODO
+      end
+      delete_pipes
+    end
+
+    def add(message)
+      @outgoing_messages.push(message)
+      @meters[:add_message].mark if debug?
+    end
+
+    private
+
+    def create_pipes
+      @in_pipe  = temp_pathname('amz-aws-kpl-in-pipe-')
+      @out_pipe = temp_pathname('amz-aws-kpl-out-pipe-')
+      system("mkfifo", @in_pipe.to_path, @out_pipe.to_path)
+      sleep 1 # TODO
+    end
+
+    def delete_pipes
+      @in_channel.close  unless @in_channel.nil?
+      @out_channel.close unless @out_channel.nil?
+      @in_pipe.unlink
+      @out_pipe.unlink
+    rescue Errno::ENOENT
+    end
+
+    def temp_pathname(basename)
+      tempfile = Tempfile.new(basename)
+      ObjectSpace.undefine_finalizer(tempfile)
+      file = tempfile.path
+      File.delete(file)
+      Pathname.new(file)
+    end
+
+    def start_child
+      start_child_daemon
+      connect_to_child
+      start_loops
+    end
+
+    def start_child_daemon
+      @pid = Process.fork do
+        Process.setsid
+        configuration = make_configuration_message
+        credentials = make_set_credentials_message
+        command = [@binary, @out_pipe.to_path, @in_pipe.to_path, to_hex(configuration), to_hex(credentials)]
+        if @metrics_credentials
+          metrics_credentials = make_set_metrics_credentials_message
+          command.push(to_hex(metrics_credentials))
+        end
+        exec(*command)
+      end
+      sleep 1 # TODO
+    end
+
+    def connect_to_child
+      @in_channel  = @in_pipe.open('r')
+      @out_channel = @out_pipe.open('w')
+    end
+
+    def start_loops
+      start_loop_for(:send_message)
+      start_loop_for(:receive_message)
+      start_loop_for(:return_message)
+      start_loop_for(:update_credentials)
+      start_loop_for(:tick) if debug?
+    end
+
+    def start_loop_for(method)
+      @executor.post do
+        while @shutdown.false?
+          send(method)
+          @meters[method].mark if debug? and @meters.include?(method)
+        end
+      end
+    end
+
+    def send_message
+      message = @outgoing_messages.pop
+      size = [message.size].pack('N*')
+      @out_channel.write(size)
+      @out_channel.write(message)
+      @out_channel.flush
+    end
+
+    def receive_message
+      size = @in_channel.read(4)
+      data = @in_channel.read(size.unpack('N*').first)
+      @incoming_messages.push(data)
+    end
+
+    def return_message
+      data = @incoming_messages.pop
+      message = KinesisProducer::Protobuf::Message.decode(data)
+      @handler.on_message(message)
+    end
+
+    def update_credentials
+      add(make_set_credentials_message)
+      add(make_set_metrics_credentials_message) if @metrics_credentials
+      sleep @credentials_refresh_delay
+    end
+
+    def make_configuration_message
+      configuration = @configuration
+      KinesisProducer::ConfigurationFields.each do |field|
+        if configuration[field.name].nil?
+          configuration[field.name] = field.default_value
+        end
+      end
+      config = KinesisProducer::Protobuf::Configuration.new(configuration)
+      make_message(0, :configuration, config)
+    end
+
+    def make_set_credentials_message
+      make_set_credential_message(@credentials)
+    end
+
+    def make_set_metrics_credentials_message
+      make_set_credential_message(@metrics_credentials, true)
+    end
+
+    def make_set_credential_message(credentials, for_metrics = false)
+      return nil if credentials.nil?
+      cred = KinesisProducer::Protobuf::Credentials.new(
+        akid:       credentials.access_key_id,
+        secret_key: credentials.secret_access_key,
+        token:      credentials.session_token
+      )
+      set_credentials = KinesisProducer::Protobuf::SetCredentials.new(credentials: cred, for_metrics: for_metrics)
+      make_message(FixnumMax, :set_credentials, set_credentials)
+    end
+
+    def make_message(id, target, value)
+      KinesisProducer::Protobuf::Message.new(id: id, target => value).encode
+    end
+
+    def to_hex(message)
+      message.unpack('H*').first
+    end
+
+    def tick
+      out = @meters.each_value.map do |meter|
+        sprintf("%5d", meter.tick)
+      end
+      @logger.debug("[#{Thread.current.object_id}] "+out.join(" ")) if debug?
+      sleep 1
+    end
+
+    def debug?
+      @debug
+    end
+
+    class Meter
+      def initialize
+        @count = Concurrent::AtomicFixnum.new(0)
+        @previous_tick_time = Time.now.to_f
+        @current_rate = 0.0
+        tick
+      end
+
+      def mark(count = 1)
+        @count.increment(count)
+      end
+
+      def tick
+        @current_rate = @count.value.to_f / (Time.now.to_f - @previous_tick_time)
+        @count.value = 0
+        @previous_tick_time = Time.now.to_f
+        current_rate
+      end
+
+      def current_rate
+        @current_rate
+      end
+    end
+  end
+end

data/lib/kinesis_producer/library.rb

@@ -0,0 +1,122 @@
+#
+#  Copyright 2014-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'concurrent'
+require 'os'
+
+module KinesisProducer
+  class Library
+    class Handler
+      def initialize(futures)
+        @futures = futures
+      end
+
+      def on_message(message)
+        if !message.put_record_result.nil?
+          on_put_record_result(message)
+        elsif !message.metrics_response.nil?
+          on_metrics_response(message)
+        end
+      end
+
+      def on_put_record_result(message)
+        source_id = message.source_id
+        result = message.put_record_result
+        f = @futures.fetch(source_id)
+        @futures.delete(source_id)
+        if result.success
+          f.set(result)
+        else
+          f.fail(StandardError.new(result.to_hash))
+        end
+      end
+
+      def on_metrics_response(message)
+        source_id = message.source_id
+        response = message.metrics_response
+        f = @futures.fetch(source_id)
+        @futures.delete(source_id)
+        f.set(response.metrics)
+      end
+    end
+
+    class << self
+      def binary
+        case
+        when OS.linux?;   Binary::Files['linux']
+        when OS.osx?;     Binary::Files['osx']
+        when OS.windows?; Binary::Files['windows']
+        else; raise
+        end
+      end
+
+      def default_binary_path
+        root_dir = File.expand_path('../../..', __FILE__)
+        File.join(root_dir, binary)
+      end
+    end
+
+    def initialize(options)
+      @binary_path = options.delete(:binary_path) || self.class.default_binary_path
+      @message_id = Concurrent::AtomicFixnum.new(1)
+      @futures = Concurrent::Map.new
+      @child = Daemon.new(@binary_path, Handler.new(@futures), options)
+      @child.start
+    end
+
+    def destroy
+      flush_sync
+      @child.destroy
+    end
+
+    def put_record(options)
+      f = Concurrent::IVar.new
+      id = add_message(:put_record, KinesisProducer::Protobuf::PutRecord.new(options))
+      @futures[id] = f
+      f
+    end
+
+    def get_metrics(options = {})
+      f = Concurrent::IVar.new
+      id = add_message(:metrics_request, KinesisProducer::Protobuf::MetricsRequest.new(options))
+      @futures[id] = f
+      f.wait
+      f.value
+    end
+
+    def flush(options = {})
+      add_message(:flush, KinesisProducer::Protobuf::Flush.new(options))
+    end
+
+    def flush_sync
+      while @futures.size > 0
+        flush()
+        sleep 0.5
+      end
+    end
+
+    private
+
+    def add_message(target, value)
+      id = get_and_inc_message_id
+      message = KinesisProducer::Protobuf::Message.new(id: id, target => value)
+      @child.add(message.encode)
+      id
+    end
+
+    def get_and_inc_message_id
+      @message_id.increment
+    end
+  end
+end