fluent-plugin-kinesis 1.3.0 → 2.0.0

data/lib/kinesis_producer.rb

@@ -1,24 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-# 
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-# 
-#  http://aws.amazon.com/asl/
-# 
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-require 'protobuf'
-require 'kinesis_producer/binary'
-require 'kinesis_producer/library'
-require 'kinesis_producer/daemon'
-require 'kinesis_producer/protobuf/config.pb'
-require 'kinesis_producer/protobuf/messages.pb'
-
-module KinesisProducer
-  ConfigurationFields = Protobuf::Configuration.all_fields.reject(&:repeated?)
-end

data/lib/kinesis_producer/binary.rb

@@ -1,10 +0,0 @@
-module KinesisProducer
-  class Binary
-    Dir = 'amazon-kinesis-producer-native-binaries'
-    Files = {
-      'linux'   => File.join(Dir, 'linux',   'kinesis_producer'),
-      'osx'     => File.join(Dir, 'osx',     'kinesis_producer'),
-      'windows' => File.join(Dir, 'windows', 'kinesis_producer.exe'),
-    }
-  end
-end

data/lib/kinesis_producer/daemon.rb

@@ -1,270 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-require 'tempfile'
-require 'concurrent'
-
-module KinesisProducer
-  class Daemon
-    FixnumMax = (2 ** (64 - 2)) - 1
-
-    def initialize(binary, handler, options)
-      @binary  = binary
-      @handler = handler
-
-      @configuration             = options[:configuration] || {}
-      @credentials               = options[:credentials]
-      @metrics_credentials       = options[:metrics_credentials]
-      @credentials_refresh_delay = options[:credentials_refresh_delay] || 5000
-      @logger                    = options[:logger]
-      @debug                     = options[:debug]
-      @enable_kpl_logging        = options[:enable_kpl_logging]
-
-      @executor = Concurrent::CachedThreadPool.new
-      @shutdown = Concurrent::AtomicBoolean.new(false)
-      @outgoing_messages = Queue.new
-      @incoming_messages = Queue.new
-
-      if debug?
-        @meters = {
-          add_message:     Meter.new,
-          send_message:    Meter.new,
-          receive_message: Meter.new,
-          return_message:  Meter.new,
-        }
-      end
-    end
-
-    def start
-      @executor.post do
-        create_pipes
-        start_child
-      end
-    end
-
-    def destroy
-      @shutdown.make_true
-      if @pid
-        Process.kill("TERM", @pid)
-        Process.waitpid(@pid)
-        sleep 1 # TODO
-      end
-      delete_pipes
-    end
-
-    def add(message)
-      @outgoing_messages.push(message)
-      @meters[:add_message].mark if debug?
-    end
-
-    private
-
-    def create_pipes
-      @in_pipe  = temp_pathname('amz-aws-kpl-in-pipe-')
-      @out_pipe = temp_pathname('amz-aws-kpl-out-pipe-')
-      system("mkfifo", @in_pipe.to_path, @out_pipe.to_path)
-      sleep 1 # TODO
-    end
-
-    def delete_pipes
-      @in_channel.close  unless @in_channel.nil?
-      @out_channel.close unless @out_channel.nil?
-      @in_pipe.unlink
-      @out_pipe.unlink
-    rescue Errno::ENOENT
-    end
-
-    def temp_pathname(basename)
-      tempfile = Tempfile.new(basename)
-      ObjectSpace.undefine_finalizer(tempfile)
-      file = tempfile.path
-      File.delete(file)
-      Pathname.new(file)
-    end
-
-    def start_child
-      start_child_daemon
-      connect_to_child
-      start_loops
-    end
-
-    def start_child_daemon
-      @child_stdout, stdout = IO.pipe
-      @child_stderr, stderr = IO.pipe
-      @pid = Process.fork do
-        Process.setsid
-        STDOUT.sync = true
-        $stdout.reopen stdout
-        $stderr.reopen stderr
-        @child_stdout.close
-        @child_stderr.close
-        configuration = make_configuration_message
-        credentials = make_set_credentials_message
-        command = [@binary, @out_pipe.to_path, @in_pipe.to_path, to_hex(configuration), to_hex(credentials)]
-        if @metrics_credentials
-          metrics_credentials = make_set_metrics_credentials_message
-          command.push(to_hex(metrics_credentials))
-        end
-        exec(*command)
-      end
-      stdout.close
-      stderr.close
-      sleep 1 # TODO
-    end
-
-    def connect_to_child
-      @in_channel  = @in_pipe.open('r')
-      @out_channel = @out_pipe.open('w')
-    end
-
-    def enable_kpl_logging?
-      @enable_kpl_logging
-    end
-
-    def start_loops
-      start_loop_for(:log_stdout) if enable_kpl_logging?
-      start_loop_for(:log_stderr) if enable_kpl_logging?
-      start_loop_for(:send_message)
-      start_loop_for(:receive_message)
-      start_loop_for(:return_message)
-      start_loop_for(:update_credentials)
-      start_loop_for(:tick) if debug?
-    end
-
-    def start_loop_for(method)
-      @executor.post do
-        while @shutdown.false?
-          send(method)
-          @meters[method].mark if debug? and @meters.include?(method)
-        end
-      end
-    end
-
-    def log_stdout
-      line = @child_stdout.readline.chomp
-      @logger.info(line)
-    end
-
-    def log_stderr
-      line = @child_stderr.readline.chomp
-      if line.include? ' [info] '
-        @logger.info(line)
-      elsif line.include? ' [warn] '
-        @logger.warn(line)
-      else
-        @logger.error(line)
-      end
-    end
-
-    def send_message
-      message = @outgoing_messages.pop
-      size = [message.size].pack('N*')
-      @out_channel.write(size)
-      @out_channel.write(message)
-      @out_channel.flush
-    end
-
-    def receive_message
-      size = @in_channel.read(4)
-      data = @in_channel.read(size.unpack('N*').first)
-      @incoming_messages.push(data)
-    end
-
-    def return_message
-      data = @incoming_messages.pop
-      message = KinesisProducer::Protobuf::Message.decode(data)
-      @handler.on_message(message)
-    end
-
-    def update_credentials
-      add(make_set_credentials_message)
-      add(make_set_metrics_credentials_message) if @metrics_credentials
-      sleep(@credentials_refresh_delay.to_f/1000)
-    end
-
-    def make_configuration_message
-      configuration = @configuration
-      KinesisProducer::ConfigurationFields.each do |field|
-        if configuration[field.name].nil?
-          configuration[field.name] = field.default_value
-        end
-      end
-      config = KinesisProducer::Protobuf::Configuration.new(configuration)
-      make_message(0, :configuration, config)
-    end
-
-    def make_set_credentials_message
-      make_set_credential_message(@credentials)
-    end
-
-    def make_set_metrics_credentials_message
-      make_set_credential_message(@metrics_credentials, true)
-    end
-
-    def make_set_credential_message(credentials, for_metrics = false)
-      return nil if credentials.nil?
-      cred = KinesisProducer::Protobuf::Credentials.new(
-        akid:       credentials.credentials.access_key_id,
-        secret_key: credentials.credentials.secret_access_key,
-        token:      credentials.credentials.session_token
-      )
-      set_credentials = KinesisProducer::Protobuf::SetCredentials.new(credentials: cred, for_metrics: for_metrics)
-      make_message(FixnumMax, :set_credentials, set_credentials)
-    end
-
-    def make_message(id, target, value)
-      KinesisProducer::Protobuf::Message.new(id: id, target => value).encode
-    end
-
-    def to_hex(message)
-      message.unpack('H*').first
-    end
-
-    def tick
-      out = @meters.each_value.map do |meter|
-        sprintf("%5d", meter.tick)
-      end
-      @logger.debug("[#{Thread.current.object_id}] "+out.join(" ")) if debug?
-      sleep 1
-    end
-
-    def debug?
-      @debug
-    end
-
-    class Meter
-      def initialize
-        @count = Concurrent::AtomicFixnum.new(0)
-        @previous_tick_time = Time.now.to_f
-        @current_rate = 0.0
-        tick
-      end
-
-      def mark(count = 1)
-        @count.increment(count)
-      end
-
-      def tick
-        @current_rate = @count.value.to_f / (Time.now.to_f - @previous_tick_time)
-        @count.value = 0
-        @previous_tick_time = Time.now.to_f
-        current_rate
-      end
-
-      def current_rate
-        @current_rate
-      end
-    end
-  end
-end

data/lib/kinesis_producer/library.rb

@@ -1,122 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-require 'concurrent'
-require 'os'
-
-module KinesisProducer
-  class Library
-    class Handler
-      def initialize(futures)
-        @futures = futures
-      end
-
-      def on_message(message)
-        if !message.put_record_result.nil?
-          on_put_record_result(message)
-        elsif !message.metrics_response.nil?
-          on_metrics_response(message)
-        end
-      end
-
-      def on_put_record_result(message)
-        source_id = message.source_id
-        result = message.put_record_result
-        f = @futures.fetch(source_id)
-        @futures.delete(source_id)
-        if result.success
-          f.set(result)
-        else
-          f.fail(StandardError.new(result.to_hash))
-        end
-      end
-
-      def on_metrics_response(message)
-        source_id = message.source_id
-        response = message.metrics_response
-        f = @futures.fetch(source_id)
-        @futures.delete(source_id)
-        f.set(response.metrics)
-      end
-    end
-
-    class << self
-      def binary
-        case
-        when OS.linux?;   Binary::Files['linux']
-        when OS.osx?;     Binary::Files['osx']
-        when OS.windows?; Binary::Files['windows']
-        else; raise
-        end
-      end
-
-      def default_binary_path
-        root_dir = File.expand_path('../../..', __FILE__)
-        File.join(root_dir, binary)
-      end
-    end
-
-    def initialize(options)
-      @binary_path = options.delete(:binary_path) || self.class.default_binary_path
-      @message_id = Concurrent::AtomicFixnum.new(1)
-      @futures = Concurrent::Map.new
-      @child = Daemon.new(@binary_path, Handler.new(@futures), options)
-      @child.start
-    end
-
-    def destroy
-      flush_sync
-      @child.destroy
-    end
-
-    def put_record(options)
-      f = Concurrent::IVar.new
-      id = add_message(:put_record, KinesisProducer::Protobuf::PutRecord.new(options))
-      @futures[id] = f
-      f
-    end
-
-    def get_metrics(options = {})
-      f = Concurrent::IVar.new
-      id = add_message(:metrics_request, KinesisProducer::Protobuf::MetricsRequest.new(options))
-      @futures[id] = f
-      f.wait
-      f.value
-    end
-
-    def flush(options = {})
-      add_message(:flush, KinesisProducer::Protobuf::Flush.new(options))
-    end
-
-    def flush_sync
-      while @futures.size > 0
-        flush()
-        sleep 0.5
-      end
-    end
-
-    private
-
-    def add_message(target, value)
-      id = get_and_inc_message_id
-      message = KinesisProducer::Protobuf::Message.new(id: id, target => value)
-      @child.add(message.encode)
-      id
-    end
-
-    def get_and_inc_message_id
-      @message_id.increment
-    end
-  end
-end

data/lib/kinesis_producer/protobuf/config.pb.rb

@@ -1,66 +0,0 @@
-#
-#  Copyright 2014-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
-#
-#  Licensed under the Amazon Software License (the "License").
-#  You may not use this file except in compliance with the License.
-#  A copy of the License is located at
-#
-#  http://aws.amazon.com/asl/
-#
-#  or in the "license" file accompanying this file. This file is distributed
-#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
-#  express or implied. See the License for the specific language governing
-#  permissions and limitations under the License.
-
-##
-# This file is auto-generated. DO NOT EDIT!
-#
-require 'protobuf/message'
-
-module KinesisProducer
-  module Protobuf
-    ##
-    # Message Classes
-    #
-    class AdditionalDimension < ::Protobuf::Message; end
-    class Configuration < ::Protobuf::Message; end
-
-
-    ##
-    # Message Fields
-    #
-    class AdditionalDimension
-      required :string, :key, 1
-      required :string, :value, 2
-      required :string, :granularity, 3
-    end
-
-    class Configuration
-      repeated ::KinesisProducer::Protobuf::AdditionalDimension, :additional_metric_dims, 128
-      optional :bool, :aggregation_enabled, 1, :default => true
-      optional :uint64, :aggregation_max_count, 2, :default => 4294967295
-      optional :uint64, :aggregation_max_size, 3, :default => 51200
-      optional :uint64, :collection_max_count, 4, :default => 500
-      optional :uint64, :collection_max_size, 5, :default => 5242880
-      optional :uint64, :connect_timeout, 6, :default => 6000
-      optional :string, :custom_endpoint, 7
-      optional :bool, :fail_if_throttled, 8, :default => false
-      optional :string, :log_level, 9, :default => "info"
-      optional :uint64, :max_connections, 10, :default => 24
-      optional :string, :metrics_granularity, 11, :default => "shard"
-      optional :string, :metrics_level, 12, :default => "detailed"
-      optional :string, :metrics_namespace, 13, :default => "KinesisProducerLibrary"
-      optional :uint64, :metrics_upload_delay, 14, :default => 60000
-      optional :uint64, :min_connections, 15, :default => 1
-      optional :uint64, :port, 16, :default => 443
-      optional :uint64, :rate_limit, 17, :default => 150
-      optional :uint64, :record_max_buffered_time, 18, :default => 100
-      optional :uint64, :record_ttl, 19, :default => 30000
-      optional :string, :region, 20
-      optional :uint64, :request_timeout, 21, :default => 6000
-      optional :bool, :verify_certificate, 22, :default => true
-    end
-
-  end
-end
-