amazon-kinesis-client-ruby 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 36265265083e58844138d431b5d5e8d0d7828008
+  data.tar.gz: b479c46db0375672f956e94f3b77e232c0f33db0
+SHA512:
+  metadata.gz: c2888132cf9830a89a5217d2878aaa12e6d899a75fb48cf4c703292eb2b7257115e257e7729343de5d2c065e5cc718e2601869e26d16cc8d086f33c6844d974a
+  data.tar.gz: 26c9dfa9195f7c570777c92b5c4b4506b507a450e1c9d49c14a33b50db10d6a993f196f677430a9cc05fe86807683f2712f028e49973f3b07625523cf299d001

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+.ruby-version

data/.rubocop.yml

@@ -0,0 +1,26 @@
+AllCops:
+  Include:
+    - '**/Rakefile'
+    - '**/*.gemspec'
+
+Style/Documentation:
+  Enabled: false
+
+Style/Blocks:
+  Enabled: true
+  Exclude:
+    - spec/**/*_spec.rb
+
+Style/WordArray:
+  Enabled: true
+  Exclude:
+    - spec/**/*_spec.rb
+
+Style/Encoding:
+  Enabled: false
+
+Style/MethodDefParentheses:
+  Enabled: false
+
+Metrics/LineLength:
+  Max: 120

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in amazon-kinesis-client-ruby.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 soloman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,226 @@
+Amazon Kinesis Client Library for Ruby
+======================================
+
+This gem provides an interface to the KCL MultiLangDaemon, which is part of the [Amazon Kinesis Client Library](https://github.com/awslabs/amazon-kinesis-client). This interface manages the interaction with the MultiLangDaemon so that developers can focus on implementing their record processor executable. A record processor executable typically looks something like:
+
+```ruby
+class SimpleProcessor
+  include RecordProcessor
+
+  def process_records records, checkpointer
+    # process records and checkpoint
+  end
+end
+```
+
+Note, the initial implementation of this gem is largely based on the reference [python implementation](https://github.com/awslabs/amazon-kinesis-client-python) provided by Amazon.
+
+
+Environment Setup
+-----------------
+
+Please ensure the following environment requirements are reviewed before using the gem:
+- make sure that your environment is configured to allow the Amazon Kinesis Client Library to use your [AWS Security Credentials](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html). By default the *DefaultAWSCredentialsProviderChain* is configured so you'll want to make your credentials available to one of the credentials providers in that provider chain. There are several ways to do this such as providing a ~/.aws/credentials file, or specifying the *AWS_ACCESS_KEY_ID* and
+*AWS_SECRET_ACCESS_KEY* environment variables.
+- ensure **JAVA** is available in the environment. This gem works by invoking the packaged *amazon-kinesis-client.jar* and which subsequently executes the target ruby record processor, therefore a compatible JVM/JDK is therefore required.
+
+
+Environment Variables
+---------------------
+- **AWS_ACCESS_KEY_ID** : AWS credential for accessing the target kinesis queue
+- **AWS_SECRET_ACCESS_KEY** : AWS credential for accessing the target kinesis queue
+- **APP_NAME** : Used by the KCL as the name of this application. It is used as the DynamoDB table name created by KCL to store checkpoints.
+- **PATH_TO_JAVA** : (optional) custom java executable path (by default `which java` is used).
+
+
+Example Consumer Client Setup
+-----------------------------
+
+Firstly please create the ruby script to run your kinesis consumer with structure similar to the following:
+
+```ruby
+# FILE_NAME: run_simple_kinesis_client.rb
+
+require 'kcl'
+
+# define a record processor
+class SimpleProcessor < Kcl::AdvancedRecordProcessor
+  def process_record data
+    p data
+  end
+end
+
+# config the executor
+Kcl::Executor.new do |executor|
+  executor.config stream_name: 'data-kinesis-queue',
+                  application_name: 'RubyKCLSample',
+                  max_records: 5,
+                  idle_time_between_reads_in_millis: 500
+
+  # setup the target record processor
+  executor.record_processor do
+    SimpleProcessor.new
+  end
+end
+
+# execute and run
+Kcl::Executor.run
+```
+
+The most essential part of this is the `Kcl::Executor.run` bit, which is required in the script that you want the consumer client to run. The configuration (i.e. `Kcl::Executor.new` bit) and record processor class (i.e. `SimpleProcessor`) can be put in other suitable places.
+
+Next, run the script with an additional argument `exec`, e.g. `ruby run_simple_kinesis_client.rb exec`. Please note, it will **not** work without the `exec` argument, because the script is intent to be invoked by the amazon-kinesis-client java process. Specifying `exec` actually triggers the java consumer process.
+
+The following shows an example of how the consumer worker can be specified in the Procfile:
+
+```bash
+worker: bundle exec <your_consumer_client_script> exec
+```
+
+
+Configurations
+--------------
+
+The properties required by the MultiLangDaemon (please refer to [**this**](https://github.com/awslabs/amazon-kinesis-client-python/blob/master/samples/sample.properties)) can be configured through the `executor.config`. That is:
+
+
+```ruby
+Kcl::Executor.new do |executor|
+  executor.config stream_name: 'data-kinesis-queue',
+                  application_name: 'RubyKCLSample',
+                  max_records: 5,
+                  idle_time_between_reads_in_millis: 500,
+                  region_name: 'us-east-1',
+                  initial_position_in_stream: 'TRIM_HORIZON'
+
+  #.....
+end
+```
+
+Under the hood, the Kcl gem will translate it to the proper java properties file for the java process. Please try to use underscore key name (i.e. `stream_name` for `streamName`), so it follows good ruby convention.
+
+Please ensure the following configuration values are specified:
+- **stream_name** : the target kinesis queue name
+- **application_name** : it is not required if the environment variable **APP_NAME** is set.
+
+
+Record Processors
+-----------------
+
+Please also specify the record processor for the `Kcl::Executor`, i.e.
+
+```ruby
+Kcl::Executor.new do |executor|
+  #.......
+  executor.record_processor do
+    YourProcessor.new
+  end
+end
+```
+
+The reason that why `SimpleProcessor.new` is initialised in the block instead of:
+
+```ruby
+executor.record_processor SimpleProcessor.new
+```
+
+is that processor should only get instantiated when invoked by the consumer client java process, and not in the first `<client_script> exec` call.
+
+
+### Kcl::RecordProcessor
+
+The RecordProcessor module offers the most basic interface to implement a record processor. The following shows a simple example:
+
+```ruby
+require 'kcl'
+
+class YourProcessor
+  include Kcl::RecordProcessor
+
+  def init shared_id
+    # Called once by a KCLProcess before any calls to process_records
+  end
+
+  def process_records records, checkpointer
+    # Called by a KCLProcess with a list of records to be processed and a
+    # checkpointer which accepts sequence numbers from the records to indicate
+    # where in the stream to checkpoint.
+  end
+
+  def shutdown checkpointer, reason
+    #Called by a KCLProcess instance to indicate that this record processor
+    # should shutdown. After this is called, there will be no more calls to
+    # any other methods of this record processor.
+  end
+end
+```
+
+Please note, with the basic `Kcl::RecordProcessor`, it is the client's responsibility to manage the checkpoints. The client are free to decide how often the checkpoint should be made through doing:
+
+```ruby
+def process_records records, checkpointer
+  checkpointer.checkpoint records.last['sequenceNumber']
+end
+```
+
+### Kcl::AdvancedRecordProcessor
+
+The AdvancedRecordProcessor class take cares the basic checkpoint logic, and the clients only required to implement the `process_record` method, for example:
+
+```ruby
+require 'kcl'
+
+class YourProcessor < Kcl::AdvancedRecordProcessor
+  def initialize
+    super sleep_seconds: 10, # default to 5
+          checkpoint_retries: 10, # default to 5
+          checkpoint_freq_seconds: 30 # default to 60
+  end
+
+  def process_record record
+    data = record[:data]
+    partition_key = record[:partition_key]
+    sequence_number = record[:sequence_number]
+
+    # do something with data
+  end
+end
+
+```
+
+
+Downloading
+-----------
+install stable releases with the following command:
+
+```bash
+gem install amazon-kinesis-client-ruby
+```
+
+The development version (hosted on Github) can be installed with:
+
+```bash
+git clone git@github.com:everydayhero/amazon-kinesis-client-ruby.git
+cd amazon-kinesis-client-ruby
+rake install
+```
+
+###Run Tests
+```bash
+rake spec
+```
+
+
+Future Roadmap
+--------------
+- dependency management for the Amazon kinesis client jar files by utilising [ruby-maven](https://github.com/mkristian/ruby-maven) (potentially).
+
+
+Contributing
+------------
+
+1. Fork it
+2. Create your feature branch (git checkout -b my-new-feature)
+3. Commit your changes (git commit -am 'Add some feature')
+4. Push to the branch (git push origin my-new-feature)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new :spec do |spec|
+  spec.rspec_opts = '--format documentation --color'
+end
+
+RuboCop::RakeTask.new

data/amazon-kinesis-client-ruby.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kcl/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'amazon-kinesis-client-ruby'
+  spec.version       = Kcl::VERSION
+  spec.authors       = ['Soloman Weng']
+  spec.email         = ['solomanw@everydayhero.com.au']
+  spec.summary       = 'Amazon Kinesis Client Library for Ruby'
+  spec.description   = 'Amazon Kinesis Client Library for Ruby'
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(/^(test|spec|features)\//)
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency 'activesupport', '~> 4.1.8'
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.1.0'
+  spec.add_development_dependency 'rubocop', '~> 0.27.1'
+
+  spec.required_ruby_version = '~> 2.0'
+end

data/lib/kcl.rb

@@ -0,0 +1,12 @@
+require 'kcl/version'
+require 'kcl/action_handler'
+require 'kcl/record_processor'
+require 'kcl/advanced_record_processor'
+require 'kcl/checkpoint_error'
+require 'kcl/checkpointer'
+require 'kcl/executor_command_builder'
+require 'kcl/io_handler'
+require 'kcl/malformed_action_error'
+require 'kcl/process'
+require 'kcl/configuration'
+require 'kcl/executor'

data/lib/kcl/action_handler.rb

@@ -0,0 +1,34 @@
+module Kcl
+  class ActionHandler
+    def initialize record_processor, checkpointer, io_handler
+      @record_processor = record_processor
+      @checkpointer = checkpointer
+      @io_handler = io_handler
+    end
+
+    # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
+    def handle action
+      case action.fetch('action')
+      when 'initialize'
+        record_processor.init action.fetch('shardId')
+      when 'processRecords'
+        record_processor.process_records action.fetch('records'), checkpointer
+      when 'shutdown'
+        record_processor.shutdown checkpointer, action.fetch('reason')
+      else
+        fail MalformedActionError,
+             "Received an action which couldn't be understood. Action was #{action}"
+      end
+    rescue KeyError => key_error
+      raise MalformedActionError,
+            "Action #{action} was expected to have key: #{key_error.message}"
+    rescue => error
+      io_handler.write_error error.backtrace.join "\n"
+    end
+    # rubocop:enable Metrics/MethodLength,Metrics/AbcSize
+
+    private
+
+    attr_reader :record_processor, :io_handler, :checkpointer
+  end
+end

data/lib/kcl/advanced_record_processor.rb

@@ -0,0 +1,105 @@
+require 'base64'
+require 'logger'
+
+module Kcl
+  class AdvancedRecordProcessor
+    include RecordProcessor
+
+    LOG = Logger.new STDOUT
+    ERR_LOG = Logger.new STDERR
+
+    DEFAULT_SLEEP_SECONDS = 5
+    DEFAULT_CHECKPOINT_RETRIES = 5
+    DEFAULT_CHECKPOINT_FREQ_SECONDS = 60
+
+    def initialize sleep_seconds: DEFAULT_SLEEP_SECONDS,
+                   checkpoint_retries: DEFAULT_CHECKPOINT_RETRIES,
+                   checkpoint_freq_seconds: DEFAULT_CHECKPOINT_FREQ_SECONDS
+      @sleep_seconds = sleep_seconds
+      @checkpoint_retries = checkpoint_retries
+      @checkpoint_freq_seconds = checkpoint_freq_seconds
+    end
+
+    def process_record _record; end
+
+    def init _shared_id
+      self.largest_seq = nil
+      self.last_checkpoint_time = Time.now
+    end
+
+    def process_records records, checkpointer
+      records.each do |record|
+        handle_record record
+      end
+
+      if Time.now - last_checkpoint_time > checkpoint_freq_seconds
+        checkpoint checkpointer
+        self.last_checkpoint_time = Time.now
+      end
+    rescue => error
+      ERR_LOG.error "Encountered an exception while processing records. Exception was #{error}\n"
+    end
+
+    def shutdown checkpointer, reason
+      if reason == 'TERMINATE'
+        LOG.info 'Was told to terminate, will attempt to checkpoint.'
+        checkpoint checkpointer, sequence_number: nil
+      else
+        LOG.info 'Shutting down due to failover. Will not checkpoint.'
+      end
+    end
+
+    private
+
+    attr_accessor :largest_seq, :last_checkpoint_time
+
+    attr_reader :sleep_seconds, :checkpoint_retries, :checkpoint_freq_seconds
+
+    def handle_record record
+      data = Base64.decode64 record['data']
+      seq = record['sequenceNumber'].to_i
+      key = record['partitionKey']
+
+      process_record data: data, partition_key: key, sequence_number: seq
+
+      self.largest_seq = seq if largest_seq.nil? || seq > largest_seq
+    end
+
+    def checkpoint checkpointer, sequence_number: largest_seq
+      checkpoint_retries.times do |try_count|
+        break if try_checkpoint checkpointer, sequence_number, try_count
+
+        sleep sleep_seconds
+      end
+    end
+
+    # rubocop:disable Metrics/MethodLength,Metrics/CyclomaticComplexity
+    def try_checkpoint checkpointer, sequence_number, try_count
+      seq = sequence_number.to_s if sequence_number
+      checkpointer.checkpoint seq
+
+      true
+    rescue CheckpointError => checkpoint_error
+      case checkpoint_error.to_s
+      when 'ShutdownException'
+        LOG.info 'Encountered shutdown execption, skipping checkpoint'
+        true
+      when 'ThrottlingException'
+        if checkpoint_retries - 1 == try_count
+          ERR_LOG.error "Failed to checkpoint after #{try_count} attempts, giving up.\n"
+          true
+        else
+          LOG.info "Was throttled while checkpointing, will attempt again in #{sleep_seconds} seconds"
+          false
+        end
+      when 'InvalidStateException'
+        ERR_LOG.error "MultiLangDaemon reported an invalid state while checkpointing.\n"
+        false
+      else
+        ERR_LOG.error "Encountered an error while checkpointing, error was #{checkpoint_error}.\n"
+        false
+      end
+    end
+    # rubocop:enable Metrics/MethodLength,Metrics/CyclomaticComplexity
+  end
+end

data/lib/kcl/checkpoint_error.rb

@@ -0,0 +1,16 @@
+module Kcl
+  class CheckpointError < StandardError
+    def initialize error_name
+      super error_name
+      @error_name = error_name
+    end
+
+    def to_s
+      error_name.to_s
+    end
+
+    private
+
+    attr_reader :error_name
+  end
+end

data/lib/kcl/checkpointer.rb

@@ -0,0 +1,37 @@
+module Kcl
+  class Checkpointer
+    def initialize io_handler
+      @io_handler = io_handler
+    end
+
+    def checkpoint sequence_number = nil
+      io_handler.write_action action: 'checkpoint', checkpoint: sequence_number
+
+      action = fetch_action
+      if action['action'] == 'checkpoint'
+        fail CheckpointError, action['error'] unless action['error'].nil?
+      else
+        fail CheckpointError, 'InvalidStateException'
+      end
+    end
+
+    private
+
+    attr_reader :io_handler
+
+    def fetch_action
+      loop do
+        action = read_action
+
+        return action unless action.nil?
+      end
+    end
+
+    def read_action
+      io_handler.read_action
+    rescue IOHandler::ReadError => read_error
+      io_handler.write_error \
+        "Could not understand line read from input: #{read_error.line}"
+    end
+  end
+end

data/lib/kcl/configuration.rb

@@ -0,0 +1,66 @@
+require 'ostruct'
+require 'active_support/core_ext/string'
+
+module Kcl
+  class Configuration < OpenStruct
+    def initialize config = {}
+      super default_config.merge config
+    end
+
+    def to_properties
+      check_config
+
+      to_h.map do |key, value|
+        "#{make_prop_key key}=#{value}"
+      end.join "\n"
+    end
+
+    private
+
+    def check_config
+      required_propertie_keys.each do |required_key|
+        fail "#{required_key} is required" unless to_h[required_key].present?
+      end
+    end
+
+    def make_prop_key key
+      default_key_map.fetch key, key.to_s.camelize(:lower)
+    end
+
+    def application_name
+      ENV['APP_NAME']
+    end
+
+    def executable_name
+      caller.each do |trace|
+        matched = trace.match(/\A(?<file>.+)\:\d+\:in.*<main>.*\Z/)
+
+        return matched[:file] if matched
+      end
+    end
+
+    def processing_language
+      "ruby/#{RUBY_VERSION}"
+    end
+
+    def default_config
+      @default_config ||= {
+        executable_name: executable_name,
+        application_name: application_name,
+        processing_language: processing_language,
+        aws_credentials_provider: 'DefaultAWSCredentialsProviderChain',
+        initial_position_in_stream: 'TRIM_HORIZON'
+      }
+    end
+
+    def required_propertie_keys
+      default_config.keys.concat [:stream_name]
+    end
+
+    def default_key_map
+      {
+        aws_credentials_provider: 'AWSCredentialsProvider'
+      }
+    end
+  end
+end

data/lib/kcl/executor.rb

@@ -0,0 +1,75 @@
+require 'tempfile'
+
+module Kcl
+  class Executor
+    LOG = Logger.new STDOUT
+
+    def initialize
+      yield self
+
+      self.class.current_executor = self
+    end
+
+    def config configuration = nil
+      @configuration = Configuration.new configuration if configuration
+
+      @configuration
+    end
+
+    def record_processor record_processor = nil
+      @record_processor_callback =
+        if record_processor
+          proc { record_processor }
+        elsif block_given?
+          Proc.new
+        else
+          fail ArgumentError, 'RecordProcessor required'
+        end
+    end
+
+    def run argv
+      if argv[0] == 'exec'
+        run_exec
+      else
+        run_record_processor
+      end
+    end
+
+    private
+
+    attr_reader :record_processor_callback, :configuration
+
+    def run_exec
+      command = ExecutorCommandBuilder.new(config_properties_path).build
+      LOG.info "execute command:\n#{command.join ' '}"
+
+      system(*command)
+    end
+
+    def run_record_processor
+      processor_instance = record_processor_callback.call
+
+      processor_instance.run
+    end
+
+    def config_properties_path
+      config_properties_file = Tempfile.new ['config', '.properties']
+      config_properties_file.write configuration.to_properties
+      config_properties_file.close
+      LOG.info "properties path: #{config_properties_file.path}"
+      LOG.info "properties:\n#{File.read config_properties_file.path}"
+
+      config_properties_file.path
+    end
+
+    class << self
+      attr_accessor :current_executor
+
+      def run argv = ARGV
+        fail 'Executor not configured' unless current_executor
+
+        current_executor.run argv
+      end
+    end
+  end
+end

data/lib/kcl/executor_command_builder.rb

@@ -0,0 +1,40 @@
+module Kcl
+  class ExecutorCommandBuilder
+    def initialize properties_file_path
+      @properties_file_path = properties_file_path
+    end
+
+    def build
+      [java, '-cp', class_path, client_class, properties_file]
+    end
+
+    private
+
+    attr_reader :properties_file_path
+
+    def java
+      command = ENV.fetch('PATH_TO_JAVA', `which java`).strip
+      fail 'Missing JAVA PATH' if command.nil? || command.empty?
+
+      command
+    end
+
+    def client_class
+      'com.amazonaws.services.kinesis.multilang.MultiLangDaemon'
+    end
+
+    def class_path
+      jar_dir = File.expand_path '../../jars', __FILE__
+
+      (Dir["#{jar_dir}/*.jar"] << properties_file_dir).join ':'
+    end
+
+    def properties_file
+      @properties_file ||= File.basename properties_file_path
+    end
+
+    def properties_file_dir
+      @properties_file_dir ||= File.dirname properties_file_path
+    end
+  end
+end

data/lib/kcl/io_handler.rb

@@ -0,0 +1,48 @@
+require 'json'
+
+module Kcl
+  class IOHandler
+    def initialize input, output, error
+      @input = input
+      @output = output
+      @error = error
+    end
+
+    def write_action response
+      write_line response.to_json
+    end
+
+    def read_action
+      line = input.gets
+      JSON.parse line unless line.nil? || line.empty?
+    rescue => error
+      raise ReadError.new(error, line)
+    end
+
+    def write_error error_message
+      error << "#{error_message}\n"
+    ensure
+      error.flush
+    end
+
+    private
+
+    attr_reader :input, :output, :error
+
+    def write_line line
+      output << "\n#{line}\n"
+    ensure
+      output.flush
+    end
+
+    class ReadError < StandardError
+      attr_reader :base_error, :line
+
+      def initialize base_error, line
+        super base_error.message
+        @base_error = base_error
+        @line = line
+      end
+    end
+  end
+end

data/lib/kcl/malformed_action_error.rb

@@ -0,0 +1,3 @@
+module Kcl
+  class MalformedActionError < StandardError; end
+end

data/lib/kcl/process.rb

@@ -0,0 +1,39 @@
+module Kcl
+  class Process
+    def initialize record_processor,
+                   input: $stdin,
+                   output: $stdout,
+                   error: $stderr
+      @record_processor = record_processor
+      @io_handler = IOHandler.new input, output, error
+      @checkpointer = Checkpointer.new @io_handler
+    end
+
+    def run
+      loop do
+        action = io_handler.read_action
+        perform action
+        report_done action
+
+        break if action.nil?
+      end
+    end
+
+    private
+
+    attr_reader :record_processor, :io_handler, :checkpointer
+
+    def perform action
+      action_handler.handle action
+    end
+
+    def action_handler
+      @action_handler ||=
+        ActionHandler.new record_processor, checkpointer, io_handler
+    end
+
+    def report_done action
+      io_handler.write_action action: 'status', responseFor: action['action']
+    end
+  end
+end

data/lib/kcl/record_processor.rb

@@ -0,0 +1,13 @@
+module Kcl
+  module RecordProcessor
+    def init _shared_id; end
+
+    def process_records _records, _checkpointer; end
+
+    def shutdown _checkpointer, _reason; end
+
+    def run
+      Process.new(self).run
+    end
+  end
+end

data/lib/kcl/version.rb

@@ -0,0 +1,3 @@
+module Kcl
+  VERSION = '0.0.1'
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+describe Kcl::Configuration do
+  describe '#to_properties' do
+    let(:config) { Kcl::Configuration.new config_options }
+
+    let(:required_options) {
+      {
+        application_name: 'MyApp',
+        stream_name: 'MyStream'
+      }
+    }
+
+    let(:config_options) { required_options }
+
+    let(:default_properties) {
+      %w(
+        AWSCredentialsProvider=DefaultAWSCredentialsProviderChain
+        initialPositionInStream=TRIM_HORIZON
+      ).join "\n"
+    }
+
+    subject { config.to_properties }
+
+    it { is_expected.to match(%r{executableName=.+/exe/rspec}) }
+
+    it { is_expected.to include "processingLanguage=ruby/#{RUBY_VERSION}" }
+
+    it { is_expected.to include default_properties }
+
+    context 'When ruby style config key is set' do
+      let(:config_options) {
+        {
+          dummy_key: 1,
+          dummy_key_two: 'two'
+        }.merge required_options
+      }
+
+      it { is_expected.to include "dummyKey=1\ndummyKeyTwo=two" }
+    end
+
+    context 'When aws_credentials_provider is set' do
+      let(:config_options) {
+        {
+          aws_credentials_provider: 'Test'
+        }.merge required_options
+      }
+
+      it { is_expected.to include 'AWSCredentialsProvider=Test' }
+    end
+
+    context 'When APP_NAME environment vairlable is set' do
+      before { ENV['APP_NAME'] = 'Test App' }
+
+      let(:config_options) { { stream_name: 'MyStream' } }
+
+      it { is_expected.to include 'applicationName=Test App' }
+    end
+
+    %w(
+      executable_name application_name processing_language
+      aws_credentials_provider initial_position_in_stream stream_name
+    ).each do |key_prop|
+      context "When missing required property #{key_prop}" do
+        let(:config_options) { { key_prop => nil } }
+
+        it { expect { subject }.to raise_error "#{key_prop} is required" }
+      end
+    end
+  end
+end

data/spec/executor_command_builder_spec.rb

@@ -0,0 +1,62 @@
+require 'spec_helper'
+
+describe Kcl::ExecutorCommandBuilder do
+  let(:properties_file_path) { File.expand_path __FILE__ }
+  let(:builder) { Kcl::ExecutorCommandBuilder.new properties_file_path }
+
+  describe '#build' do
+    let(:command) { builder.build }
+
+    describe 'comamnd<java>' do
+      let(:java) { command[0] }
+
+      context 'With PATH_TO_JAVA environment variable' do
+        before { ENV['PATH_TO_JAVA'] = 'my_java' }
+
+        it { expect(java).to eq 'my_java' }
+      end
+
+      context 'Without PATH_TO_JAVA set' do
+        before { ENV['PATH_TO_JAVA'] = nil }
+
+        it { expect(java).to eq `which java`.strip }
+      end
+
+      context 'Without java executable available' do
+        before {
+          ENV['PATH_TO_JAVA'] = nil
+          allow(builder).to receive(:'`').with('which java').and_return ''
+        }
+
+        it { expect { java }.to raise_error 'Missing JAVA PATH' }
+      end
+    end
+
+    describe 'command<-cp>' do
+      it { expect(command[1]).to eq '-cp' }
+    end
+
+    describe 'command<classpath>' do
+      let(:classpath) { command[2] }
+
+      it { expect(classpath).to match(/\A(.+\.jar\:)+.+\z/) }
+
+      it { expect(classpath).to include File.dirname(properties_file_path) }
+    end
+
+    describe 'command<client_class>' do
+      let(:client_class) { command[3] }
+      let(:expected_client_class) {
+        'com.amazonaws.services.kinesis.multilang.MultiLangDaemon'
+      }
+
+      it { expect(client_class).to eq expected_client_class }
+    end
+
+    describe 'command<properties_file>' do
+      let(:properties_file) { command[4] }
+
+      it { expect(properties_file).to eq File.basename(properties_file_path) }
+    end
+  end
+end

data/spec/executor_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe Kcl::Executor do
+  describe '#initialize' do
+    context 'When no block provided' do
+      it { expect { Kcl::Executor.new }.to raise_error LocalJumpError  }
+    end
+  end
+
+  describe '.run' do
+    context 'Without initialize with #new' do
+      it { expect { Kcl::Executor.run }.to raise_error 'Executor not configured' }
+    end
+
+    context 'With executor initialized' do
+      before {
+        expect_any_instance_of(Kcl::Executor).to receive(:run)
+
+        Kcl::Executor.new {}
+      }
+
+      it { expect Kcl::Executor.run }
+    end
+  end
+end

data/spec/io_handler_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+require 'stringio'
+
+describe Kcl::IOHandler do
+  let(:input) { StringIO.new }
+  let(:output) { StringIO.new }
+  let(:error) { StringIO.new }
+
+  subject { Kcl::IOHandler.new input, output, error }
+
+  describe '#write_action' do
+    let(:action) { { action: 'test', value: 123 } }
+
+    before { subject.write_action action }
+
+    it { expect(output.string).to eq "\n#{action.to_json}\n" }
+
+    it { expect(input.string).to eq '' }
+
+    it { expect(error.string).to eq '' }
+  end
+
+  describe '#write_error' do
+    let(:message) { 'Some error' }
+
+    before { subject.write_error message }
+
+    it { expect(error.string).to eq "#{message}\n" }
+
+    it { expect(input.string).to eq '' }
+
+    it { expect(output.string).to eq '' }
+  end
+
+  describe '#read_action' do
+    context 'With empty line input' do
+      it { expect(subject.read_action).to be_nil }
+    end
+
+    context 'With valid action input' do
+      let(:action) { { 'action' => 'test', 'value' => 1 } }
+
+      before { input.string = action.to_json }
+
+      it { expect(subject.read_action).to eq action }
+    end
+
+    context 'With invalid action input' do
+      before { input.string = 'dummy' }
+
+      it {
+        expect { subject.read_action }.to raise_error Kcl::IOHandler::ReadError
+      }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'kcl'

metadata

@@ -0,0 +1,154 @@
+--- !ruby/object:Gem::Specification
+name: amazon-kinesis-client-ruby
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Soloman Weng
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.1.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.1.8
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.27.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.27.1
+description: Amazon Kinesis Client Library for Ruby
+email:
+- solomanw@everydayhero.com.au
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- amazon-kinesis-client-ruby.gemspec
+- lib/jars/amazon-kinesis-client-1.2.0.jar
+- lib/jars/aws-java-sdk-1.7.13.jar
+- lib/jars/commons-codec-1.3.jar
+- lib/jars/commons-logging-1.1.1.jar
+- lib/jars/httpclient-4.2.jar
+- lib/jars/httpcore-4.2.jar
+- lib/jars/jackson-annotations-2.1.1.jar
+- lib/jars/jackson-core-2.1.1.jar
+- lib/jars/jackson-databind-2.1.1.jar
+- lib/jars/joda-time-2.4.jar
+- lib/kcl.rb
+- lib/kcl/action_handler.rb
+- lib/kcl/advanced_record_processor.rb
+- lib/kcl/checkpoint_error.rb
+- lib/kcl/checkpointer.rb
+- lib/kcl/configuration.rb
+- lib/kcl/executor.rb
+- lib/kcl/executor_command_builder.rb
+- lib/kcl/io_handler.rb
+- lib/kcl/malformed_action_error.rb
+- lib/kcl/process.rb
+- lib/kcl/record_processor.rb
+- lib/kcl/version.rb
+- spec/configuration_spec.rb
+- spec/executor_command_builder_spec.rb
+- spec/executor_spec.rb
+- spec/io_handler_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: Amazon Kinesis Client Library for Ruby
+test_files:
+- spec/configuration_spec.rb
+- spec/executor_command_builder_spec.rb
+- spec/executor_spec.rb
+- spec/io_handler_spec.rb
+- spec/spec_helper.rb