aws-kclrb 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 08f02b6b3c02cdfc113ee2a43823c81d94b5b2bb
+  data.tar.gz: b55771d377c8b4b6880245049f93fc9498bd18f7
+SHA512:
+  metadata.gz: 9602145a147e21e3cf5a2285bfc0933ed747303dc354683fa8b0e83a943ea92b94a19997e61d3a37faa173ce38cc20851e11dbe3f10d71d94adba75e78e72940
+  data.tar.gz: 6993ffb17e798357b00a22a00da7d6ba1fd43283c98b03651445ae07e4c58766899df26e5421f4f5454b7fa39cc068bd4d4be291ced66a5e998bed2213ff2bcb

data/.rspec

@@ -0,0 +1,4 @@
+--color
+--format progress
+-r spec_helper
+-I./lib

data/.yardopts

@@ -0,0 +1,6 @@
+--title 'Amazon Kinesis Client Library for Ruby'
+--markup markdown
+--markup-provider redcarpet
+--hide-api private
+lib/**/*.rb - README.md LICENSE.txt NOTICE.txt
+

data/LICENSE.txt

@@ -0,0 +1,40 @@
+
+Amazon Software License
+
+This Amazon Software License (“License”) governs your use, reproduction, and distribution of the accompanying software as specified below.
+1. Definitions
+
+“Licensor” means any person or entity that distributes its Work.
+
+“Software” means the original work of authorship made available under this License.
+
+“Work” means the Software and any additions to or derivative works of the Software that are made available under this License.
+
+The terms “reproduce,” “reproduction,” “derivative works,” and “distribution” have the meaning as provided under U.S. copyright law; provided, however, that for the purposes of this License, derivative works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work.
+
+Works, including the Software, are “made available” under this License by including in or with the Work either (a) a copyright notice referencing the applicability of this License to the Work, or (b) a copy of this License.
+2. License Grants
+
+2.1 Copyright Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free, copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense and distribute its Work and any resulting derivative works in any form.
+
+2.2 Patent Grant. Subject to the terms and conditions of this License, each Licensor grants to you a perpetual, worldwide, non-exclusive, royalty-free patent license to make, have made, use, sell, offer for sale, import, and otherwise transfer its Work, in whole or in part. The foregoing license applies only to the patent claims licensable by Licensor that would be infringed by Licensor’s Work (or portion thereof) individually and excluding any combinations with any other materials or technology.
+3. Limitations
+
+3.1 Redistribution. You may reproduce or distribute the Work only if (a) you do so under this License, (b) you include a complete copy of this License with your distribution, and (c) you retain without modification any copyright, patent, trademark, or attribution notices that are present in the Work.
+
+3.2 Derivative Works. You may specify that additional or different terms apply to the use, reproduction, and distribution of your derivative works of the Work (“Your Terms”) only if (a) Your Terms provide that the use limitation in Section 3.3 applies to your derivative works, and (b) you identify the specific derivative works that are subject to Your Terms. Notwithstanding Your Terms, this License (including the redistribution requirements in Section 3.1) will continue to apply to the Work itself.
+
+3.3 Use Limitation. The Work and any derivative works thereof only may be used or intended for use with the web services, computing platforms or applications provided by Amazon.com, Inc. or its affiliates, including Amazon Web Services, Inc.
+
+3.4 Patent Claims. If you bring or threaten to bring a patent claim against any Licensor (including any claim, cross-claim or counterclaim in a lawsuit) to enforce any patents that you allege are infringed by any Work, then your rights under this License from such Licensor (including the grants in Sections 2.1 and 2.2) will terminate immediately.
+
+3.5 Trademarks. This License does not grant any rights to use any Licensor’s or its affiliates’ names, logos, or trademarks, except as necessary to reproduce the notices described in this License.
+
+3.6 Termination. If you violate any term of this License, then your rights under this License (including the grants in Sections 2.1 and 2.2) will terminate immediately.
+4. Disclaimer of Warranty.
+
+THE WORK IS PROVIDED “AS IS” WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF M ERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER THIS LICENSE. SOME STATES’ CONSUMER LAWS DO NOT ALLOW EXCLUSION OF AN IMPLIED WARRANTY, SO THIS DISCLAIMER MAY NOT APPLY TO YOU.
+5. Limitation of Liability.
+
+EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER COMM ERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+

data/NOTICE.txt

@@ -0,0 +1,2 @@
+Amazon Kinesis Client Library for Ruby
+Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.

data/README.md

@@ -0,0 +1,195 @@
+# Amazon Kinesis Client Library for Ruby
+
+This package provides an interface to the Amazon Kinesis Client Library's (KCL) [MultiLangDaemon][multi-lang-daemon]
+for the Ruby language.
+Developers can use the [Amazon KCL][amazon-kcl] to build distributed applications that process streaming data reliably
+at scale. The [Amazon KCL][amazon-kcl] takes care of many of the complex tasks associated with distributed computing,
+such as load-balancing across multiple instances, responding to instance failures, checkpointing processed records,
+and reacting to changes in stream volume.
+This package wraps and manages the interaction with the [MultiLangDaemon][multi-lang-daemon] which is part of the
+[Amazon KCL for Java][amazon-kcl-github] so that developers can focus on implementing their record processor
+executable. A record processor in Ruby typically looks something like:
+
+```ruby
+    #! /usr/bin/env ruby
+
+    require 'aws/kclrb'
+
+    class SampleRecordProcessor < Aws::KCLrb::RecordProcessorBase
+      def init_processor(shard_id)
+        # initialize
+      end
+
+      def process_records(records, checkpointer)
+        # process batch of records
+      end
+
+      def shutdown(checkpointer, reason)
+        # cleanup
+      end
+    end
+
+    if __FILE__ == $0
+      # Start the main processing loop
+      record_processor = SampleRecordProcessor.new
+      driver = Aws::KCLrb::KCLProcess.new(record_processor)
+      driver.run
+    end
+```
+
+## Before You Get Started
+
+Before running the samples, you'll want to make sure that your environment is
+configured to allow the samples to use your
+[AWS Security Credentials](http://docs.aws.amazon.com/general/latest/gr/aws-security-credentials.html).
+
+By default the samples use the [DefaultAWSCredentialsProviderChain][DefaultAWSCredentialsProviderChain]
+so you'll want to make your credentials available to one of the credentials providers in that
+provider chain. There are several ways to do this such as providing a `~/.aws/credentials` file,
+or if you're running on Amazon EC2, you can associate an IAM role with your instance with appropriate
+access.
+
+For questions regarding [Amazon Kinesis Service][amazon-kinesis] and the client libraries please check the
+[official documentation][amazon-kinesis-docs] as well as the [Amazon Kinesis Forums][kinesis-forum].
+
+## Running the Sample
+
+Using the Amazon KCL for Ruby package requires the [MultiLangDaemon][multi-lang-daemon] which
+is provided by the [Amazon KCL for Java][amazon-kcl-github]. Rake tasks are provided to start the sample
+application(s) and download all the required dependencies.
+
+The sample application consists of two components:
+
+* A data producer (`samples/sample_kcl_producer.rb`): this script creates an Amazon Kinesis
+  stream and starts putting random records into it.
+* A data processor (`samples/sample_kcl.rb`): this script is invoked by the
+  [MultiLangDaemon][multi-lang-daemon] and consumes the data from the Amazon Kinesis
+  stream and stores it into files (1 file per shard).
+
+The following defaults are used in the sample application:
+
+* *Stream name*: `kclrbsample`
+* *Number of shards*: 2
+* *Amazon KCL application name*: `RubyKCLSample`
+* *Amazon DynamoDB table for Amazon KCL application*: `RubyKCLSample`
+* *Sample application output directory*: `/tmp/kclrbsample/`
+
+### Running the Data Producer
+
+To run the data producer, run the following commands:
+
+```sh
+    cd samples
+    rake run_producer
+```
+
+#### Notes
+
+* The [AWS Ruby SDK gem][aws-ruby-sdk-gem] needs to be installed as a pre-requisite. To install,
+  run:
+
+  ```sh
+      sudo gem install aws-sdk
+  ```
+
+* The script `samples/sample_kcl_producer.rb` takes several parameters that you can use
+  to customize its behavior. To see the available options, run:
+
+  ```sh
+      samples/sample_kcl_producer.rb --help
+  ```
+
+### Running the Data Processor
+
+To run the data processor, run the following commands:
+
+```sh
+    cd samples
+    rake run
+```
+
+#### Notes
+
+* The `JAVA_HOME` environment variable needs to point to a valid JVM.
+* The rake task invokes the [MultiLangDaemon][multi-lang-daemon] passing to it the
+  properties file `samples/sample.properties`. This file contains the
+  information needed to bootstrap the sample application, e.g.
+
+  * `executableName = samples/sample_kcl.rb`
+  * `streamName = kclrbsample`
+  * `applicationName = RubyKCLSample`
+
+### Cleaning Up
+
+This sample application creates a real Amazon Kinesis stream and sends real data to it, and
+create a real DynamoDB table to track the Amazon KCL application state, thus potentially
+incurring AWS costs. Once done, you can log in to AWS management console and delete these
+resources. Specifically, the sample application will create in your default AWS region
+
+* an *Amazon Kinesis stream* named `kclrbsample`
+* an *Amazon DynamoDB table* named `RubyKCLSample`
+
+## Running on Amazon EC2
+
+Running on Amazon EC2 is simple. Assuming you are already logged into an Amazon EC2
+instance running Amazon Linux, the following steps will prepare your environment
+for running the sample application. Note the version of Java that ships with
+Amazon Linux can be found at `/usr/bin/java` and should be 1.7 or greater.
+
+```sh
+    # install some prerequisites if missing
+    sudo yum install gcc patch git ruby rake rubygems ruby-devel
+    # install the AWS Ruby SDK (pre-requisuite for producer)
+    sudo gem install aws-sdk aws-kclrb
+    # clone the git repository to work with the samples
+    git clone https://github.com/awslabs/amazon-kinesis-client-ruby.git kclrb
+    # run the sample
+    cd kclrb/samples
+    rake run_producer
+    # ... and in another terminal
+    rake run
+```
+
+## Under the Hood - What You Should Know about Amazon KCL's [MultiLangDaemon][multi-lang-daemon]
+
+Amazon KCL for Ruby uses [Amazon KCL for Java][amazon-kcl-github] internally. We have implemented
+a Java-based daemon, called the *MultiLangDaemon* that does all the heavy lifting. Our approach
+has the daemon spawn the user-defined record processor script/program as a sub-process. The
+*MultiLangDaemon* communicates with this sub-process over standard input/output using a simple
+protocol, and therefore the record processor script/program can be written in any language.
+
+At runtime, there will always be a one-to-one correspondence between a record processor, a child process,
+and an [Amazon Kinesis Shard][amazon-kinesis-shard]. The *MultiLangDaemon* will make sure of
+that, without any need for the developer to intervene.
+
+In this release, we have abstracted these implementation details away and exposed an interface that enables
+you to focus on writing record processing logic in Ruby. This approach enables [Amazon KCL][amazon-kcl] to
+be language agnostic, while providing identical features and similar parallel processing model across
+all languages.
+
+## See Also
+
+* [Developing Consumer Applications for Amazon Kinesis Using the Amazon Kinesis Client Library][amazon-kcl]
+* The [Amazon KCL for Java][amazon-kcl-github]
+* The [Amazon KCL for Python][amazon-kinesis-python-github]
+* The [Amazon Kinesis Documentation][amazon-kinesis-docs]
+* The [Amazon Kinesis Forum][kinesis-forum]
+
+## Release Notes
+
+### Release 1.0.0 (December 30, 2014)
+* **aws-kclrb** gem which exposes an interface to allow implementation of record processors in Ruby
+  using the Amazon KCL's [MultiLangDaemon][multi-lang-daemon]
+* **samples** directory contains a sample producer and processing applications using the Amazon KCL
+  for Ruby library.
+
+[amazon-kinesis]: http://aws.amazon.com/kinesis
+[amazon-kinesis-docs]: http://aws.amazon.com/documentation/kinesis/
+[amazon-kinesis-shard]: http://docs.aws.amazon.com/kinesis/latest/dev/key-concepts.html
+[amazon-kcl]: http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-app.html
+[amazon-kcl-github]: https://github.com/awslabs/amazon-kinesis-client
+[amazon-kinesis-python-github]: https://github.com/awslabs/amazon-kinesis-client-python
+[multi-lang-daemon]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java
+[DefaultAWSCredentialsProviderChain]: http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html
+[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
+[aws-ruby-sdk-gem]: https://rubygems.org/gems/aws-sdk

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/aws/kclrb.rb

@@ -0,0 +1,16 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'aws/kclrb/record_processor'
+require 'aws/kclrb/kcl_process'

data/lib/aws/kclrb/checkpointer.rb

@@ -0,0 +1,99 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'aws/kclrb/io_proxy'
+
+module Aws
+  module KCLrb
+    # Error class used for wrapping exception names passed through the 
+    # input stream.
+    class CheckpointError < RuntimeError
+      # @!attribute [r] value
+      # @return [String] the name of the exception wrapped by this instance.
+      attr_reader :value
+  
+      # @param value [String] The name of the exception that was received 
+      #   while checkpointing. For more details see 
+      #   {https://github.com/awslabs/amazon-kinesis-client/tree/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/exceptions KCL exceptions}.
+      #   Any of these exception names could be returned by the {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon}
+      #   as a response to a checkpoint action.
+      def initialize(value)
+        @value = value
+      end
+  
+      # @return [String] the name of the wrapped exception.
+      def to_s
+        @value.to_s
+      end
+    end
+  
+    # @abstract
+    # A checkpointer class which allows you to make checkpoint requests.
+    # 
+    # A checkpoint marks a point in a shard where you've successfully 
+    # processed to. If this processor fails or loses its lease to that 
+    # shard, another processor will be started either by this 
+    # {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon} 
+    # or a different instance and resume at the most recent checkpoint 
+    # in this shard.
+    class Checkpointer
+  
+      # Checkpoints at a particular sequence number you provide or if `nil`
+      # was passed, the checkpoint will be at the end of the most recently
+      # delivered list of records.      
+      #
+      # @param sequence_number [String, nil] The sequence number to checkpoint at 
+      #   or `nil` if you want to checkpoint at the farthest record.
+      # @raise [CheckpointError] if the {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon}
+      #   returned a response indicating an error, or if the checkpointer
+      #   encountered unexpected input.
+      def checkpoint(sequence_number=nil)
+        fail NotImplementedError.new
+      end
+    end
+    
+    
+    # @api private
+    # Default implementation of the {Checkpointer} abstract class.
+    class CheckpointerImpl
+      # @param io_proxy [IOProxy]  An {IOProxy} object to be used to read/write
+      #   checkpoint actions from/to the {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon}.
+      def initialize(io_proxy)
+        @io_proxy = io_proxy
+      end
+  
+      # (see Checkpointer#checkpoint)
+      def checkpoint(sequence_number=nil)
+        @io_proxy.write_action('checkpoint', 'checkpoint' => sequence_number)
+        # Consume the response action
+        action = @io_proxy.read_action
+        # Happy response is expected to be of the form:
+        #   `{"action":"checkpoint","checkpoint":"<seq-number>"}`
+        # Error response would look like the following:
+        #   `{"action":"checkpoint","checkpoint":"<seq-number>","error":"<error-type>"}`
+        if action && action['action'] == 'checkpoint'
+          raise CheckpointError.new(action['error'])  if action['error']
+        else
+          # We are in an invalid state. We will raise a checkpoint exception
+          #  to the RecordProcessor indicating that the KCL (or KCLrb) is in
+          #  an invalid state. See KCL documentation for description of this
+          #  exception. Note that the documented guidance is that this exception
+          #  is NOT retriable so the client code should exit (see 
+          #  https://github.com/awslabs/amazon-kinesis-client/tree/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/exceptions)
+          raise CheckpointError.new('InvalidStateException')
+        end
+      end
+    end
+  end
+end

data/lib/aws/kclrb/io_proxy.rb

@@ -0,0 +1,99 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+#
+
+require 'multi_json'
+
+module Aws
+  module KCLrb
+    # @api private
+    # Internal class used by {KCLProcess} and {Checkpointer} to communicate 
+    # with the the {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon} via the input and output streams.
+    class IOProxy
+      # @param input [IO, #readline] An `IO`-like object to read input lines from (e.g. `$stdin`).
+      # @param output [IO] An `IO`-like object to write output lines to (e.g. `$stdout`).
+      # @param error [IO] An `IO`-like object to write error lines to (e.g. `$stderr`).
+      def initialize(input, output, error)
+        @input = input
+        @output = output
+        @error = error
+      end
+  
+      # Reads one line from the input IO, strips it from any
+      # leading/trailing whitespaces, skipping empty lines.
+      #
+      # @return [String, nil] The line read from the input IO or `nil`
+      #   if end of stream was reached.
+      def read_line
+        line = nil
+        begin
+          line = @input.readline
+          break  unless line
+          line.strip!
+        end while line.empty?
+        line
+      rescue EOFError
+        nil
+      end
+  
+      # Reads a line and decodes it as a message from the {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon}.
+      #
+      # @return [Hash]  A hash representing the contents of the line, e.g. 
+      #   `{"action" => "initialize", "shardId" => "shardId-000001"}`
+      def read_action
+        line = read_line
+        if line
+          MultiJson.load(line)
+        end
+      end
+  
+      # Writes a line to the output stream. The line is preceded and followed by a 
+      # new line because other libraries could be writing to the output stream as 
+      # well (e.g. some libs might write debugging info to `$stdout`) so we would
+      # like to prevent our lines from being interlaced with other messages so 
+      # the MultiLangDaemon can understand them.
+      #
+      # @param line [String] A line to write to the output stream, e.g. 
+      #   `{"action":"status","responseFor":"<someAction>"}`
+      def write_line(line)
+        @output.write("\n#{line}\n")
+        @output.flush
+      end
+  
+  
+      # Writes a line to the error file.
+      #
+      # @param error [String,Exception] An exception or error message
+      def write_error(error)
+        if error.is_a?(Exception)
+          error = "#{error.class}: #{error.message}\n\t#{error.backtrace.join("\n\t")}"
+        end
+        @error.write("#{error}\n")
+        @error.flush
+      end
+  
+      # Writes a response action to the {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon},
+      # in JSON of the form:
+      #   `{"action":"<action>","detail1":"value1",...}`
+      # where the details depend on the type of the action. See {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon documentation} for more infortmation.
+      #
+      # @param action [String] The action name that will be put into the output JSON's `action` attribute.
+      # @param details [Hash] Additional key-value pairs to be added to the action response.
+      def write_action(action, details={})
+        response = {'action' => action}.merge(details)
+        write_line(MultiJson.dump(response))
+      end
+    end
+  end
+end

data/lib/aws/kclrb/kcl_process.rb

@@ -0,0 +1,95 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'aws/kclrb/io_proxy'
+require 'aws/kclrb/checkpointer'
+
+module Aws
+  module KCLrb
+    # Error raised if the {KCLProcess} received an input action that it
+    # could not parse or it could not handle.
+    class MalformedAction < RuntimeError; end
+  
+    # Entry point for a KCL application in Ruby.
+    #
+    # Implementers of KCL applications in Ruby should instantiate this
+    # class and invoke the {#run} method to start processing records.
+    class KCLProcess
+      # @param processor [RecordProcessorBase] A record processor
+      #   to use for processing a shard.
+      # @param input [IO] An `IO`-like object to read input lines from.
+      # @param output [IO] An `IO`-like object to write output lines to.
+      # @param error [IO] An `IO`-like object to write error lines to.
+      def initialize(processor, input=$stdin, output=$stdout, error=$stderr)
+        @processor = processor
+        @io_proxy = IOProxy.new(input, output, error)
+        @checkpointer = CheckpointerImpl.new(@io_proxy)
+      end
+  
+      # Starts this KCL processor's main loop.
+      def run
+        action = @io_proxy.read_action
+        while action do
+          process_action(action)
+          action = @io_proxy.read_action
+        end
+      end
+  
+      private
+      
+      # @api private
+      # Parses an input action and invokes the appropriate method of the
+      # record processor.
+      #
+      # @param action [Hash] A hash that represents an action to take with
+      #   appropriate attributes, as retrieved from {IOProxy#read_action}, e.g.
+      #
+      #   - `{"action":"initialize","shardId":"shardId-123"}`
+      #   - `{"action":"processRecords","records":[{"data":"bWVvdw==","partitionKey":"cat","sequenceNumber":"456"}]}`
+      #   - `{"action":"shutdown","reason":"TERMINATE"}`
+      # @raise [MalformedAction] if the action is missing expected attributes.
+      def process_action(action)
+        action_name = action.fetch('action')
+        case action_name
+        when 'initialize'
+          dispatch_to_processor(:init_processor, action.fetch('shardId'))
+        when 'processRecords'
+          dispatch_to_processor(:process_records, action.fetch('records'), @checkpointer)
+        when 'shutdown'
+          dispatch_to_processor(:shutdown, @checkpointer, action.fetch('reason'))
+        else
+          raise MalformedAction.new("Received an action which couldn't be understood. Action was '#{action}'")
+        end
+        @io_proxy.write_action('status', {'responseFor' => action_name})
+      rescue KeyError => ke
+        raise MalformedAction.new("Action '#{action}': #{ke.message}")
+      end
+  
+      # @api private
+      # Calls the specified method on the record processor, and handles
+      # any resulting exceptions by writing to the error stream.
+      def dispatch_to_processor(method, *args)
+        @processor.send(method, *args)
+      rescue => processor_error
+        # We don't know what the client's code could raise and we have
+        #   no way to recover if we let it propagate up further. We will
+        #   mimic the KCL and pass over client errors. We print their
+        #   stack trace to STDERR to help them notice and debug this type
+        #   of issue.
+        @io_proxy.write_error(processor_error)
+      end
+  
+    end
+  end
+end

data/lib/aws/kclrb/record_processor.rb

@@ -0,0 +1,77 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+module Aws
+  module KCLrb
+    # @abstract
+    # Base class for implementing a record processor.
+    # 
+    # A `RecordProcessor` processes a shard in a stream. See {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/clientlibrary/interfaces/IRecordProcessor.java the corresponding KCL interface}.
+    # Its methods will be called as follows:
+    #
+    # 1. {#init_processor} will be called once
+    # 2. {#process_records} will be called zero or more times
+    # 3. {#shutdown} will be called if this {https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/com/amazonaws/services/kinesis/multilang/package-info.java MultiLangDaemon}
+    #    instance loses the lease to this shard
+    class RecordProcessorBase
+      # @abstract
+      # Called once by a KCLProcess before any calls to process_records.
+      #
+      # @param shard_id [String] The shard id that this processor is going to be working on.
+      def init_processor(shard_id)
+        fail NotImplementedError.new
+      end
+  
+      # @abstract
+      # Called by a KCLProcess with a list of records to be processed and a checkpointer
+      # which accepts sequence numbers from the records to indicate where in the stream
+      # to checkpoint.
+      #
+      # @param records [Array<Hash>] A list of records that are to be processed. A record 
+      #   looks like:
+      #
+      #   ```
+      #   {"data":"<base64 encoded string>","partitionKey":"someKey","sequenceNumber":"1234567890"}
+      #   ```
+      #
+      #   Note that `data` attribute is a base64 encoded string. You can use `Base64.decode64`
+      #   in the `base64` module to get the original data as a string.
+      # @param checkpointer [Checkpointer] A checkpointer which accepts a sequence
+      #   number or no parameters.
+      def process_records(records, checkpointer)
+        fail NotImplementedError.new
+      end
+  
+      # @abstract
+      # Called by a KCLProcess instance to indicate that this record processor
+      # should shutdown. After this is called, there will be no more calls to
+      # any other methods of this record processor.
+      #
+      # @param checkpointer [Checkpointer] A checkpointer which accepts a sequence
+      #   number or no parameters.
+      # @param reason [String] The reason this record processor is being shutdown,
+      #   can be either `TERMINATE` or `ZOMBIE`.
+      #
+      #   - If `ZOMBIE`, clients should not checkpoint because there is possibly
+      #     another record processor which has acquired the lease for this shard.
+      #   - If `TERMINATE` then `checkpointer.checkpoint()` (without parameters)
+      #     should be called to checkpoint at the end of the shard so that this
+      #     processor will be shutdown and new processor(s) will be created to
+      #     for the child(ren) of this shard.
+      def shutdown(checkpointer, reason)
+        fail NotImplementedError.new
+      end
+    end
+  end
+end

data/spec/checkpointer_spec.rb

@@ -0,0 +1,50 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'aws/kclrb/io_proxy.rb'
+require 'aws/kclrb/checkpointer.rb'
+
+module Aws::KCLrb
+  describe Checkpointer do
+    describe "#checkpoint" do
+      it "should emit a checkpoint action and consume response action" do
+        seq_number = rand(100_000).to_s
+        expected_output_string = %Q[{"action":"checkpoint","checkpoint":"#{seq_number}"}]
+        input_string = %Q[{"action":"checkpoint","checkpoint":"#{seq_number}"}]
+        input = StringIO.new(input_string)
+        output = StringIO.new
+        error = StringIO.new
+        io_proxy = IOProxy.new(input, output, error)
+        checkpointer = CheckpointerImpl.new(io_proxy)
+        checkpointer.checkpoint(seq_number)
+        expect( output.string.strip ).to eq(expected_output_string.strip)
+        expect( input.eof? ).to eq(true)
+      end
+
+      it "should raise a CheckpointError when error is received from MultiLangDaemon" do
+        seq_number = rand(100_000).to_s
+        expected_output_string = %Q[{"action":"checkpoint","checkpoint":"#{seq_number}"}]
+        input_string = %Q[{"action":"checkpoint","checkpoint":"#{seq_number}","error":"ThrottlingException"}]
+        input = StringIO.new(input_string)
+        output = StringIO.new
+        error = StringIO.new
+        io_proxy = IOProxy.new(input, output, error)
+        checkpointer = CheckpointerImpl.new(io_proxy)
+        expect { checkpointer.checkpoint(seq_number) }.to raise_error(CheckpointError, /ThrottlingException/)
+        expect( output.string.strip ).to eq(expected_output_string.strip)
+        expect( input.eof? ).to eq(true)
+      end
+    end
+  end
+end

data/spec/io_proxy_spec.rb

@@ -0,0 +1,75 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+#
+#  http://aws.amazon.com/asl/
+#
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'aws/kclrb/io_proxy'
+
+module Aws::KCLrb
+  describe IOProxy do
+    describe "#read_line" do
+      it "should skip blank lines" do
+        input_string = "    \nline1\n\n\n  \nline2\n   \n"
+        input = StringIO.new(input_string)
+        output = StringIO.new
+        error = StringIO.new
+        io_proxy = IOProxy.new(input, output, error)
+        expect( io_proxy.read_line ).to eq("line1")
+        expect( io_proxy.read_line ).to eq("line2")
+        expect( io_proxy.read_line ).to be_nil
+      end
+      it "should return nil on EOF" do
+        input_string = "line1\n"
+        input = StringIO.new(input_string)
+        output = StringIO.new
+        error = StringIO.new
+        io_proxy = IOProxy.new(input, output, error)
+        expect( io_proxy.read_line ).to eq("line1")
+        expect( io_proxy.read_line ).to be_nil
+        expect( io_proxy.read_line ).to be_nil
+      end
+    end
+    describe "#write_error" do
+      it "should write an error message to the error stream" do
+        input = StringIO.new
+        output = StringIO.new
+        error = StringIO.new
+        io_proxy = IOProxy.new(input, output, error)
+        io_proxy.write_error('an error message')
+        expect( error.string.strip ).to eq('an error message')
+      end
+      it "should write exception details to the error stream" do
+        input = StringIO.new
+        output = StringIO.new
+        error = StringIO.new
+        io_proxy = IOProxy.new(input, output, error)
+        begin
+          raise RuntimeError.new("Test error")
+        rescue => e
+          io_proxy.write_error(e)
+        end
+        #puts error.string
+        expect( error.string.strip ).to match(/RuntimeError.*Test error/)
+      end
+    end
+    describe "#write_action" do
+      it "should write a valid JSON action to the output stream" do
+        input = StringIO.new
+        output = StringIO.new
+        error = StringIO.new
+        io_proxy = IOProxy.new(input, output, error)
+        io_proxy.write_action('status', 'responseFor' => 'initialize')
+        expect( output.string.strip ).to eq('{"action":"status","responseFor":"initialize"}')
+      end
+    end
+  end
+end

data/spec/kcl_process_spec.rb

@@ -0,0 +1,103 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'aws/kclrb/kcl_process'
+require 'aws/kclrb/record_processor'
+
+module Aws::KCLrb
+  # Dummy test class.
+  # The {#process_reocrds} method will retry the checkpointing call
+  #   in case of a throttling exception.
+  class TestRecordProcessor < RecordProcessorBase
+    def init_processor(shard_id)
+      # no-op
+    end
+
+    def process_records(records, checkpointer)
+      seq = records[0]['sequenceNumber']
+      begin
+        checkpointer.checkpoint(seq)
+      rescue CheckpointError => cpe
+        if cpe.value == 'ThrottlingException'
+          checkpointer.checkpoint(seq)
+        else
+          raise
+        end
+      end
+    end
+
+    def shutdown(checkpointer, reason)
+      checkpointer.checkpoint  if reason == 'TERMINATE'
+    end
+  end
+
+  describe KCLProcess do
+    describe "#run" do
+      it "should respond to each action by invoking the corresponding processor's method and write a status message to the output IO" do
+        input_specs = [
+          {:method => :init_processor, :action => 'initialize', :input => '{"action":"initialize","shardId":"shard-000001"}'},
+          {:method => :process_records, :action => 'processRecords', :input => '{"action":"processRecords","records":[]}'},
+          {:method => :shutdown, :action => 'shutdown', :input => '{"action":"shutdown","reason":"TERMINATE"}'},
+        ]
+        # pick any of the actions randomly to avoid writing a test for each
+        input_spec = input_specs.sample
+        processor = double(RecordProcessorBase)
+        expect(processor).to receive(input_spec[:method]).once
+        input = StringIO.new(input_spec[:input])
+        output = StringIO.new
+        error = StringIO.new
+        driver = KCLProcess.new(processor, input, output, error)
+        driver.run
+
+        expected_output = %Q[{"action":"status","responseFor":"#{input_spec[:action]}"}]
+        expect( output.string.gsub(/\s+/, "") ).to eq(expected_output.gsub(/\s+/, ""))
+        expect( error.string ).to eq("")
+        expect( input.eof? ).to eq(true)
+      end
+      it "should process a normal stream of actions and produce expected output" do
+        input_string = <<-INPUT
+{"action":"initialize","shardId":"shardId-123"}
+{"action":"processRecords","records":[{"data":"bWVvdw==","partitionKey":"cat","sequenceNumber":"456"}]}
+{"action":"checkpoint","checkpoint":"456","error":"ThrottlingException"}
+{"action":"checkpoint","checkpoint":"456"}
+{"action":"shutdown","reason":"TERMINATE"}
+{"action":"checkpoint","checkpoint":"456"}
+        INPUT
+
+        # NOTE: The first checkpoint is expected to fail
+        #       with a ThrottlingException and hence the
+        #       retry.
+        expected_output_string = <<-OUTPUT
+{"action":"status","responseFor":"initialize"}
+{"action":"checkpoint","checkpoint":"456"}
+{"action":"checkpoint","checkpoint":"456"}
+{"action":"status","responseFor":"processRecords"}
+{"action":"checkpoint","checkpoint":null}
+{"action":"status","responseFor":"shutdown"}
+        OUTPUT
+        processor = TestRecordProcessor.new
+        input = StringIO.new(input_string)
+        output = StringIO.new
+        error = StringIO.new
+        driver = KCLProcess.new(processor, input, output, error)
+        driver.run
+
+        # outputs should be same modulo some whitespaces
+        expect( output.string.gsub(/\s+/, "") ).to eq(expected_output_string.gsub(/\s+/, ""))
+        expect( error.string ).to eq("")
+        expect( input.eof? ).to eq(true)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+#
+#  Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+#  Licensed under the Amazon Software License (the "License").
+#  You may not use this file except in compliance with the License.
+#  A copy of the License is located at
+# 
+#  http://aws.amazon.com/asl/
+# 
+#  or in the "license" file accompanying this file. This file is distributed
+#  on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+#  express or implied. See the License for the specific language governing
+#  permissions and limitations under the License.
+
+require 'simplecov'
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+end

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: aws-kclrb
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Amazon Web Services
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: A ruby interface for the Amazon Kinesis Client Library MultiLangDaemon
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".yardopts"
+- LICENSE.txt
+- NOTICE.txt
+- README.md
+- VERSION
+- lib/aws/kclrb.rb
+- lib/aws/kclrb/checkpointer.rb
+- lib/aws/kclrb/io_proxy.rb
+- lib/aws/kclrb/kcl_process.rb
+- lib/aws/kclrb/record_processor.rb
+- spec/checkpointer_spec.rb
+- spec/io_proxy_spec.rb
+- spec/kcl_process_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/awslabs/amazon-kinesis-client-ruby
+licenses:
+- Amazon Software License
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Amazon Kinesis Client Library for Ruby
+test_files: []