amazon-kinesis-client-ruby 0.0.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8124a97c0e7e8ea976c9f9cc547fafff165775a7
-  data.tar.gz: 37382747dd7c3d8e9e54774c97983e392278e0d8
+  metadata.gz: be4ed94e247161da0babb53aa73747f699e956a0
+  data.tar.gz: 8b478ab09b7ef3a37a10f22c9d7bcf7ecb37ca71
 SHA512:
-  metadata.gz: c3d106ff42b7fbac173f1a220156d5ad8bd7f1df57838f02fd00ec4c5dba64e2c1a832fb035e4ac1719f1dc0409251dffd491a9ae29dbeca0c580f96a468bdc6
-  data.tar.gz: 1fd05ff47e18134be1fbbebaa54f769d7772f637a33ecdff9f2531d5f0824f9044bed24001df887dcc7a5176caaa2dd1ebda7bb0c75ab95b2903fc38faaa3401
+  metadata.gz: ac16033602dc1dc372217127396f1576b92bb11be4d3c3a08edc303e3a623aa3251a9c92f6d76f6bd6b7361aa3c1e9e48d972a2ff28af192e50769371024ccf9
+  data.tar.gz: d3bb58d51eec7dc2785da0a80e3c007ca19c51911611a594c0853186da6ac33f565ac76c05e6e2a8c84ad9f19a42b4bb5e27450803f62bce93ac6ed42fe08759

data/README.md

@@ -137,7 +137,7 @@ require 'kcl'
 class YourProcessor
   include Kcl::RecordProcessor
 
-  def init shared_id
+  def init shard_id
     # Called once by a KCLProcess before any calls to process_records
   end
 

data/Rakefile

@@ -1,7 +1,6 @@
 require 'bundler/gem_tasks'
 require 'rspec/core/rake_task'
 require 'rubocop/rake_task'
-require 'rubygems/tasks'
 require 'maven/ruby/maven'
 
 RSpec::Core::RakeTask.new :spec do |spec|

data/amazon-kinesis-client-ruby.gemspec

@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_runtime_dependency 'activesupport', '~> 4.1'
+  spec.add_runtime_dependency 'aws-kclrb', '~> 1.0.0'
 
   spec.add_development_dependency 'bundler', '~> 1.7'
   spec.add_development_dependency 'rake', '~> 10.0'

data/lib/kcl.rb

@@ -1,12 +1,6 @@
 require 'kcl/version'
-require 'kcl/action_handler'
 require 'kcl/record_processor'
 require 'kcl/advanced_record_processor'
-require 'kcl/checkpoint_error'
-require 'kcl/checkpointer'
 require 'kcl/executor_command_builder'
-require 'kcl/io_handler'
-require 'kcl/malformed_action_error'
-require 'kcl/process'
 require 'kcl/configuration'
 require 'kcl/executor'

data/lib/kcl/advanced_record_processor.rb

@@ -1,12 +1,11 @@
 require 'base64'
 require 'logger'
+require 'aws/kclrb'
 
 module Kcl
   class AdvancedRecordProcessor
     include RecordProcessor
-
-    LOG = Logger.new STDOUT
-    ERR_LOG = Logger.new STDERR
+    LOG = Logger.new STDERR
 
     DEFAULT_SLEEP_SECONDS = 5
     DEFAULT_CHECKPOINT_RETRIES = 5
@@ -22,22 +21,22 @@ module Kcl
 
     def process_record _record; end
 
-    def init _shared_id
+    def init shard_id
+      LOG.info "Start consumming at shard: #{shard_id}"
       self.largest_seq = nil
       self.last_checkpoint_time = Time.now
     end
 
     def process_records records, checkpointer
-      records.each do |record|
-        handle_record record
-      end
-
-      if Time.now - last_checkpoint_time > checkpoint_freq_seconds
+      records.each { |record| handle_record record }
+    rescue => error
+      LOG.error "Encountered an exception while processing records. Exception was #{error}"
+      force_checkpoint = true
+    ensure
+      if (Time.now - last_checkpoint_time > checkpoint_freq_seconds) || force_checkpoint
         checkpoint checkpointer
         self.last_checkpoint_time = Time.now
       end
-    rescue => error
-      ERR_LOG.error "Encountered an exception while processing records. Exception was #{error}\n"
     end
 
     def shutdown checkpointer, reason
@@ -79,24 +78,24 @@ module Kcl
       checkpointer.checkpoint seq
 
       true
-    rescue CheckpointError => checkpoint_error
+    rescue Aws::KCLrb::CheckpointError => checkpoint_error
       case checkpoint_error.to_s
       when 'ShutdownException'
         LOG.info 'Encountered shutdown execption, skipping checkpoint'
         true
       when 'ThrottlingException'
         if checkpoint_retries - 1 == try_count
-          ERR_LOG.error "Failed to checkpoint after #{try_count} attempts, giving up.\n"
+          LOG.error "Failed to checkpoint after #{try_count} attempts, giving up."
           true
         else
           LOG.info "Was throttled while checkpointing, will attempt again in #{sleep_seconds} seconds"
           false
         end
       when 'InvalidStateException'
-        ERR_LOG.error "MultiLangDaemon reported an invalid state while checkpointing.\n"
+        LOG.error "MultiLangDaemon reported an invalid state while checkpointing."
         false
       else
-        ERR_LOG.error "Encountered an error while checkpointing, error was #{checkpoint_error}.\n"
+        LOG.error "Encountered an error while checkpointing, error was #{checkpoint_error}."
         false
       end
     end

data/lib/kcl/record_processor.rb

@@ -1,13 +1,32 @@
+require 'aws/kclrb'
+require 'forwardable'
+
 module Kcl
   module RecordProcessor
-    def init _shared_id; end
+    def init _shard_id; end
 
     def process_records _records, _checkpointer; end
 
     def shutdown _checkpointer, _reason; end
 
     def run
-      Process.new(self).run
+      processor = RecordProcessorAdapter.new self
+
+      Aws::KCLrb::KCLProcess.new(processor).run
+    end
+
+    class RecordProcessorAdapter < Aws::KCLrb::RecordProcessorBase
+      extend Forwardable
+
+      def_delegator :@record_processor, :process_records, :shutdown
+
+      def initialize record_processor
+        @record_processor = record_processor
+      end
+
+      def init_processor shard_id
+        @record_processor.init shard_id
+      end
     end
   end
 end

data/lib/kcl/version.rb

@@ -1,3 +1,3 @@
 module Kcl
-  VERSION = '0.0.3'
+  VERSION = '1.0.0'
 end

data/shippable.yml

@@ -0,0 +1,18 @@
+build_image: shippableimages/ubuntu1404_ruby
+language: ruby
+rvm:
+  - 2.1.2
+before_install:
+  - source ~/.rvm/scripts/rvm
+  - rvm install $SHIPPABLE_RUBY --verify-downloads 1
+  - source ~/.bashrc && ~/.rvm/scripts/rvm && rvm use $SHIPPABLE_RUBY
+
+install:
+  - gem install bundler -v 1.7
+  - bundle install
+  - bundle exec rake install
+
+script:
+  - bundle exec rake spec
+
+cache: true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: amazon-kinesis-client-ruby
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Soloman Weng
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-25 00:00:00.000000000 Z
+date: 2015-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: aws-kclrb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -119,23 +133,17 @@ files:
 - lib/jars/jackson-databind-2.1.1.jar
 - lib/jars/joda-time-2.5.jar
 - lib/kcl.rb
-- lib/kcl/action_handler.rb
 - lib/kcl/advanced_record_processor.rb
-- lib/kcl/checkpoint_error.rb
-- lib/kcl/checkpointer.rb
 - lib/kcl/configuration.rb
 - lib/kcl/executor.rb
 - lib/kcl/executor_command_builder.rb
-- lib/kcl/io_handler.rb
-- lib/kcl/malformed_action_error.rb
-- lib/kcl/process.rb
 - lib/kcl/record_processor.rb
 - lib/kcl/version.rb
 - pom.xml
+- shippable.yml
 - spec/configuration_spec.rb
 - spec/executor_command_builder_spec.rb
 - spec/executor_spec.rb
-- spec/io_handler_spec.rb
 - spec/spec_helper.rb
 homepage: ''
 licenses:
@@ -157,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.2
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Amazon Kinesis Client Library for Ruby
@@ -165,5 +173,4 @@ test_files:
 - spec/configuration_spec.rb
 - spec/executor_command_builder_spec.rb
 - spec/executor_spec.rb
-- spec/io_handler_spec.rb
 - spec/spec_helper.rb

data/lib/kcl/action_handler.rb

@@ -1,34 +0,0 @@
-module Kcl
-  class ActionHandler
-    def initialize record_processor, checkpointer, io_handler
-      @record_processor = record_processor
-      @checkpointer = checkpointer
-      @io_handler = io_handler
-    end
-
-    # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
-    def handle action
-      case action.fetch('action')
-      when 'initialize'
-        record_processor.init action.fetch('shardId')
-      when 'processRecords'
-        record_processor.process_records action.fetch('records'), checkpointer
-      when 'shutdown'
-        record_processor.shutdown checkpointer, action.fetch('reason')
-      else
-        fail MalformedActionError,
-             "Received an action which couldn't be understood. Action was #{action}"
-      end
-    rescue KeyError => key_error
-      raise MalformedActionError,
-            "Action #{action} was expected to have key: #{key_error.message}"
-    rescue => error
-      io_handler.write_error error.backtrace.join "\n"
-    end
-    # rubocop:enable Metrics/MethodLength,Metrics/AbcSize
-
-    private
-
-    attr_reader :record_processor, :io_handler, :checkpointer
-  end
-end

data/lib/kcl/checkpoint_error.rb

@@ -1,16 +0,0 @@
-module Kcl
-  class CheckpointError < StandardError
-    def initialize error_name
-      super error_name
-      @error_name = error_name
-    end
-
-    def to_s
-      error_name.to_s
-    end
-
-    private
-
-    attr_reader :error_name
-  end
-end

data/lib/kcl/checkpointer.rb

@@ -1,37 +0,0 @@
-module Kcl
-  class Checkpointer
-    def initialize io_handler
-      @io_handler = io_handler
-    end
-
-    def checkpoint sequence_number = nil
-      io_handler.write_action action: 'checkpoint', checkpoint: sequence_number
-
-      action = fetch_action
-      if action['action'] == 'checkpoint'
-        fail CheckpointError, action['error'] unless action['error'].nil?
-      else
-        fail CheckpointError, 'InvalidStateException'
-      end
-    end
-
-    private
-
-    attr_reader :io_handler
-
-    def fetch_action
-      loop do
-        action = read_action
-
-        return action unless action.nil?
-      end
-    end
-
-    def read_action
-      io_handler.read_action
-    rescue IOHandler::ReadError => read_error
-      io_handler.write_error \
-        "Could not understand line read from input: #{read_error.line}"
-    end
-  end
-end

data/lib/kcl/io_handler.rb

@@ -1,48 +0,0 @@
-require 'json'
-
-module Kcl
-  class IOHandler
-    def initialize input, output, error
-      @input = input
-      @output = output
-      @error = error
-    end
-
-    def write_action response
-      write_line response.to_json
-    end
-
-    def read_action
-      line = input.gets
-      JSON.parse line unless line.nil? || line.empty?
-    rescue => error
-      raise ReadError.new(error, line)
-    end
-
-    def write_error error_message
-      error << "#{error_message}\n"
-    ensure
-      error.flush
-    end
-
-    private
-
-    attr_reader :input, :output, :error
-
-    def write_line line
-      output << "\n#{line}\n"
-    ensure
-      output.flush
-    end
-
-    class ReadError < StandardError
-      attr_reader :base_error, :line
-
-      def initialize base_error, line
-        super base_error.message
-        @base_error = base_error
-        @line = line
-      end
-    end
-  end
-end

data/lib/kcl/malformed_action_error.rb

@@ -1,3 +0,0 @@
-module Kcl
-  class MalformedActionError < StandardError; end
-end

data/lib/kcl/process.rb

@@ -1,39 +0,0 @@
-module Kcl
-  class Process
-    def initialize record_processor,
-                   input: $stdin,
-                   output: $stdout,
-                   error: $stderr
-      @record_processor = record_processor
-      @io_handler = IOHandler.new input, output, error
-      @checkpointer = Checkpointer.new @io_handler
-    end
-
-    def run
-      loop do
-        action = io_handler.read_action
-        perform action
-        report_done action
-
-        break if action.nil?
-      end
-    end
-
-    private
-
-    attr_reader :record_processor, :io_handler, :checkpointer
-
-    def perform action
-      action_handler.handle action
-    end
-
-    def action_handler
-      @action_handler ||=
-        ActionHandler.new record_processor, checkpointer, io_handler
-    end
-
-    def report_done action
-      io_handler.write_action action: 'status', responseFor: action['action']
-    end
-  end
-end

data/spec/io_handler_spec.rb

@@ -1,56 +0,0 @@
-require 'spec_helper'
-require 'stringio'
-
-describe Kcl::IOHandler do
-  let(:input) { StringIO.new }
-  let(:output) { StringIO.new }
-  let(:error) { StringIO.new }
-
-  subject { Kcl::IOHandler.new input, output, error }
-
-  describe '#write_action' do
-    let(:action) { { action: 'test', value: 123 } }
-
-    before { subject.write_action action }
-
-    it { expect(output.string).to eq "\n#{action.to_json}\n" }
-
-    it { expect(input.string).to eq '' }
-
-    it { expect(error.string).to eq '' }
-  end
-
-  describe '#write_error' do
-    let(:message) { 'Some error' }
-
-    before { subject.write_error message }
-
-    it { expect(error.string).to eq "#{message}\n" }
-
-    it { expect(input.string).to eq '' }
-
-    it { expect(output.string).to eq '' }
-  end
-
-  describe '#read_action' do
-    context 'With empty line input' do
-      it { expect(subject.read_action).to be_nil }
-    end
-
-    context 'With valid action input' do
-      let(:action) { { 'action' => 'test', 'value' => 1 } }
-
-      before { input.string = action.to_json }
-
-      it { expect(subject.read_action).to eq action }
-    end
-
-    context 'With invalid action input' do
-      before { input.string = 'dummy' }
-
-      it {
-        expect { subject.read_action }.to raise_error Kcl::IOHandler::ReadError
-      }
-    end
-  end
-end