amazon-kinesis-client-ruby 0.0.3 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/Rakefile +0 -1
- data/amazon-kinesis-client-ruby.gemspec +1 -0
- data/lib/kcl.rb +0 -6
- data/lib/kcl/advanced_record_processor.rb +14 -15
- data/lib/kcl/record_processor.rb +21 -2
- data/lib/kcl/version.rb +1 -1
- data/shippable.yml +18 -0
- metadata +18 -11
- data/lib/kcl/action_handler.rb +0 -34
- data/lib/kcl/checkpoint_error.rb +0 -16
- data/lib/kcl/checkpointer.rb +0 -37
- data/lib/kcl/io_handler.rb +0 -48
- data/lib/kcl/malformed_action_error.rb +0 -3
- data/lib/kcl/process.rb +0 -39
- data/spec/io_handler_spec.rb +0 -56
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: be4ed94e247161da0babb53aa73747f699e956a0
|
4
|
+
data.tar.gz: 8b478ab09b7ef3a37a10f22c9d7bcf7ecb37ca71
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ac16033602dc1dc372217127396f1576b92bb11be4d3c3a08edc303e3a623aa3251a9c92f6d76f6bd6b7361aa3c1e9e48d972a2ff28af192e50769371024ccf9
|
7
|
+
data.tar.gz: d3bb58d51eec7dc2785da0a80e3c007ca19c51911611a594c0853186da6ac33f565ac76c05e6e2a8c84ad9f19a42b4bb5e27450803f62bce93ac6ed42fe08759
|
data/README.md
CHANGED
data/Rakefile
CHANGED
@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
21
|
spec.add_runtime_dependency 'activesupport', '~> 4.1'
|
22
|
+
spec.add_runtime_dependency 'aws-kclrb', '~> 1.0.0'
|
22
23
|
|
23
24
|
spec.add_development_dependency 'bundler', '~> 1.7'
|
24
25
|
spec.add_development_dependency 'rake', '~> 10.0'
|
data/lib/kcl.rb
CHANGED
@@ -1,12 +1,6 @@
|
|
1
1
|
require 'kcl/version'
|
2
|
-
require 'kcl/action_handler'
|
3
2
|
require 'kcl/record_processor'
|
4
3
|
require 'kcl/advanced_record_processor'
|
5
|
-
require 'kcl/checkpoint_error'
|
6
|
-
require 'kcl/checkpointer'
|
7
4
|
require 'kcl/executor_command_builder'
|
8
|
-
require 'kcl/io_handler'
|
9
|
-
require 'kcl/malformed_action_error'
|
10
|
-
require 'kcl/process'
|
11
5
|
require 'kcl/configuration'
|
12
6
|
require 'kcl/executor'
|
@@ -1,12 +1,11 @@
|
|
1
1
|
require 'base64'
|
2
2
|
require 'logger'
|
3
|
+
require 'aws/kclrb'
|
3
4
|
|
4
5
|
module Kcl
|
5
6
|
class AdvancedRecordProcessor
|
6
7
|
include RecordProcessor
|
7
|
-
|
8
|
-
LOG = Logger.new STDOUT
|
9
|
-
ERR_LOG = Logger.new STDERR
|
8
|
+
LOG = Logger.new STDERR
|
10
9
|
|
11
10
|
DEFAULT_SLEEP_SECONDS = 5
|
12
11
|
DEFAULT_CHECKPOINT_RETRIES = 5
|
@@ -22,22 +21,22 @@ module Kcl
|
|
22
21
|
|
23
22
|
def process_record _record; end
|
24
23
|
|
25
|
-
def init
|
24
|
+
def init shard_id
|
25
|
+
LOG.info "Start consumming at shard: #{shard_id}"
|
26
26
|
self.largest_seq = nil
|
27
27
|
self.last_checkpoint_time = Time.now
|
28
28
|
end
|
29
29
|
|
30
30
|
def process_records records, checkpointer
|
31
|
-
records.each
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
31
|
+
records.each { |record| handle_record record }
|
32
|
+
rescue => error
|
33
|
+
LOG.error "Encountered an exception while processing records. Exception was #{error}"
|
34
|
+
force_checkpoint = true
|
35
|
+
ensure
|
36
|
+
if (Time.now - last_checkpoint_time > checkpoint_freq_seconds) || force_checkpoint
|
36
37
|
checkpoint checkpointer
|
37
38
|
self.last_checkpoint_time = Time.now
|
38
39
|
end
|
39
|
-
rescue => error
|
40
|
-
ERR_LOG.error "Encountered an exception while processing records. Exception was #{error}\n"
|
41
40
|
end
|
42
41
|
|
43
42
|
def shutdown checkpointer, reason
|
@@ -79,24 +78,24 @@ module Kcl
|
|
79
78
|
checkpointer.checkpoint seq
|
80
79
|
|
81
80
|
true
|
82
|
-
rescue CheckpointError => checkpoint_error
|
81
|
+
rescue Aws::KCLrb::CheckpointError => checkpoint_error
|
83
82
|
case checkpoint_error.to_s
|
84
83
|
when 'ShutdownException'
|
85
84
|
LOG.info 'Encountered shutdown execption, skipping checkpoint'
|
86
85
|
true
|
87
86
|
when 'ThrottlingException'
|
88
87
|
if checkpoint_retries - 1 == try_count
|
89
|
-
|
88
|
+
LOG.error "Failed to checkpoint after #{try_count} attempts, giving up."
|
90
89
|
true
|
91
90
|
else
|
92
91
|
LOG.info "Was throttled while checkpointing, will attempt again in #{sleep_seconds} seconds"
|
93
92
|
false
|
94
93
|
end
|
95
94
|
when 'InvalidStateException'
|
96
|
-
|
95
|
+
LOG.error "MultiLangDaemon reported an invalid state while checkpointing."
|
97
96
|
false
|
98
97
|
else
|
99
|
-
|
98
|
+
LOG.error "Encountered an error while checkpointing, error was #{checkpoint_error}."
|
100
99
|
false
|
101
100
|
end
|
102
101
|
end
|
data/lib/kcl/record_processor.rb
CHANGED
@@ -1,13 +1,32 @@
|
|
1
|
+
require 'aws/kclrb'
|
2
|
+
require 'forwardable'
|
3
|
+
|
1
4
|
module Kcl
|
2
5
|
module RecordProcessor
|
3
|
-
def init
|
6
|
+
def init _shard_id; end
|
4
7
|
|
5
8
|
def process_records _records, _checkpointer; end
|
6
9
|
|
7
10
|
def shutdown _checkpointer, _reason; end
|
8
11
|
|
9
12
|
def run
|
10
|
-
|
13
|
+
processor = RecordProcessorAdapter.new self
|
14
|
+
|
15
|
+
Aws::KCLrb::KCLProcess.new(processor).run
|
16
|
+
end
|
17
|
+
|
18
|
+
class RecordProcessorAdapter < Aws::KCLrb::RecordProcessorBase
|
19
|
+
extend Forwardable
|
20
|
+
|
21
|
+
def_delegator :@record_processor, :process_records, :shutdown
|
22
|
+
|
23
|
+
def initialize record_processor
|
24
|
+
@record_processor = record_processor
|
25
|
+
end
|
26
|
+
|
27
|
+
def init_processor shard_id
|
28
|
+
@record_processor.init shard_id
|
29
|
+
end
|
11
30
|
end
|
12
31
|
end
|
13
32
|
end
|
data/lib/kcl/version.rb
CHANGED
data/shippable.yml
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
build_image: shippableimages/ubuntu1404_ruby
|
2
|
+
language: ruby
|
3
|
+
rvm:
|
4
|
+
- 2.1.2
|
5
|
+
before_install:
|
6
|
+
- source ~/.rvm/scripts/rvm
|
7
|
+
- rvm install $SHIPPABLE_RUBY --verify-downloads 1
|
8
|
+
- source ~/.bashrc && ~/.rvm/scripts/rvm && rvm use $SHIPPABLE_RUBY
|
9
|
+
|
10
|
+
install:
|
11
|
+
- gem install bundler -v 1.7
|
12
|
+
- bundle install
|
13
|
+
- bundle exec rake install
|
14
|
+
|
15
|
+
script:
|
16
|
+
- bundle exec rake spec
|
17
|
+
|
18
|
+
cache: true
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: amazon-kinesis-client-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Soloman Weng
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-02-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -24,6 +24,20 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '4.1'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: aws-kclrb
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.0.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 1.0.0
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: bundler
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -119,23 +133,17 @@ files:
|
|
119
133
|
- lib/jars/jackson-databind-2.1.1.jar
|
120
134
|
- lib/jars/joda-time-2.5.jar
|
121
135
|
- lib/kcl.rb
|
122
|
-
- lib/kcl/action_handler.rb
|
123
136
|
- lib/kcl/advanced_record_processor.rb
|
124
|
-
- lib/kcl/checkpoint_error.rb
|
125
|
-
- lib/kcl/checkpointer.rb
|
126
137
|
- lib/kcl/configuration.rb
|
127
138
|
- lib/kcl/executor.rb
|
128
139
|
- lib/kcl/executor_command_builder.rb
|
129
|
-
- lib/kcl/io_handler.rb
|
130
|
-
- lib/kcl/malformed_action_error.rb
|
131
|
-
- lib/kcl/process.rb
|
132
140
|
- lib/kcl/record_processor.rb
|
133
141
|
- lib/kcl/version.rb
|
134
142
|
- pom.xml
|
143
|
+
- shippable.yml
|
135
144
|
- spec/configuration_spec.rb
|
136
145
|
- spec/executor_command_builder_spec.rb
|
137
146
|
- spec/executor_spec.rb
|
138
|
-
- spec/io_handler_spec.rb
|
139
147
|
- spec/spec_helper.rb
|
140
148
|
homepage: ''
|
141
149
|
licenses:
|
@@ -157,7 +165,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
157
165
|
version: '0'
|
158
166
|
requirements: []
|
159
167
|
rubyforge_project:
|
160
|
-
rubygems_version: 2.
|
168
|
+
rubygems_version: 2.2.2
|
161
169
|
signing_key:
|
162
170
|
specification_version: 4
|
163
171
|
summary: Amazon Kinesis Client Library for Ruby
|
@@ -165,5 +173,4 @@ test_files:
|
|
165
173
|
- spec/configuration_spec.rb
|
166
174
|
- spec/executor_command_builder_spec.rb
|
167
175
|
- spec/executor_spec.rb
|
168
|
-
- spec/io_handler_spec.rb
|
169
176
|
- spec/spec_helper.rb
|
data/lib/kcl/action_handler.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
module Kcl
|
2
|
-
class ActionHandler
|
3
|
-
def initialize record_processor, checkpointer, io_handler
|
4
|
-
@record_processor = record_processor
|
5
|
-
@checkpointer = checkpointer
|
6
|
-
@io_handler = io_handler
|
7
|
-
end
|
8
|
-
|
9
|
-
# rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
10
|
-
def handle action
|
11
|
-
case action.fetch('action')
|
12
|
-
when 'initialize'
|
13
|
-
record_processor.init action.fetch('shardId')
|
14
|
-
when 'processRecords'
|
15
|
-
record_processor.process_records action.fetch('records'), checkpointer
|
16
|
-
when 'shutdown'
|
17
|
-
record_processor.shutdown checkpointer, action.fetch('reason')
|
18
|
-
else
|
19
|
-
fail MalformedActionError,
|
20
|
-
"Received an action which couldn't be understood. Action was #{action}"
|
21
|
-
end
|
22
|
-
rescue KeyError => key_error
|
23
|
-
raise MalformedActionError,
|
24
|
-
"Action #{action} was expected to have key: #{key_error.message}"
|
25
|
-
rescue => error
|
26
|
-
io_handler.write_error error.backtrace.join "\n"
|
27
|
-
end
|
28
|
-
# rubocop:enable Metrics/MethodLength,Metrics/AbcSize
|
29
|
-
|
30
|
-
private
|
31
|
-
|
32
|
-
attr_reader :record_processor, :io_handler, :checkpointer
|
33
|
-
end
|
34
|
-
end
|
data/lib/kcl/checkpoint_error.rb
DELETED
data/lib/kcl/checkpointer.rb
DELETED
@@ -1,37 +0,0 @@
|
|
1
|
-
module Kcl
|
2
|
-
class Checkpointer
|
3
|
-
def initialize io_handler
|
4
|
-
@io_handler = io_handler
|
5
|
-
end
|
6
|
-
|
7
|
-
def checkpoint sequence_number = nil
|
8
|
-
io_handler.write_action action: 'checkpoint', checkpoint: sequence_number
|
9
|
-
|
10
|
-
action = fetch_action
|
11
|
-
if action['action'] == 'checkpoint'
|
12
|
-
fail CheckpointError, action['error'] unless action['error'].nil?
|
13
|
-
else
|
14
|
-
fail CheckpointError, 'InvalidStateException'
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
private
|
19
|
-
|
20
|
-
attr_reader :io_handler
|
21
|
-
|
22
|
-
def fetch_action
|
23
|
-
loop do
|
24
|
-
action = read_action
|
25
|
-
|
26
|
-
return action unless action.nil?
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def read_action
|
31
|
-
io_handler.read_action
|
32
|
-
rescue IOHandler::ReadError => read_error
|
33
|
-
io_handler.write_error \
|
34
|
-
"Could not understand line read from input: #{read_error.line}"
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
data/lib/kcl/io_handler.rb
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'json'
|
2
|
-
|
3
|
-
module Kcl
|
4
|
-
class IOHandler
|
5
|
-
def initialize input, output, error
|
6
|
-
@input = input
|
7
|
-
@output = output
|
8
|
-
@error = error
|
9
|
-
end
|
10
|
-
|
11
|
-
def write_action response
|
12
|
-
write_line response.to_json
|
13
|
-
end
|
14
|
-
|
15
|
-
def read_action
|
16
|
-
line = input.gets
|
17
|
-
JSON.parse line unless line.nil? || line.empty?
|
18
|
-
rescue => error
|
19
|
-
raise ReadError.new(error, line)
|
20
|
-
end
|
21
|
-
|
22
|
-
def write_error error_message
|
23
|
-
error << "#{error_message}\n"
|
24
|
-
ensure
|
25
|
-
error.flush
|
26
|
-
end
|
27
|
-
|
28
|
-
private
|
29
|
-
|
30
|
-
attr_reader :input, :output, :error
|
31
|
-
|
32
|
-
def write_line line
|
33
|
-
output << "\n#{line}\n"
|
34
|
-
ensure
|
35
|
-
output.flush
|
36
|
-
end
|
37
|
-
|
38
|
-
class ReadError < StandardError
|
39
|
-
attr_reader :base_error, :line
|
40
|
-
|
41
|
-
def initialize base_error, line
|
42
|
-
super base_error.message
|
43
|
-
@base_error = base_error
|
44
|
-
@line = line
|
45
|
-
end
|
46
|
-
end
|
47
|
-
end
|
48
|
-
end
|
data/lib/kcl/process.rb
DELETED
@@ -1,39 +0,0 @@
|
|
1
|
-
module Kcl
|
2
|
-
class Process
|
3
|
-
def initialize record_processor,
|
4
|
-
input: $stdin,
|
5
|
-
output: $stdout,
|
6
|
-
error: $stderr
|
7
|
-
@record_processor = record_processor
|
8
|
-
@io_handler = IOHandler.new input, output, error
|
9
|
-
@checkpointer = Checkpointer.new @io_handler
|
10
|
-
end
|
11
|
-
|
12
|
-
def run
|
13
|
-
loop do
|
14
|
-
action = io_handler.read_action
|
15
|
-
perform action
|
16
|
-
report_done action
|
17
|
-
|
18
|
-
break if action.nil?
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
|
24
|
-
attr_reader :record_processor, :io_handler, :checkpointer
|
25
|
-
|
26
|
-
def perform action
|
27
|
-
action_handler.handle action
|
28
|
-
end
|
29
|
-
|
30
|
-
def action_handler
|
31
|
-
@action_handler ||=
|
32
|
-
ActionHandler.new record_processor, checkpointer, io_handler
|
33
|
-
end
|
34
|
-
|
35
|
-
def report_done action
|
36
|
-
io_handler.write_action action: 'status', responseFor: action['action']
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
data/spec/io_handler_spec.rb
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'stringio'
|
3
|
-
|
4
|
-
describe Kcl::IOHandler do
|
5
|
-
let(:input) { StringIO.new }
|
6
|
-
let(:output) { StringIO.new }
|
7
|
-
let(:error) { StringIO.new }
|
8
|
-
|
9
|
-
subject { Kcl::IOHandler.new input, output, error }
|
10
|
-
|
11
|
-
describe '#write_action' do
|
12
|
-
let(:action) { { action: 'test', value: 123 } }
|
13
|
-
|
14
|
-
before { subject.write_action action }
|
15
|
-
|
16
|
-
it { expect(output.string).to eq "\n#{action.to_json}\n" }
|
17
|
-
|
18
|
-
it { expect(input.string).to eq '' }
|
19
|
-
|
20
|
-
it { expect(error.string).to eq '' }
|
21
|
-
end
|
22
|
-
|
23
|
-
describe '#write_error' do
|
24
|
-
let(:message) { 'Some error' }
|
25
|
-
|
26
|
-
before { subject.write_error message }
|
27
|
-
|
28
|
-
it { expect(error.string).to eq "#{message}\n" }
|
29
|
-
|
30
|
-
it { expect(input.string).to eq '' }
|
31
|
-
|
32
|
-
it { expect(output.string).to eq '' }
|
33
|
-
end
|
34
|
-
|
35
|
-
describe '#read_action' do
|
36
|
-
context 'With empty line input' do
|
37
|
-
it { expect(subject.read_action).to be_nil }
|
38
|
-
end
|
39
|
-
|
40
|
-
context 'With valid action input' do
|
41
|
-
let(:action) { { 'action' => 'test', 'value' => 1 } }
|
42
|
-
|
43
|
-
before { input.string = action.to_json }
|
44
|
-
|
45
|
-
it { expect(subject.read_action).to eq action }
|
46
|
-
end
|
47
|
-
|
48
|
-
context 'With invalid action input' do
|
49
|
-
before { input.string = 'dummy' }
|
50
|
-
|
51
|
-
it {
|
52
|
-
expect { subject.read_action }.to raise_error Kcl::IOHandler::ReadError
|
53
|
-
}
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|