kcl-rb 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/main.yml +58 -0
- data/.gitignore +11 -0
- data/.rubocop.yml +93 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +90 -0
- data/LICENSE.txt +21 -0
- data/README.md +130 -0
- data/Rakefile +2 -0
- data/aws/config +3 -0
- data/aws/credentials +3 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/demo/Gemfile +5 -0
- data/demo/Gemfile.lock +60 -0
- data/demo/README.md +38 -0
- data/demo/Rakefile +31 -0
- data/demo/aws/config +3 -0
- data/demo/aws/credentials +3 -0
- data/demo/docker-compose.yml +23 -0
- data/demo/lib/kcl_demo.rb +49 -0
- data/demo/lib/kcl_demo/demo_record_processor.rb +43 -0
- data/demo/lib/kcl_demo/demo_record_processor_factory.rb +7 -0
- data/demo/terraform/main.tf +35 -0
- data/docker-compose.yml +22 -0
- data/kcl-rb.gemspec +36 -0
- data/lib/kcl.rb +32 -0
- data/lib/kcl/checkpointer.rb +179 -0
- data/lib/kcl/checkpoints/sentinel.rb +17 -0
- data/lib/kcl/config.rb +35 -0
- data/lib/kcl/errors.rb +6 -0
- data/lib/kcl/logger.rb +3 -0
- data/lib/kcl/proxies/dynamo_db_proxy.rb +132 -0
- data/lib/kcl/proxies/kinesis_proxy.rb +56 -0
- data/lib/kcl/record_processor.rb +13 -0
- data/lib/kcl/record_processor_factory.rb +5 -0
- data/lib/kcl/types/extended_sequence_number.rb +89 -0
- data/lib/kcl/types/initialization_input.rb +13 -0
- data/lib/kcl/types/records_input.rb +15 -0
- data/lib/kcl/types/shutdown_input.rb +13 -0
- data/lib/kcl/version.rb +3 -0
- data/lib/kcl/worker.rb +159 -0
- data/lib/kcl/workers/consumer.rb +80 -0
- data/lib/kcl/workers/record_checkpointer.rb +14 -0
- data/lib/kcl/workers/shard_info.rb +47 -0
- data/lib/kcl/workers/shutdown_reason.rb +6 -0
- data/terraform/main.tf +35 -0
- metadata +191 -0
data/bin/setup
ADDED
data/demo/Gemfile
ADDED
data/demo/Gemfile.lock
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
PATH
|
2
|
+
remote: ..
|
3
|
+
specs:
|
4
|
+
kcl-rb (1.0.0)
|
5
|
+
activesupport (>= 5.0)
|
6
|
+
aws-sdk-dynamodb (~> 1)
|
7
|
+
aws-sdk-kinesis (~> 1)
|
8
|
+
eventmachine (~> 1.2.7)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: https://rubygems.org/
|
12
|
+
specs:
|
13
|
+
activesupport (6.0.3.1)
|
14
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
15
|
+
i18n (>= 0.7, < 2)
|
16
|
+
minitest (~> 5.1)
|
17
|
+
tzinfo (~> 1.1)
|
18
|
+
zeitwerk (~> 2.2, >= 2.2.2)
|
19
|
+
aws-eventstream (1.1.0)
|
20
|
+
aws-partitions (1.326.0)
|
21
|
+
aws-sdk-core (3.98.0)
|
22
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
23
|
+
aws-partitions (~> 1, >= 1.239.0)
|
24
|
+
aws-sigv4 (~> 1.1)
|
25
|
+
jmespath (~> 1.0)
|
26
|
+
aws-sdk-dynamodb (1.48.0)
|
27
|
+
aws-sdk-core (~> 3, >= 3.71.0)
|
28
|
+
aws-sigv4 (~> 1.1)
|
29
|
+
aws-sdk-kinesis (1.23.0)
|
30
|
+
aws-sdk-core (~> 3, >= 3.71.0)
|
31
|
+
aws-sigv4 (~> 1.1)
|
32
|
+
aws-sigv4 (1.1.4)
|
33
|
+
aws-eventstream (~> 1.0, >= 1.0.2)
|
34
|
+
coderay (1.1.3)
|
35
|
+
concurrent-ruby (1.1.6)
|
36
|
+
eventmachine (1.2.7)
|
37
|
+
i18n (1.8.3)
|
38
|
+
concurrent-ruby (~> 1.0)
|
39
|
+
jmespath (1.4.0)
|
40
|
+
method_source (1.0.0)
|
41
|
+
minitest (5.14.1)
|
42
|
+
pry (0.13.1)
|
43
|
+
coderay (~> 1.1)
|
44
|
+
method_source (~> 1.0)
|
45
|
+
rake (12.3.3)
|
46
|
+
thread_safe (0.3.6)
|
47
|
+
tzinfo (1.2.7)
|
48
|
+
thread_safe (~> 0.1)
|
49
|
+
zeitwerk (2.3.0)
|
50
|
+
|
51
|
+
PLATFORMS
|
52
|
+
ruby
|
53
|
+
|
54
|
+
DEPENDENCIES
|
55
|
+
kcl-rb!
|
56
|
+
pry
|
57
|
+
rake (~> 12.0)
|
58
|
+
|
59
|
+
BUNDLED WITH
|
60
|
+
2.1.4
|
data/demo/README.md
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# kcl-rb Demo App
|
2
|
+
|
3
|
+
## Build and Run
|
4
|
+
|
5
|
+
Run localstack container (mock for Kinesis and DynamoDB).
|
6
|
+
|
7
|
+
```
|
8
|
+
$ docker-compose up
|
9
|
+
```
|
10
|
+
|
11
|
+
Create resources on localstack using Terraform
|
12
|
+
|
13
|
+
```
|
14
|
+
$ cd terraform
|
15
|
+
$ terraform init
|
16
|
+
$ terraform plan
|
17
|
+
$ terraform apply
|
18
|
+
```
|
19
|
+
|
20
|
+
Build dependencies
|
21
|
+
|
22
|
+
```
|
23
|
+
$ bundle install --path vendor/bundle
|
24
|
+
```
|
25
|
+
|
26
|
+
Run Demo KCL application
|
27
|
+
|
28
|
+
```
|
29
|
+
$ bundle exec rake run
|
30
|
+
```
|
31
|
+
|
32
|
+
Put records to Kinesis stream
|
33
|
+
|
34
|
+
```
|
35
|
+
$ RECORD_COUNT=10 bundle exec rake seed
|
36
|
+
```
|
37
|
+
|
38
|
+
You can see in console that the input data is distributed and processed by each consumer.
|
data/demo/Rakefile
ADDED
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
require 'pry'
|
3
|
+
|
4
|
+
require 'kcl'
|
5
|
+
require_relative './lib/kcl_demo'
|
6
|
+
|
7
|
+
task :default => :run
|
8
|
+
task :run do
|
9
|
+
KclDemo::App.initialize
|
10
|
+
KclDemo::App.run
|
11
|
+
end
|
12
|
+
|
13
|
+
task :seed do
|
14
|
+
KclDemo::App.initialize
|
15
|
+
record_count = Integer(ENV['RECORD_COUNT'] ||0)
|
16
|
+
if record_count.zero?
|
17
|
+
puts 'Set over 1 for RECORD_COUNT'
|
18
|
+
return
|
19
|
+
end
|
20
|
+
KclDemo::App.seed(record_count)
|
21
|
+
end
|
22
|
+
|
23
|
+
task :debug do
|
24
|
+
KclDemo::App.initialize
|
25
|
+
|
26
|
+
kinesis = Kcl::Proxies::KinesisProxy.new(KclDemo::App.config)
|
27
|
+
dynamodb = Kcl::Proxies::DynamoDbProxy.new(KclDemo::App.config)
|
28
|
+
# rubocop:disable Lint/Debugger
|
29
|
+
binding.pry
|
30
|
+
# rubocop:enable Lint/Debugger
|
31
|
+
end
|
data/demo/aws/config
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
version: '3'
|
2
|
+
|
3
|
+
volumes:
|
4
|
+
localstack-data:
|
5
|
+
driver: local
|
6
|
+
|
7
|
+
services:
|
8
|
+
localstack:
|
9
|
+
image: localstack/localstack:0.11.0
|
10
|
+
container_name: localstack-for-kcl-demo
|
11
|
+
ports:
|
12
|
+
- "8080:8080"
|
13
|
+
- "4566:4566"
|
14
|
+
environment:
|
15
|
+
- DATA_DIR=/tmp/localstack/data
|
16
|
+
- DEBUG=${LOCALSTACK_DEBUG:-true}
|
17
|
+
- DEFAULT_REGION=ap-northeast-1
|
18
|
+
- SERVICES=dynamodb,kinesis
|
19
|
+
- USE_SSL=true
|
20
|
+
volumes:
|
21
|
+
- "${PWD}/aws:/root/.aws"
|
22
|
+
- "/var/run/docker.sock:/var/run/docker.sock"
|
23
|
+
- "localstack-data:/tmp/localstack"
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'securerandom'
|
3
|
+
|
4
|
+
require_relative './kcl_demo/demo_record_processor'
|
5
|
+
require_relative './kcl_demo/demo_record_processor_factory'
|
6
|
+
|
7
|
+
module KclDemo
|
8
|
+
class App
|
9
|
+
def self.initialize
|
10
|
+
Kcl.configure do |config|
|
11
|
+
config.aws_region = 'ap-northeast-1'
|
12
|
+
config.aws_access_key_id = 'dummy'
|
13
|
+
config.aws_secret_access_key = 'dummy'
|
14
|
+
config.dynamodb_endpoint = 'https://localhost:4566'
|
15
|
+
config.dynamodb_table_name = 'kcl-rb-demo'
|
16
|
+
config.kinesis_endpoint = 'https://localhost:4566'
|
17
|
+
config.kinesis_stream_name = 'kcl-rb-demo'
|
18
|
+
config.use_ssl = false
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.config
|
23
|
+
Kcl.config
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.run
|
27
|
+
factory = KclDemo::DemoRecordProcessorFactory.new
|
28
|
+
Kcl::Worker.run('kcl-demo', factory)
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.seed(record_count = 1000)
|
32
|
+
proxy = Kcl::Proxies::KinesisProxy.new(config)
|
33
|
+
|
34
|
+
# puts records
|
35
|
+
record_count.times do |i|
|
36
|
+
str = SecureRandom.alphanumeric
|
37
|
+
hash = JSON.generate({ id: i, name: str })
|
38
|
+
resp = proxy.put_record(
|
39
|
+
{
|
40
|
+
stream_name: config.kinesis_stream_name,
|
41
|
+
data: Base64.strict_encode64(hash),
|
42
|
+
partition_key: str
|
43
|
+
}
|
44
|
+
)
|
45
|
+
puts resp
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'pry'
|
2
|
+
|
3
|
+
module KclDemo
|
4
|
+
class DemoRecordProcessor < Kcl::RecordProcessor
|
5
|
+
# @implement
|
6
|
+
def after_initialize(initialization_input)
|
7
|
+
Kcl.logger.info("Initialization at #{initialization_input}")
|
8
|
+
end
|
9
|
+
|
10
|
+
# @implement
|
11
|
+
def process_records(records_input)
|
12
|
+
Kcl.logger.info('Processing records...')
|
13
|
+
|
14
|
+
# レコードのリストを取得
|
15
|
+
return if records_input.records.empty?
|
16
|
+
|
17
|
+
# rubocop:disable Lint/Debugger
|
18
|
+
binding.pry if ENV['DEBUG'] == '1'
|
19
|
+
# rubocop:enable Lint/Debugger
|
20
|
+
|
21
|
+
records_input.records.each do |record|
|
22
|
+
Kcl.logger.info("Record = #{record}")
|
23
|
+
end
|
24
|
+
|
25
|
+
# チェックポイントを記録
|
26
|
+
last_sequence_number = records_input.records[-1].sequence_number
|
27
|
+
Kcl.logger.info(
|
28
|
+
"Checkpoint progress at: #{last_sequence_number}" \
|
29
|
+
", MillisBehindLatest = #{records_input.millis_behind_latest}"
|
30
|
+
)
|
31
|
+
records_input.record_checkpointer.update_checkpoint(last_sequence_number)
|
32
|
+
end
|
33
|
+
|
34
|
+
# @implement
|
35
|
+
def shutdown(shutdown_input)
|
36
|
+
Kcl.logger.info("Shutdown reason: #{shutdown_input.shutdown_reason}")
|
37
|
+
|
38
|
+
if shutdown_input.shutdown_reason == Kcl::Workers::ShutdownReason::TERMINATE
|
39
|
+
shutdown_input.record_checkpointer.update_checkpoint(nil)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
#-------------------------------------------------------------------------------
|
2
|
+
# 開発環境 (LocalStack)
|
3
|
+
#-------------------------------------------------------------------------------
|
4
|
+
|
5
|
+
provider "aws" {
|
6
|
+
version = "~> 2.60"
|
7
|
+
access_key = "dummy"
|
8
|
+
secret_key = "dummy"
|
9
|
+
region = "ap-northeast-1"
|
10
|
+
insecure = true
|
11
|
+
skip_credentials_validation = true
|
12
|
+
skip_metadata_api_check = true
|
13
|
+
skip_requesting_account_id = true
|
14
|
+
|
15
|
+
endpoints {
|
16
|
+
dynamodb = "https://localhost:4566"
|
17
|
+
kinesis = "https://localhost:4566"
|
18
|
+
}
|
19
|
+
}
|
20
|
+
|
21
|
+
|
22
|
+
#-------------------------------------------------------------------------------
|
23
|
+
# Kinesis stream
|
24
|
+
#-------------------------------------------------------------------------------
|
25
|
+
|
26
|
+
resource "aws_kinesis_stream" "kcl-rb-demo_stream" {
|
27
|
+
name = "kcl-rb-demo"
|
28
|
+
shard_count = 5
|
29
|
+
retention_period = 24
|
30
|
+
|
31
|
+
tags = {
|
32
|
+
Environment = "test"
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
version: '3'
|
2
|
+
|
3
|
+
volumes:
|
4
|
+
localstack-data:
|
5
|
+
driver: local
|
6
|
+
|
7
|
+
services:
|
8
|
+
localstack:
|
9
|
+
image: localstack/localstack:0.11.0
|
10
|
+
container_name: localstack-for-kcl
|
11
|
+
ports:
|
12
|
+
- "8080:8080"
|
13
|
+
- "4566:4566"
|
14
|
+
environment:
|
15
|
+
- DATA_DIR=/tmp/localstack/data
|
16
|
+
- DEBUG=${LOCALSTACK_DEBUG:-true}
|
17
|
+
- DEFAULT_REGION=ap-northeast-1
|
18
|
+
- SERVICES=dynamodb,kinesis
|
19
|
+
- USE_SSL=true
|
20
|
+
volumes:
|
21
|
+
- "${PWD}/aws:/root/.aws"
|
22
|
+
- "localstack-data:/tmp/localstack"
|
data/kcl-rb.gemspec
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
require_relative 'lib/kcl/version'
|
2
|
+
|
3
|
+
Gem::Specification.new do |spec|
|
4
|
+
spec.name = 'kcl-rb'
|
5
|
+
spec.version = Kcl::VERSION
|
6
|
+
spec.authors = ['yo_waka']
|
7
|
+
spec.email = ['y.wakahara@gmail.com']
|
8
|
+
|
9
|
+
spec.summary = 'Amazon.Kinesis Client Library for Ruby.'
|
10
|
+
spec.description = 'A pure ruby interface for Amazon Kinesis Client.'
|
11
|
+
spec.homepage = 'https://github.com/waka/kcl-rb'
|
12
|
+
spec.license = 'MIT'
|
13
|
+
spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
|
14
|
+
|
15
|
+
spec.metadata['homepage_uri'] = spec.homepage
|
16
|
+
spec.metadata['source_code_uri'] = 'https://github.com/waka/kcl-rb'
|
17
|
+
spec.metadata['changelog_uri'] = 'https://github.com/waka/kcl-rb/CHANGELOG'
|
18
|
+
|
19
|
+
# Specify which files should be added to the gem when it is released.
|
20
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
21
|
+
spec.files = Dir.chdir(File.expand_path(__dir__)) do
|
22
|
+
`git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
23
|
+
end
|
24
|
+
spec.bindir = 'exe'
|
25
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
26
|
+
spec.require_paths = ['lib']
|
27
|
+
|
28
|
+
spec.add_dependency 'activesupport', '>= 5.0'
|
29
|
+
spec.add_dependency 'aws-sdk-dynamodb', '~> 1'
|
30
|
+
spec.add_dependency 'aws-sdk-kinesis', '~> 1'
|
31
|
+
spec.add_dependency 'eventmachine', '~> 1.2.7'
|
32
|
+
|
33
|
+
spec.add_development_dependency 'rake', '~> 12.0'
|
34
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
35
|
+
spec.add_development_dependency 'rubocop', '~> 0.86.0'
|
36
|
+
end
|
data/lib/kcl.rb
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'kcl/checkpointer'
|
2
|
+
require 'kcl/checkpoints/sentinel'
|
3
|
+
require 'kcl/config'
|
4
|
+
require 'kcl/errors'
|
5
|
+
require 'kcl/logger'
|
6
|
+
require 'kcl/proxies/dynamo_db_proxy'
|
7
|
+
require 'kcl/proxies/kinesis_proxy'
|
8
|
+
require 'kcl/record_processor'
|
9
|
+
require 'kcl/record_processor_factory'
|
10
|
+
require 'kcl/types/extended_sequence_number'
|
11
|
+
require 'kcl/types/initialization_input'
|
12
|
+
require 'kcl/types/records_input'
|
13
|
+
require 'kcl/types/shutdown_input'
|
14
|
+
require 'kcl/worker'
|
15
|
+
require 'kcl/workers/consumer'
|
16
|
+
require 'kcl/workers/record_checkpointer'
|
17
|
+
require 'kcl/workers/shard_info'
|
18
|
+
require 'kcl/workers/shutdown_reason'
|
19
|
+
|
20
|
+
module Kcl
|
21
|
+
def self.configure
|
22
|
+
yield config
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.config
|
26
|
+
@_config ||= Kcl::Config.new
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.logger
|
30
|
+
@_logger ||= (config.logger || Kcl::Logger.new($stdout))
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,179 @@
|
|
1
|
+
require 'time'
|
2
|
+
|
3
|
+
class Kcl::Checkpointer
|
4
|
+
DYNAMO_DB_LEASE_PRIMARY_KEY = 'shard_id'.freeze
|
5
|
+
DYNAMO_DB_LEASE_OWNER_KEY = 'assigned_to'.freeze
|
6
|
+
DYNAMO_DB_LEASE_TIMEOUT_KEY = 'lease_timeout'.freeze
|
7
|
+
DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY = 'checkpoint'.freeze
|
8
|
+
DYNAMO_DB_PARENT_SHARD_KEY = 'parent_shard_id'.freeze
|
9
|
+
|
10
|
+
attr_reader :dynamodb
|
11
|
+
|
12
|
+
# @param [Kcl::Config] config
|
13
|
+
def initialize(config)
|
14
|
+
@dynamodb = Kcl::Proxies::DynamoDbProxy.new(config)
|
15
|
+
@table_name = config.dynamodb_table_name
|
16
|
+
|
17
|
+
return if @dynamodb.exists?(@table_name)
|
18
|
+
@dynamodb.create_table(
|
19
|
+
@table_name,
|
20
|
+
[{
|
21
|
+
attribute_name: DYNAMO_DB_LEASE_PRIMARY_KEY,
|
22
|
+
attribute_type: 'S'
|
23
|
+
}],
|
24
|
+
[{
|
25
|
+
attribute_name: DYNAMO_DB_LEASE_PRIMARY_KEY,
|
26
|
+
key_type: 'HASH'
|
27
|
+
}],
|
28
|
+
{
|
29
|
+
read_capacity_units: config.dynamodb_read_capacity,
|
30
|
+
write_capacity_units: config.dynamodb_write_capacity
|
31
|
+
}
|
32
|
+
)
|
33
|
+
Kcl.logger.info("Created DynamoDB table: #{@table_name}")
|
34
|
+
end
|
35
|
+
|
36
|
+
# Retrieves the checkpoint for the given shard
|
37
|
+
# @params [Kcl::Workers::ShardInfo] shard
|
38
|
+
# @return [Kcl::Workers::ShardInfo]
|
39
|
+
def fetch_checkpoint(shard)
|
40
|
+
checkpoint = @dynamodb.get_item(
|
41
|
+
@table_name,
|
42
|
+
{ "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id }
|
43
|
+
)
|
44
|
+
return shard if checkpoint.nil?
|
45
|
+
|
46
|
+
if checkpoint[DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY]
|
47
|
+
shard.checkpoint = checkpoint[DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY]
|
48
|
+
end
|
49
|
+
if checkpoint[DYNAMO_DB_LEASE_OWNER_KEY]
|
50
|
+
shard.assigned_to = checkpoint[DYNAMO_DB_LEASE_OWNER_KEY]
|
51
|
+
end
|
52
|
+
Kcl.logger.info("Retrieves checkpoint of shard at #{shard.to_h}")
|
53
|
+
|
54
|
+
shard
|
55
|
+
end
|
56
|
+
|
57
|
+
# Write the checkpoint for the given shard
|
58
|
+
# @params [Kcl::Workers::ShardInfo] shard
|
59
|
+
# @return [Kcl::Workers::ShardInfo]
|
60
|
+
def update_checkpoint(shard)
|
61
|
+
item = {
|
62
|
+
"#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id,
|
63
|
+
"#{DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY}" => shard.checkpoint,
|
64
|
+
"#{DYNAMO_DB_LEASE_OWNER_KEY}" => shard.assigned_to,
|
65
|
+
"#{DYNAMO_DB_LEASE_TIMEOUT_KEY}" => shard.lease_timeout.to_s
|
66
|
+
}
|
67
|
+
if shard.parent_shard_id > 0
|
68
|
+
item[DYNAMO_DB_PARENT_SHARD_KEY] = shard.parent_shard_id
|
69
|
+
end
|
70
|
+
|
71
|
+
result = @dynamodb.put_item(@table_name, item)
|
72
|
+
if result
|
73
|
+
Kcl.logger.info("Write checkpoint of shard at #{shard.to_h}")
|
74
|
+
else
|
75
|
+
Kcl.logger.info("Failed to write checkpoint for shard at #{shard.to_h}")
|
76
|
+
end
|
77
|
+
|
78
|
+
shard
|
79
|
+
end
|
80
|
+
|
81
|
+
# Attempt to gain a lock on the given shard
|
82
|
+
# @params [Kcl::Workers::ShardInfo] shard
|
83
|
+
# @params [String] next_assigned_to
|
84
|
+
# @return [Kcl::Workers::ShardInfo]
|
85
|
+
def lease(shard, next_assigned_to)
|
86
|
+
now = Time.now.utc
|
87
|
+
next_lease_timeout = now + Kcl.config.dynamodb_failover_seconds
|
88
|
+
|
89
|
+
checkpoint = @dynamodb.get_item(
|
90
|
+
@table_name,
|
91
|
+
{ "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id }
|
92
|
+
)
|
93
|
+
assigned_to = checkpoint && checkpoint[DYNAMO_DB_LEASE_OWNER_KEY]
|
94
|
+
lease_timeout = checkpoint && checkpoint[DYNAMO_DB_LEASE_TIMEOUT_KEY]
|
95
|
+
|
96
|
+
if assigned_to && lease_timeout
|
97
|
+
if now > Time.parse(lease_timeout) && assigned_to != next_assigned_to
|
98
|
+
raise Kcl::Errors::LeaseNotAquiredError
|
99
|
+
end
|
100
|
+
condition_expression = 'shard_id = :shard_id AND assigned_to = :assigned_to AND lease_timeout = :lease_timeout'
|
101
|
+
expression_attributes = {
|
102
|
+
':shard_id' => shard.shard_id,
|
103
|
+
':assigned_to' => assigned_to,
|
104
|
+
':lease_timeout' => lease_timeout
|
105
|
+
}
|
106
|
+
Kcl.logger.info("Attempting to get a lock for shard: #{shard.to_h}")
|
107
|
+
else
|
108
|
+
condition_expression = 'attribute_not_exists(assigned_to)'
|
109
|
+
expression_attributes = nil
|
110
|
+
end
|
111
|
+
|
112
|
+
item = {
|
113
|
+
"#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id,
|
114
|
+
"#{DYNAMO_DB_LEASE_OWNER_KEY}" => next_assigned_to,
|
115
|
+
"#{DYNAMO_DB_LEASE_TIMEOUT_KEY}" => next_lease_timeout.to_s
|
116
|
+
}
|
117
|
+
if shard.checkpoint != ''
|
118
|
+
item[DYNAMO_DB_CHECKPOINT_SEQUENCE_NUMBER_KEY] = shard.checkpoint
|
119
|
+
end
|
120
|
+
if shard.parent_shard_id > 0
|
121
|
+
item[DYNAMO_DB_PARENT_SHARD_KEY] = shard.parent_shard_id
|
122
|
+
end
|
123
|
+
|
124
|
+
result = @dynamodb.conditional_update_item(
|
125
|
+
@table_name,
|
126
|
+
item,
|
127
|
+
condition_expression,
|
128
|
+
expression_attributes
|
129
|
+
)
|
130
|
+
if result
|
131
|
+
shard.assigned_to = next_assigned_to
|
132
|
+
shard.lease_timeout = next_lease_timeout
|
133
|
+
Kcl.logger.info("Get lease for shard at #{shard.to_h}")
|
134
|
+
else
|
135
|
+
Kcl.logger.info("Failed to get lease for shard at #{shard.to_h}")
|
136
|
+
end
|
137
|
+
|
138
|
+
shard
|
139
|
+
end
|
140
|
+
|
141
|
+
# Remove the shard entry
|
142
|
+
# @params [Kcl::Workers::ShardInfo] shard
|
143
|
+
# @return [Kcl::Workers::ShardInfo]
|
144
|
+
def remove_lease(shard)
|
145
|
+
result = @dynamodb.remove_item(
|
146
|
+
@table_name,
|
147
|
+
{ "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id }
|
148
|
+
)
|
149
|
+
if result
|
150
|
+
shard.assigned_to = nil
|
151
|
+
shard.checkpoint = nil
|
152
|
+
shard.lease_timeout = nil
|
153
|
+
Kcl.logger.info("Remove lease for shard at #{shard.to_h}")
|
154
|
+
else
|
155
|
+
Kcl.logger.info("Failed to remove lease for shard at #{shard.to_h}")
|
156
|
+
end
|
157
|
+
|
158
|
+
shard
|
159
|
+
end
|
160
|
+
|
161
|
+
# Remove lease owner for the shard entry
|
162
|
+
# @params [Kcl::Workers::ShardInfo] shard
|
163
|
+
# @return [Kcl::Workers::ShardInfo]
|
164
|
+
def remove_lease_owner(shard)
|
165
|
+
result = @dynamodb.update_item(
|
166
|
+
@table_name,
|
167
|
+
{ "#{DYNAMO_DB_LEASE_PRIMARY_KEY}" => shard.shard_id },
|
168
|
+
"remove #{DYNAMO_DB_LEASE_OWNER_KEY}"
|
169
|
+
)
|
170
|
+
if result
|
171
|
+
shard.assigned_to = nil
|
172
|
+
Kcl.logger.info("Remove lease owner for shard at #{shard.to_h}")
|
173
|
+
else
|
174
|
+
Kcl.logger.info("Failed to remove lease owner for shard at #{shard.to_h}")
|
175
|
+
end
|
176
|
+
|
177
|
+
shard
|
178
|
+
end
|
179
|
+
end
|