kcl-rb 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/main.yml +58 -0
- data/.gitignore +11 -0
- data/.rubocop.yml +93 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +90 -0
- data/LICENSE.txt +21 -0
- data/README.md +130 -0
- data/Rakefile +2 -0
- data/aws/config +3 -0
- data/aws/credentials +3 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/demo/Gemfile +5 -0
- data/demo/Gemfile.lock +60 -0
- data/demo/README.md +38 -0
- data/demo/Rakefile +31 -0
- data/demo/aws/config +3 -0
- data/demo/aws/credentials +3 -0
- data/demo/docker-compose.yml +23 -0
- data/demo/lib/kcl_demo.rb +49 -0
- data/demo/lib/kcl_demo/demo_record_processor.rb +43 -0
- data/demo/lib/kcl_demo/demo_record_processor_factory.rb +7 -0
- data/demo/terraform/main.tf +35 -0
- data/docker-compose.yml +22 -0
- data/kcl-rb.gemspec +36 -0
- data/lib/kcl.rb +32 -0
- data/lib/kcl/checkpointer.rb +179 -0
- data/lib/kcl/checkpoints/sentinel.rb +17 -0
- data/lib/kcl/config.rb +35 -0
- data/lib/kcl/errors.rb +6 -0
- data/lib/kcl/logger.rb +3 -0
- data/lib/kcl/proxies/dynamo_db_proxy.rb +132 -0
- data/lib/kcl/proxies/kinesis_proxy.rb +56 -0
- data/lib/kcl/record_processor.rb +13 -0
- data/lib/kcl/record_processor_factory.rb +5 -0
- data/lib/kcl/types/extended_sequence_number.rb +89 -0
- data/lib/kcl/types/initialization_input.rb +13 -0
- data/lib/kcl/types/records_input.rb +15 -0
- data/lib/kcl/types/shutdown_input.rb +13 -0
- data/lib/kcl/version.rb +3 -0
- data/lib/kcl/worker.rb +159 -0
- data/lib/kcl/workers/consumer.rb +80 -0
- data/lib/kcl/workers/record_checkpointer.rb +14 -0
- data/lib/kcl/workers/shard_info.rb +47 -0
- data/lib/kcl/workers/shutdown_reason.rb +6 -0
- data/terraform/main.tf +35 -0
- metadata +191 -0
@@ -0,0 +1,80 @@
|
|
1
|
+
module Kcl::Workers
|
2
|
+
# Shard : Consumer = 1 : 1
|
3
|
+
# - get records from stream
|
4
|
+
# - send to record processor
|
5
|
+
# - create record checkpoint
|
6
|
+
class Consumer
|
7
|
+
def initialize(shard, record_processor, kinesis_proxy, checkpointer)
|
8
|
+
@shard = shard
|
9
|
+
@record_processor = record_processor
|
10
|
+
@kinesis = kinesis_proxy
|
11
|
+
@checkpointer = checkpointer
|
12
|
+
end
|
13
|
+
|
14
|
+
def consume!
|
15
|
+
initialize_input = create_initialize_input
|
16
|
+
@record_processor.after_initialize(initialize_input)
|
17
|
+
|
18
|
+
record_checkpointer = Kcl::Workers::RecordCheckpointer.new(@shard, @checkpointer)
|
19
|
+
shard_iterator = start_shard_iterator
|
20
|
+
|
21
|
+
loop do
|
22
|
+
result = @kinesis.get_records(shard_iterator)
|
23
|
+
|
24
|
+
records_input = create_records_input(
|
25
|
+
result[:records],
|
26
|
+
result[:millis_behind_latest],
|
27
|
+
record_checkpointer
|
28
|
+
)
|
29
|
+
@record_processor.process_records(records_input)
|
30
|
+
|
31
|
+
shard_iterator = result[:next_shard_iterator]
|
32
|
+
break if result[:records].empty? || shard_iterator.nil?
|
33
|
+
end
|
34
|
+
|
35
|
+
shutdown_reason = shard_iterator.nil? ?
|
36
|
+
Kcl::Workers::ShutdownReason::TERMINATE :
|
37
|
+
Kcl::Workers::ShutdownReason::REQUESTED
|
38
|
+
shutdown_input = create_shutdown_input(shutdown_reason, record_checkpointer)
|
39
|
+
@record_processor.shutdown(shutdown_input)
|
40
|
+
end
|
41
|
+
|
42
|
+
def start_shard_iterator
|
43
|
+
shard = @checkpointer.fetch_checkpoint(@shard)
|
44
|
+
if shard.checkpoint.nil?
|
45
|
+
return @kinesis.get_shard_iterator(
|
46
|
+
@shard.shard_id,
|
47
|
+
Kcl::Checkpoints::Sentinel::TRIM_HORIZON
|
48
|
+
)
|
49
|
+
end
|
50
|
+
|
51
|
+
@kinesis.get_shard_iterator(
|
52
|
+
@shard.shard_id,
|
53
|
+
Kcl::Checkpoints::Sentinel::AFTER_SEQUENCE_NUMBER,
|
54
|
+
@shard.checkpoint
|
55
|
+
)
|
56
|
+
end
|
57
|
+
|
58
|
+
def create_initialize_input
|
59
|
+
Kcl::Types::InitializationInput.new(
|
60
|
+
@shard.shard_id,
|
61
|
+
Kcl::Types::ExtendedSequenceNumber.new(@shard.checkpoint)
|
62
|
+
)
|
63
|
+
end
|
64
|
+
|
65
|
+
def create_records_input(records, millis_behind_latest, record_checkpointer)
|
66
|
+
Kcl::Types::RecordsInput.new(
|
67
|
+
records,
|
68
|
+
millis_behind_latest,
|
69
|
+
record_checkpointer
|
70
|
+
)
|
71
|
+
end
|
72
|
+
|
73
|
+
def create_shutdown_input(shutdown_reason, record_checkpointer)
|
74
|
+
Kcl::Types::ShutdownInput.new(
|
75
|
+
shutdown_reason,
|
76
|
+
record_checkpointer
|
77
|
+
)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Kcl::Workers
|
2
|
+
class RecordCheckpointer
|
3
|
+
def initialize(shard, checkpointer)
|
4
|
+
@shard = shard
|
5
|
+
@checkpointer = checkpointer
|
6
|
+
end
|
7
|
+
|
8
|
+
def update_checkpoint(sequence_number)
|
9
|
+
# checkpoint the last sequence of a closed shard
|
10
|
+
@shard.checkpoint = sequence_number || Kcl::Checkpoints::Sentinel::SHARD_END
|
11
|
+
@checkpointer.update_checkpoint(@shard)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module Kcl::Workers
|
2
|
+
class ShardInfo
|
3
|
+
attr_reader :shard_id,
|
4
|
+
:parent_shard_id,
|
5
|
+
:starting_sequence_number,
|
6
|
+
:ending_sequence_number
|
7
|
+
attr_accessor :assigned_to, :checkpoint, :lease_timeout
|
8
|
+
|
9
|
+
# @param [String] shard_id
|
10
|
+
# @param [String] parent_shard_id
|
11
|
+
# @param [Hash] sequence_number_range
|
12
|
+
def initialize(shard_id, parent_shard_id, sequence_number_range)
|
13
|
+
@shard_id = shard_id
|
14
|
+
@parent_shard_id = parent_shard_id || 0
|
15
|
+
@starting_sequence_number = sequence_number_range[:starting_sequence_number]
|
16
|
+
@ending_sequence_number = sequence_number_range[:ending_sequence_number]
|
17
|
+
@assigned_to = nil
|
18
|
+
@checkpoint = nil
|
19
|
+
@lease_timeout = nil
|
20
|
+
end
|
21
|
+
|
22
|
+
def lease_owner
|
23
|
+
@assigned_to
|
24
|
+
end
|
25
|
+
|
26
|
+
def lease_owner=(assigned_to)
|
27
|
+
@assigned_to = assigned_to
|
28
|
+
end
|
29
|
+
|
30
|
+
def completed?
|
31
|
+
@checkpoint == Kcl::Checkpoints::Sentinel::SHARD_END
|
32
|
+
end
|
33
|
+
|
34
|
+
# For debug
|
35
|
+
def to_h
|
36
|
+
{
|
37
|
+
shard_id: shard_id,
|
38
|
+
parent_shard_id: parent_shard_id,
|
39
|
+
starting_sequence_number: starting_sequence_number,
|
40
|
+
ending_sequence_number: ending_sequence_number,
|
41
|
+
assigned_to: assigned_to,
|
42
|
+
checkpoint: checkpoint,
|
43
|
+
lease_timeout: lease_timeout
|
44
|
+
}
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
data/terraform/main.tf
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
#-------------------------------------------------------------------------------
|
2
|
+
# 開発環境 (LocalStack)
|
3
|
+
#-------------------------------------------------------------------------------
|
4
|
+
|
5
|
+
provider "aws" {
|
6
|
+
version = "~> 2.60"
|
7
|
+
access_key = "dummy"
|
8
|
+
secret_key = "dummy"
|
9
|
+
region = "ap-northeast-1"
|
10
|
+
insecure = true
|
11
|
+
skip_credentials_validation = true
|
12
|
+
skip_metadata_api_check = true
|
13
|
+
skip_requesting_account_id = true
|
14
|
+
|
15
|
+
endpoints {
|
16
|
+
dynamodb = "https://localhost:4566"
|
17
|
+
kinesis = "https://localhost:4566"
|
18
|
+
}
|
19
|
+
}
|
20
|
+
|
21
|
+
|
22
|
+
#-------------------------------------------------------------------------------
|
23
|
+
# Kinesis stream
|
24
|
+
#-------------------------------------------------------------------------------
|
25
|
+
|
26
|
+
resource "aws_kinesis_stream" "kcl-rb-test_stream" {
|
27
|
+
name = "kcl-rb-test"
|
28
|
+
shard_count = 5
|
29
|
+
retention_period = 24
|
30
|
+
|
31
|
+
tags = {
|
32
|
+
Environment = "test"
|
33
|
+
}
|
34
|
+
}
|
35
|
+
|
metadata
ADDED
@@ -0,0 +1,191 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: kcl-rb
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- yo_waka
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2020-07-06 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '5.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '5.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: aws-sdk-dynamodb
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: aws-sdk-kinesis
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: eventmachine
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 1.2.7
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 1.2.7
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rake
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '12.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '12.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rspec
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '3.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 0.86.0
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 0.86.0
|
111
|
+
description: A pure ruby interface for Amazon Kinesis Client.
|
112
|
+
email:
|
113
|
+
- y.wakahara@gmail.com
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files: []
|
117
|
+
files:
|
118
|
+
- ".github/workflows/main.yml"
|
119
|
+
- ".gitignore"
|
120
|
+
- ".rubocop.yml"
|
121
|
+
- ".ruby-version"
|
122
|
+
- Gemfile
|
123
|
+
- Gemfile.lock
|
124
|
+
- LICENSE.txt
|
125
|
+
- README.md
|
126
|
+
- Rakefile
|
127
|
+
- aws/config
|
128
|
+
- aws/credentials
|
129
|
+
- bin/console
|
130
|
+
- bin/setup
|
131
|
+
- demo/Gemfile
|
132
|
+
- demo/Gemfile.lock
|
133
|
+
- demo/README.md
|
134
|
+
- demo/Rakefile
|
135
|
+
- demo/aws/config
|
136
|
+
- demo/aws/credentials
|
137
|
+
- demo/docker-compose.yml
|
138
|
+
- demo/lib/kcl_demo.rb
|
139
|
+
- demo/lib/kcl_demo/demo_record_processor.rb
|
140
|
+
- demo/lib/kcl_demo/demo_record_processor_factory.rb
|
141
|
+
- demo/terraform/main.tf
|
142
|
+
- docker-compose.yml
|
143
|
+
- kcl-rb.gemspec
|
144
|
+
- lib/kcl.rb
|
145
|
+
- lib/kcl/checkpointer.rb
|
146
|
+
- lib/kcl/checkpoints/sentinel.rb
|
147
|
+
- lib/kcl/config.rb
|
148
|
+
- lib/kcl/errors.rb
|
149
|
+
- lib/kcl/logger.rb
|
150
|
+
- lib/kcl/proxies/dynamo_db_proxy.rb
|
151
|
+
- lib/kcl/proxies/kinesis_proxy.rb
|
152
|
+
- lib/kcl/record_processor.rb
|
153
|
+
- lib/kcl/record_processor_factory.rb
|
154
|
+
- lib/kcl/types/extended_sequence_number.rb
|
155
|
+
- lib/kcl/types/initialization_input.rb
|
156
|
+
- lib/kcl/types/records_input.rb
|
157
|
+
- lib/kcl/types/shutdown_input.rb
|
158
|
+
- lib/kcl/version.rb
|
159
|
+
- lib/kcl/worker.rb
|
160
|
+
- lib/kcl/workers/consumer.rb
|
161
|
+
- lib/kcl/workers/record_checkpointer.rb
|
162
|
+
- lib/kcl/workers/shard_info.rb
|
163
|
+
- lib/kcl/workers/shutdown_reason.rb
|
164
|
+
- terraform/main.tf
|
165
|
+
homepage: https://github.com/waka/kcl-rb
|
166
|
+
licenses:
|
167
|
+
- MIT
|
168
|
+
metadata:
|
169
|
+
homepage_uri: https://github.com/waka/kcl-rb
|
170
|
+
source_code_uri: https://github.com/waka/kcl-rb
|
171
|
+
changelog_uri: https://github.com/waka/kcl-rb/CHANGELOG
|
172
|
+
post_install_message:
|
173
|
+
rdoc_options: []
|
174
|
+
require_paths:
|
175
|
+
- lib
|
176
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: 2.3.0
|
181
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
182
|
+
requirements:
|
183
|
+
- - ">="
|
184
|
+
- !ruby/object:Gem::Version
|
185
|
+
version: '0'
|
186
|
+
requirements: []
|
187
|
+
rubygems_version: 3.1.2
|
188
|
+
signing_key:
|
189
|
+
specification_version: 4
|
190
|
+
summary: Amazon.Kinesis Client Library for Ruby.
|
191
|
+
test_files: []
|