kcl-rb 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.github/workflows/main.yml +58 -0
- data/.gitignore +11 -0
- data/.rubocop.yml +93 -0
- data/.ruby-version +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +90 -0
- data/LICENSE.txt +21 -0
- data/README.md +130 -0
- data/Rakefile +2 -0
- data/aws/config +3 -0
- data/aws/credentials +3 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/demo/Gemfile +5 -0
- data/demo/Gemfile.lock +60 -0
- data/demo/README.md +38 -0
- data/demo/Rakefile +31 -0
- data/demo/aws/config +3 -0
- data/demo/aws/credentials +3 -0
- data/demo/docker-compose.yml +23 -0
- data/demo/lib/kcl_demo.rb +49 -0
- data/demo/lib/kcl_demo/demo_record_processor.rb +43 -0
- data/demo/lib/kcl_demo/demo_record_processor_factory.rb +7 -0
- data/demo/terraform/main.tf +35 -0
- data/docker-compose.yml +22 -0
- data/kcl-rb.gemspec +36 -0
- data/lib/kcl.rb +32 -0
- data/lib/kcl/checkpointer.rb +179 -0
- data/lib/kcl/checkpoints/sentinel.rb +17 -0
- data/lib/kcl/config.rb +35 -0
- data/lib/kcl/errors.rb +6 -0
- data/lib/kcl/logger.rb +3 -0
- data/lib/kcl/proxies/dynamo_db_proxy.rb +132 -0
- data/lib/kcl/proxies/kinesis_proxy.rb +56 -0
- data/lib/kcl/record_processor.rb +13 -0
- data/lib/kcl/record_processor_factory.rb +5 -0
- data/lib/kcl/types/extended_sequence_number.rb +89 -0
- data/lib/kcl/types/initialization_input.rb +13 -0
- data/lib/kcl/types/records_input.rb +15 -0
- data/lib/kcl/types/shutdown_input.rb +13 -0
- data/lib/kcl/version.rb +3 -0
- data/lib/kcl/worker.rb +159 -0
- data/lib/kcl/workers/consumer.rb +80 -0
- data/lib/kcl/workers/record_checkpointer.rb +14 -0
- data/lib/kcl/workers/shard_info.rb +47 -0
- data/lib/kcl/workers/shutdown_reason.rb +6 -0
- data/terraform/main.tf +35 -0
- metadata +191 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 647ea8327b5ed00e957b90c4072b05fa7898ebf2ba3b64c6b8a7e2e13e327445
|
4
|
+
data.tar.gz: f5664d982cd02f065bd8d995e574e781864d6d3aa3808e96a13994a82de814af
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4f6e8f989caa626203dfaa6a681ff766299e9337a054e70ad98422ebb57eae11ba108a9651216b818ff010bf46c0d722a76c7b4bcd06b7bdb13b9451f50ff8ff
|
7
|
+
data.tar.gz: e4534afe312c6bf4915d263116163f53dcab44d7bb92928d27a5f2e9f2b6e68b2db987b2a9038b6c89e5e75a101ace4369740b9e939abfcd39b85ed1b5c8b796
|
@@ -0,0 +1,58 @@
|
|
1
|
+
name: CI
|
2
|
+
|
3
|
+
on:
|
4
|
+
push:
|
5
|
+
branches:
|
6
|
+
- master
|
7
|
+
paths-ignore:
|
8
|
+
- '**.md'
|
9
|
+
- '**.txt'
|
10
|
+
pull_request:
|
11
|
+
paths-ignore:
|
12
|
+
- '**.md'
|
13
|
+
- '**.txt'
|
14
|
+
|
15
|
+
jobs:
|
16
|
+
build:
|
17
|
+
runs-on: ubuntu-latest
|
18
|
+
|
19
|
+
steps:
|
20
|
+
- name: Checkout
|
21
|
+
uses: actions/checkout@v2
|
22
|
+
|
23
|
+
- name: Up docker-compose
|
24
|
+
run: docker-compose -f docker-compose.yml up -d
|
25
|
+
|
26
|
+
- name: Sleep for 30 seconds
|
27
|
+
uses: jakejarvis/wait-action@master
|
28
|
+
with:
|
29
|
+
time: '30s'
|
30
|
+
|
31
|
+
- name: Set up Terraform
|
32
|
+
uses: hashicorp/setup-terraform@v1
|
33
|
+
with:
|
34
|
+
terraform_version: 0.12.24
|
35
|
+
- id: init
|
36
|
+
run: terraform init
|
37
|
+
working-directory: ${{ github.workspace }}/terraform
|
38
|
+
- id: apply
|
39
|
+
run: terraform apply -auto-approve -no-color
|
40
|
+
working-directory: ${{ github.workspace }}/terraform
|
41
|
+
|
42
|
+
- name: Set up Ruby 2.7
|
43
|
+
uses: actions/setup-ruby@v1
|
44
|
+
with:
|
45
|
+
ruby-version: 2.7
|
46
|
+
|
47
|
+
- name: Bundle install
|
48
|
+
run: |
|
49
|
+
gem install bundler
|
50
|
+
bundle install --jobs 4 --retry 3
|
51
|
+
|
52
|
+
- name: Rubocop
|
53
|
+
run: |
|
54
|
+
bundle exec rubocop
|
55
|
+
|
56
|
+
- name: RSpec
|
57
|
+
run: |
|
58
|
+
bundle exec rspec
|
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,93 @@
|
|
1
|
+
AllCops:
|
2
|
+
Exclude:
|
3
|
+
- "vendor/**/*"
|
4
|
+
- "demo/vendor/**/*"
|
5
|
+
DisplayCopNames: true
|
6
|
+
|
7
|
+
Layout/ArgumentAlignment:
|
8
|
+
Enabled: false
|
9
|
+
|
10
|
+
Layout/ExtraSpacing:
|
11
|
+
Enabled: false
|
12
|
+
|
13
|
+
Layout/HashAlignment:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
Lint/NonDeterministicRequireOrder:
|
17
|
+
Enabled: false
|
18
|
+
|
19
|
+
Layout/SpaceAroundOperators:
|
20
|
+
Enabled: false
|
21
|
+
|
22
|
+
Metrics/AbcSize:
|
23
|
+
Enabled: false
|
24
|
+
|
25
|
+
Metrics/BlockLength:
|
26
|
+
Enabled: false
|
27
|
+
|
28
|
+
Metrics/ClassLength:
|
29
|
+
Max: 300
|
30
|
+
|
31
|
+
Metrics/MethodLength:
|
32
|
+
Max: 50
|
33
|
+
|
34
|
+
Naming/MemoizedInstanceVariableName:
|
35
|
+
Enabled: false
|
36
|
+
|
37
|
+
Metrics/CyclomaticComplexity:
|
38
|
+
Max: 15
|
39
|
+
|
40
|
+
Metrics/PerceivedComplexity:
|
41
|
+
Max: 15
|
42
|
+
|
43
|
+
# 日本語のコメントを許可する
|
44
|
+
Style/AsciiComments:
|
45
|
+
Enabled: false
|
46
|
+
|
47
|
+
Style/ClassAndModuleChildren:
|
48
|
+
Enabled: false
|
49
|
+
|
50
|
+
Style/Documentation:
|
51
|
+
Enabled: false
|
52
|
+
|
53
|
+
Style/EmptyLineAfterGuardClause:
|
54
|
+
Enabled: false
|
55
|
+
|
56
|
+
Style/EmptyMethod:
|
57
|
+
Enabled: false
|
58
|
+
|
59
|
+
Style/FrozenStringLiteralComment:
|
60
|
+
Enabled: false
|
61
|
+
|
62
|
+
Style/GuardClause:
|
63
|
+
MinBodyLength: 5
|
64
|
+
|
65
|
+
Style/HashSyntax:
|
66
|
+
Enabled: false
|
67
|
+
|
68
|
+
Style/IfUnlessModifier:
|
69
|
+
Enabled: false
|
70
|
+
|
71
|
+
Style/MultilineTernaryOperator:
|
72
|
+
Enabled: false
|
73
|
+
|
74
|
+
Style/NumericPredicate:
|
75
|
+
Enabled: false
|
76
|
+
|
77
|
+
Style/RaiseArgs:
|
78
|
+
Enabled: false
|
79
|
+
|
80
|
+
Style/RedundantBegin:
|
81
|
+
Enabled: false
|
82
|
+
|
83
|
+
Style/RedundantInterpolation:
|
84
|
+
Enabled: false
|
85
|
+
|
86
|
+
Style/RedundantSelf:
|
87
|
+
Enabled: false
|
88
|
+
|
89
|
+
Style/RescueStandardError:
|
90
|
+
Enabled: false
|
91
|
+
|
92
|
+
Style/SymbolArray:
|
93
|
+
Enabled: false
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.7.1
|
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
kcl-rb (1.0.0)
|
5
|
+
activesupport (>= 5.0)
|
6
|
+
aws-sdk-dynamodb (~> 1)
|
7
|
+
aws-sdk-kinesis (~> 1)
|
8
|
+
eventmachine (~> 1.2.7)
|
9
|
+
|
10
|
+
GEM
|
11
|
+
remote: https://rubygems.org/
|
12
|
+
specs:
|
13
|
+
activesupport (6.0.3.1)
|
14
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
15
|
+
i18n (>= 0.7, < 2)
|
16
|
+
minitest (~> 5.1)
|
17
|
+
tzinfo (~> 1.1)
|
18
|
+
zeitwerk (~> 2.2, >= 2.2.2)
|
19
|
+
ast (2.4.1)
|
20
|
+
aws-eventstream (1.1.0)
|
21
|
+
aws-partitions (1.326.0)
|
22
|
+
aws-sdk-core (3.98.0)
|
23
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
24
|
+
aws-partitions (~> 1, >= 1.239.0)
|
25
|
+
aws-sigv4 (~> 1.1)
|
26
|
+
jmespath (~> 1.0)
|
27
|
+
aws-sdk-dynamodb (1.48.0)
|
28
|
+
aws-sdk-core (~> 3, >= 3.71.0)
|
29
|
+
aws-sigv4 (~> 1.1)
|
30
|
+
aws-sdk-kinesis (1.23.0)
|
31
|
+
aws-sdk-core (~> 3, >= 3.71.0)
|
32
|
+
aws-sigv4 (~> 1.1)
|
33
|
+
aws-sigv4 (1.1.4)
|
34
|
+
aws-eventstream (~> 1.0, >= 1.0.2)
|
35
|
+
concurrent-ruby (1.1.6)
|
36
|
+
diff-lcs (1.4.2)
|
37
|
+
eventmachine (1.2.7)
|
38
|
+
i18n (1.8.3)
|
39
|
+
concurrent-ruby (~> 1.0)
|
40
|
+
jmespath (1.4.0)
|
41
|
+
minitest (5.14.1)
|
42
|
+
parallel (1.19.2)
|
43
|
+
parser (2.7.1.4)
|
44
|
+
ast (~> 2.4.1)
|
45
|
+
rainbow (3.0.0)
|
46
|
+
rake (12.3.3)
|
47
|
+
regexp_parser (1.7.1)
|
48
|
+
rexml (3.2.4)
|
49
|
+
rspec (3.9.0)
|
50
|
+
rspec-core (~> 3.9.0)
|
51
|
+
rspec-expectations (~> 3.9.0)
|
52
|
+
rspec-mocks (~> 3.9.0)
|
53
|
+
rspec-core (3.9.2)
|
54
|
+
rspec-support (~> 3.9.3)
|
55
|
+
rspec-expectations (3.9.2)
|
56
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
57
|
+
rspec-support (~> 3.9.0)
|
58
|
+
rspec-mocks (3.9.1)
|
59
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
60
|
+
rspec-support (~> 3.9.0)
|
61
|
+
rspec-support (3.9.3)
|
62
|
+
rubocop (0.86.0)
|
63
|
+
parallel (~> 1.10)
|
64
|
+
parser (>= 2.7.0.1)
|
65
|
+
rainbow (>= 2.2.2, < 4.0)
|
66
|
+
regexp_parser (>= 1.7)
|
67
|
+
rexml
|
68
|
+
rubocop-ast (>= 0.0.3, < 1.0)
|
69
|
+
ruby-progressbar (~> 1.7)
|
70
|
+
unicode-display_width (>= 1.4.0, < 2.0)
|
71
|
+
rubocop-ast (0.1.0)
|
72
|
+
parser (>= 2.7.0.1)
|
73
|
+
ruby-progressbar (1.10.1)
|
74
|
+
thread_safe (0.3.6)
|
75
|
+
tzinfo (1.2.7)
|
76
|
+
thread_safe (~> 0.1)
|
77
|
+
unicode-display_width (1.7.0)
|
78
|
+
zeitwerk (2.3.0)
|
79
|
+
|
80
|
+
PLATFORMS
|
81
|
+
ruby
|
82
|
+
|
83
|
+
DEPENDENCIES
|
84
|
+
kcl-rb!
|
85
|
+
rake (~> 12.0)
|
86
|
+
rspec (~> 3.0)
|
87
|
+
rubocop (~> 0.86.0)
|
88
|
+
|
89
|
+
BUNDLED WITH
|
90
|
+
2.1.4
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2020 yo_waka
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
# kcl-rb
|
2
|
+
|
3
|
+
## Overview
|
4
|
+
|
5
|
+
The Amazon Kinesis Client Library for Pure Ruby (Amazon KCL) enables Ruby developers to easily consume and process data from [Amazon Kinesis](http://aws.amazon.com/kinesis).
|
6
|
+
|
7
|
+
Already [KCL for Ruby](https://github.com/awslabs/amazon-kinesis-client-ruby) is provided by AWS, but Java is required for the operating environment because MultiLangDaemon is used.
|
8
|
+
**kcl-rb** is built on Pure Ruby, not depend on Java.
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
Add this line to your application's Gemfile:
|
13
|
+
|
14
|
+
```ruby
|
15
|
+
gem 'kcl-rb'
|
16
|
+
```
|
17
|
+
|
18
|
+
And then execute:
|
19
|
+
|
20
|
+
$ bundle install
|
21
|
+
|
22
|
+
Or install it yourself as:
|
23
|
+
|
24
|
+
$ gem install kcl-rb
|
25
|
+
|
26
|
+
## Usage
|
27
|
+
|
28
|
+
It's okay if you develop it according to [the KCL specifications](https://docs.aws.amazon.com/streams/latest/dev/kinesis-record-processor-implementation-app-java.html).
|
29
|
+
|
30
|
+
### Implement the RecordProcessor
|
31
|
+
|
32
|
+
```rb
|
33
|
+
class RecordProcessor < Kcl::RecordProcessor
|
34
|
+
def after_initialize(initialization_input)
|
35
|
+
puts "SHARD_ID: #{initialization_input.shard_id}"
|
36
|
+
end
|
37
|
+
|
38
|
+
def process_records(records_input)
|
39
|
+
puts "Current behind: #{records_input.millis_behind_latest}"
|
40
|
+
records_input.records.each do |record|
|
41
|
+
puts "Record: #{record}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def shutdown(shutdown_input)
|
46
|
+
puts "Shutdown reason: #{shutdown_input.shutdown_reason}"
|
47
|
+
|
48
|
+
if shutdown_input.shutdown_reason == Kcl::Workers::ShutdownReason::TERMINATE
|
49
|
+
shutdown_input.record_checkpointer.update_checkpoint(nil)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
```
|
54
|
+
|
55
|
+
### Implement a Class Factory for the RecordProcessor
|
56
|
+
|
57
|
+
```rb
|
58
|
+
class RecordProcessorFactory < Kcl::RecordProcessorFactory
|
59
|
+
def create_processor
|
60
|
+
RecordProcessor.new
|
61
|
+
end
|
62
|
+
end
|
63
|
+
```
|
64
|
+
|
65
|
+
### Initialize KCL configurations
|
66
|
+
|
67
|
+
```rb
|
68
|
+
Kcl.configure do |config|
|
69
|
+
config.aws_region = 'ap-northeast-1'
|
70
|
+
config.aws_access_key_id = 'dummy'
|
71
|
+
config.aws_secret_access_key = 'dummy'
|
72
|
+
config.dynamodb_endpoint = 'https://localhost:4566'
|
73
|
+
config.dynamodb_table_name = 'kcl-rb'
|
74
|
+
config.kinesis_endpoint = 'https://localhost:4566'
|
75
|
+
config.kinesis_stream_name = 'kcl-rb'
|
76
|
+
config.use_ssl = false
|
77
|
+
end
|
78
|
+
```
|
79
|
+
|
80
|
+
If you want to see all the setting items, please see [config class file](https://github.com/waka/kcl-rb/blob/master/lib/kcl/config.rb).
|
81
|
+
|
82
|
+
### Run a Worker
|
83
|
+
|
84
|
+
```rb
|
85
|
+
worker_id = 'kcl-worker'
|
86
|
+
factory = RecordProcessorFactory.new
|
87
|
+
Kcl::Worker.run(worker_id, factory)
|
88
|
+
```
|
89
|
+
|
90
|
+
If you want more concrete example, look under [the demo directory](https://github.com/waka/kcl-rb/tree/master/demo).
|
91
|
+
|
92
|
+
## Development
|
93
|
+
|
94
|
+
### Prerequisites
|
95
|
+
|
96
|
+
- Install Ruby 2.7.1
|
97
|
+
- Install docker
|
98
|
+
- Install Terraform
|
99
|
+
|
100
|
+
### Build & Run for RSpec
|
101
|
+
|
102
|
+
Create Kinesis resources on localstack using Terraform
|
103
|
+
|
104
|
+
```sh
|
105
|
+
$ docker-compose up -d
|
106
|
+
$ cd terraform
|
107
|
+
$ terraform init
|
108
|
+
$ terraform apply
|
109
|
+
```
|
110
|
+
|
111
|
+
Build dependencies.
|
112
|
+
|
113
|
+
```
|
114
|
+
$ bundle install --path vendor/bundle
|
115
|
+
```
|
116
|
+
|
117
|
+
And run RSpec.
|
118
|
+
|
119
|
+
```sh
|
120
|
+
$ bundle exec rspec
|
121
|
+
```
|
122
|
+
|
123
|
+
## Contributing
|
124
|
+
|
125
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/waka/kcl-rb.
|
126
|
+
|
127
|
+
|
128
|
+
## License
|
129
|
+
|
130
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/aws/config
ADDED
data/aws/credentials
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'kcl'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
require 'pry'
|
11
|
+
Pry.start
|
12
|
+
|
13
|
+
# require 'irb'
|
14
|
+
# IRB.start(__FILE__)
|