aws-kclrb 2.2.2 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +88 -67
- data/VERSION +1 -1
- data/lib/aws/kclrb/kcl_process.rb +2 -2
- data/pom.xml +21 -15
- data/spec/kcl_process_spec.rb +99 -41
- metadata +29 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '0590e08cb8aeb08e0dedec5c6fb485c8f9fe9971882367bbdb26712003681a32'
|
|
4
|
+
data.tar.gz: '0760288c8a697240e83875e4127f7a6a2c2bf715f33c4e1184912a7419299b65'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bdc90bc9c8804ec3f0426c6c1e24396506065f3f77fd0df6a26070281a8c557f505fd7a5a0c83dd14876be956f199576c2c2a592d63e35fb36a7d455c643e291
|
|
7
|
+
data.tar.gz: 1ec72d31c3ced10560b34f8ddbaaafbae44b8b64f4a40f224ebe14e753e47b880991d05baf3a0b35961cece247a40ae330a9d70c7ed6306df942c258ed9647a4
|
data/README.md
CHANGED
|
@@ -11,38 +11,38 @@ This package wraps and manages the interaction with the [MultiLangDaemon][multi-
|
|
|
11
11
|
executable. A record processor in Ruby typically looks something like:
|
|
12
12
|
|
|
13
13
|
```ruby
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
14
|
+
#! /usr/bin/env ruby
|
|
15
|
+
|
|
16
|
+
require 'aws/kclrb'
|
|
17
|
+
|
|
18
|
+
class SampleRecordProcessor < Aws::KCLrb::V2::RecordProcessorBase
|
|
19
|
+
def init_processor(initialize_input)
|
|
20
|
+
# initialize
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def process_records(process_records_input)
|
|
24
|
+
# process batch of records
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def lease_lost(lease_lost_input)
|
|
28
|
+
# lease was lost, cleanup
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def shard_ended(shard_ended_input)
|
|
32
|
+
# shard has ended, cleanup
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def shutdown_requested(shutdown_requested_input)
|
|
36
|
+
# shutdown has been requested
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
if __FILE__ == $0
|
|
41
|
+
# Start the main processing loop
|
|
42
|
+
record_processor = SampleRecordProcessor.new
|
|
43
|
+
driver = Aws::KCLrb::KCLProcess.new(record_processor)
|
|
44
|
+
driver.run
|
|
45
|
+
end
|
|
46
46
|
```
|
|
47
47
|
|
|
48
48
|
## Before You Get Started
|
|
@@ -60,16 +60,6 @@ access.
|
|
|
60
60
|
For questions regarding [Amazon Kinesis Service][amazon-kinesis] and the client libraries please check the
|
|
61
61
|
[official documentation][amazon-kinesis-docs] as well as the [Amazon Kinesis Forums][kinesis-forum].
|
|
62
62
|
|
|
63
|
-
## 🚨Important: Migration to Ruby KCL 2.2.0 or later with MultiLangDaemon - Credential Provider Changes Required
|
|
64
|
-
Java KCL version 2.7.0 and later uses AWS SDK for Java 2.x instead of AWS SDK for Java 1.x. For the KCL Ruby 2.x versions,
|
|
65
|
-
v2.2.0 is the first Ruby release to use Java KCL 2.7.0. All MultiLangDaemon users upgrading from earlier versions must update
|
|
66
|
-
their credential provider configuration in the `.properties` file to use credentials provider name for AWS SDK for Java 2.x.
|
|
67
|
-
Failure to do this will cause your multilang KCL application to fail during startup with credential provider construction errors.
|
|
68
|
-
Please check the following link for the credentials provider mapping and MultiLangDaemon credentials provider configuration guide
|
|
69
|
-
|
|
70
|
-
- [AWS SDK for Java 1.x to 2.x Credentials Provider Mapping](aws.amazon.com/sdk-for-java/latest/developer-guide/migration-client-credentials.html#credentials-changes-mapping)
|
|
71
|
-
- [KCL Multilang Credentials Provider Configuration Guide](https://github.com/aws/amazon-kinesis-client/blob/master/docs/multilang/configuring-credential-providers.md)
|
|
72
|
-
|
|
73
63
|
## Running the Sample
|
|
74
64
|
|
|
75
65
|
Using the Amazon KCL for Ruby package requires the [MultiLangDaemon][multi-lang-daemon] which
|
|
@@ -95,20 +85,28 @@ The following defaults are used in the sample application:
|
|
|
95
85
|
|
|
96
86
|
### Running the Data Producer
|
|
97
87
|
|
|
98
|
-
To run the data producer, run the following commands:
|
|
88
|
+
To run the data producer indefinitely, run the following commands:
|
|
99
89
|
|
|
100
90
|
```sh
|
|
101
91
|
cd samples
|
|
102
92
|
rake run_producer
|
|
103
93
|
```
|
|
104
94
|
|
|
95
|
+
To run the data producer with a timeout, run the following commands:
|
|
96
|
+
|
|
97
|
+
```sh
|
|
98
|
+
cd samples
|
|
99
|
+
rake "run_producer[t]"
|
|
100
|
+
```
|
|
101
|
+
* *note: replace t with number of seconds to run the producer*
|
|
102
|
+
|
|
105
103
|
#### Notes
|
|
106
104
|
|
|
107
|
-
* The [AWS Ruby SDK gem][aws-ruby-sdk-gem] needs to be installed as a pre-requisite. To install,
|
|
105
|
+
* The [AWS Ruby SDK gem][aws-ruby-sdk-gem] for Kinesis needs to be installed as a pre-requisite. To install,
|
|
108
106
|
run:
|
|
109
107
|
|
|
110
108
|
```sh
|
|
111
|
-
sudo gem install aws-sdk
|
|
109
|
+
sudo gem install aws-sdk-kinesis
|
|
112
110
|
```
|
|
113
111
|
|
|
114
112
|
* The script `samples/sample_kcl_producer.rb` takes several parameters that you can use
|
|
@@ -196,27 +194,50 @@ all languages.
|
|
|
196
194
|
* The [Amazon Kinesis Forum][kinesis-forum]
|
|
197
195
|
|
|
198
196
|
## Release Notes
|
|
199
|
-
### Release
|
|
200
|
-
* [#
|
|
201
|
-
* [#
|
|
202
|
-
* [#
|
|
203
|
-
* [#
|
|
204
|
-
* [#
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
*
|
|
209
|
-
|
|
210
|
-
### Release
|
|
211
|
-
|
|
212
|
-
*
|
|
213
|
-
*
|
|
214
|
-
|
|
215
|
-
*
|
|
216
|
-
*
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
*
|
|
197
|
+
### Release 3.1.0 (October 1, 2025)
|
|
198
|
+
* [#108](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/108) Add multi-lang support for leaseAssignmentIntervalMillis
|
|
199
|
+
* [#116](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/116) Upgrade amazon-kinesis-client from 3.0.0 to 3.1.3
|
|
200
|
+
* [#116](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/116) Upgrade aws-sdk from 2.25.64 to 2.33.0
|
|
201
|
+
* [#116](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/116) Upgrade netty.version from 4.1.108.Final to 4.2.4.Final
|
|
202
|
+
* [#116](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/116) Upgrade fasterxml-jackson from 2.13.5 to 2.15.0
|
|
203
|
+
* [#116](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/116) Upgrade checker-qual from 2.5.2 to 3.49.4
|
|
204
|
+
* [#116](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/116) Upgrade org.apache.commons:commons-lang3 from 3.14.0 to 3.18.0
|
|
205
|
+
* [#93](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/93) Upgrade logback.version from 1.3.14 to 1.13.15
|
|
206
|
+
* [#96](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/96) Upgrade commons-beanutils from 1.9.4 to 1.11.0
|
|
207
|
+
|
|
208
|
+
### Release 3.0.0 (November 6, 2024)
|
|
209
|
+
* New lease assignment / load balancing algorithm
|
|
210
|
+
* KCL 3.x introduces a new lease assignment and load balancing algorithm. It assigns leases among workers based on worker utilization metrics and throughput on each lease, replacing the previous lease count-based lease assignment algorithm.
|
|
211
|
+
* When KCL detects higher variance in CPU utilization among workers, it proactively reassigns leases from over-utilized workers to under-utilized workers for even load balancing. This ensures even CPU utilization across workers and removes the need to over-provision the stream processing compute hosts.
|
|
212
|
+
* Optimized DynamoDB RCU usage
|
|
213
|
+
* KCL 3.x optimizes DynamoDB read capacity unit (RCU) usage on the lease table by implementing a global secondary index with leaseOwner as the partition key. This index mirrors the leaseKey attribute from the base lease table, allowing workers to efficiently discover their assigned leases by querying the index instead of scanning the entire table.
|
|
214
|
+
* This approach significantly reduces read operations compared to earlier KCL versions, where workers performed full table scans, resulting in higher RCU consumption.
|
|
215
|
+
* Graceful lease handoff
|
|
216
|
+
* KCL 3.x introduces a feature called "graceful lease handoff" to minimize data reprocessing during lease reassignments. Graceful lease handoff allows the current worker to complete checkpointing of processed records before transferring the lease to another worker. For graceful lease handoff, you should implement checkpointing logic within the existing `shutdownRequested()` method.
|
|
217
|
+
* This feature is enabled by default in KCL 3.x, but you can turn off this feature by adjusting the configuration property `isGracefulLeaseHandoffEnabled`.
|
|
218
|
+
* While this approach significantly reduces the probability of data reprocessing during lease transfers, it doesn't completely eliminate the possibility. To maintain data integrity and consistency, it's crucial to design your downstream consumer applications to be idempotent. This ensures that the application can handle potential duplicate record processing without adverse effects.
|
|
219
|
+
* New DynamoDB metadata management artifacts
|
|
220
|
+
* KCL 3.x introduces two new DynamoDB tables for improved lease management:
|
|
221
|
+
* Worker metrics table: Records CPU utilization metrics from each worker. KCL uses these metrics for optimal lease assignments, balancing resource utilization across workers. If CPU utilization metric is not available, KCL assigns leases to balance the total sum of shard throughput per worker instead.
|
|
222
|
+
* Coordinator state table: Stores internal state information for workers. Used to coordinate in-place migration from KCL 2.x to KCL 3.x and leader election among workers.
|
|
223
|
+
* Follow this [documentation](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html#kcl-migration-from-2-3-IAM-permissions) to add required IAM permissions for your KCL application.
|
|
224
|
+
* Other improvements and changes
|
|
225
|
+
* Dependency on the AWS SDK for Java 1.x has been fully removed.
|
|
226
|
+
* The Glue Schema Registry integration functionality no longer depends on AWS SDK for Java 1.x. Previously, it required this as a transient dependency.
|
|
227
|
+
* Multilangdaemon has been upgraded to use AWS SDK for Java 2.x. It no longer depends on AWS SDK for Java 1.x.
|
|
228
|
+
* `idleTimeBetweenReadsInMillis` (PollingConfig) now has a minimum default value of 200.
|
|
229
|
+
* This polling configuration property determines the [publishers](https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client/src/main/java/software/amazon/kinesis/retrieval/polling/PrefetchRecordsPublisher.java) wait time between GetRecords calls in both success and failure cases. Previously, setting this value below 200 caused unnecessary throttling. This is because Amazon Kinesis Data Streams supports up to five read transactions per second per shard for shared-throughput consumers.
|
|
230
|
+
* Shard lifecycle management is improved to deal with edge cases around shard splits and merges to ensure records continue being processed as expected.
|
|
231
|
+
* Migration
|
|
232
|
+
* The programming interfaces of KCL 3.x remain identical with KCL 2.x for an easier migration. For detailed migration instructions, please refer to the [Migrate consumers from KCL 2.x to KCL 3.x](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration-from-2-3.html) page in the Amazon Kinesis Data Streams developer guide.
|
|
233
|
+
* Configuration properties
|
|
234
|
+
* New configuration properties introduced in KCL 3.x are listed in this [doc](https://github.com/awslabs/amazon-kinesis-client/blob/master/docs/kcl-configurations.md#new-configurations-in-kcl-3x).
|
|
235
|
+
* Deprecated configuration properties in KCL 3.x are listed in this [doc](https://github.com/awslabs/amazon-kinesis-client/blob/master/docs/kcl-configurations.md#discontinued-configuration-properties-in-kcl-3x). You need to keep the deprecated configuration properties during the migration from any previous KCL version to KCL 3.x.
|
|
236
|
+
* Metrics
|
|
237
|
+
* New CloudWatch metrics introduced in KCL 3.x are explained in the [Monitor the Kinesis Client Library with Amazon CloudWatch](https://docs.aws.amazon.com/streams/latest/dev/monitoring-with-kcl.html) in the Amazon Kinesis Data Streams developer guide. The following operations are newly added in KCL 3.x:
|
|
238
|
+
* `LeaseAssignmentManager`
|
|
239
|
+
* `WorkerMetricStatsReporter`
|
|
240
|
+
* `LeaseDiscovery`
|
|
220
241
|
|
|
221
242
|
### Release 2.1.1 (February 21, 2023)
|
|
222
243
|
* [#69](https://github.com/awslabs/amazon-kinesis-client-ruby/pull/69) Include `pom.xml` in the gemspec
|
|
@@ -263,7 +284,7 @@ all languages.
|
|
|
263
284
|
[amazon-kcl]: http://docs.aws.amazon.com/kinesis/latest/dev/kinesis-record-processor-app.html
|
|
264
285
|
[amazon-kcl-github]: https://github.com/awslabs/amazon-kinesis-client
|
|
265
286
|
[amazon-kinesis-python-github]: https://github.com/awslabs/amazon-kinesis-client-python
|
|
266
|
-
[multi-lang-daemon]: https://github.com/awslabs/amazon-kinesis-client/blob/master/src/main/java/
|
|
287
|
+
[multi-lang-daemon]: https://github.com/awslabs/amazon-kinesis-client/blob/master/amazon-kinesis-client-multilang/src/main/java/software/amazon/kinesis/multilang/package-info.java
|
|
267
288
|
[DefaultCredentialsProvider]: https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html
|
|
268
289
|
[kinesis-forum]: http://developer.amazonwebservices.com/connect/forum.jspa?forumID=169
|
|
269
290
|
[aws-ruby-sdk-gem]: https://rubygems.org/gems/aws-sdk
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
3.1.0
|
|
@@ -52,8 +52,8 @@ module Aws
|
|
|
52
52
|
# @param action [Hash] A hash that represents an action to take with
|
|
53
53
|
# appropriate attributes, as retrieved from {IOProxy#read_action}, e.g.
|
|
54
54
|
#
|
|
55
|
-
# - `{"action":"initialize","shardId":"shardId-123"}`
|
|
56
|
-
# - `{"action":"processRecords","records":[{"data":"bWVvdw==","partitionKey":"cat","sequenceNumber":"456"}]}`
|
|
55
|
+
# - `{"action":"initialize","shardId":"shardId-123","sequenceNumber":123}`
|
|
56
|
+
# - `{"action":"processRecords","records":[{"data":"bWVvdw==","partitionKey":"cat","sequenceNumber":"456"}],"millisBehindLatest":"0"}`
|
|
57
57
|
# - `{"action":"shutdown","reason":"TERMINATE"}`
|
|
58
58
|
# @raise [MalformedAction] if the action is missing expected attributes.
|
|
59
59
|
def process_action(action)
|
data/pom.xml
CHANGED
|
@@ -3,12 +3,10 @@
|
|
|
3
3
|
<modelVersion>4.0.0</modelVersion>
|
|
4
4
|
<properties>
|
|
5
5
|
<awssdk.version>2.33.0</awssdk.version>
|
|
6
|
-
<
|
|
7
|
-
<kcl.version>2.7.2</kcl.version>
|
|
6
|
+
<kcl.version>3.1.3</kcl.version>
|
|
8
7
|
<netty.version>4.2.4.Final</netty.version>
|
|
9
8
|
<netty-reactive.version>2.0.6</netty-reactive.version>
|
|
10
9
|
<fasterxml-jackson.version>2.15.0</fasterxml-jackson.version>
|
|
11
|
-
<!--do not upgrade to logback version 1.5.x, as this is not compatible with JDK 8-->
|
|
12
10
|
<logback.version>1.3.15</logback.version>
|
|
13
11
|
</properties>
|
|
14
12
|
<dependencies>
|
|
@@ -32,6 +30,18 @@
|
|
|
32
30
|
<artifactId>dynamodb</artifactId>
|
|
33
31
|
<version>${awssdk.version}</version>
|
|
34
32
|
</dependency>
|
|
33
|
+
<!-- https://mvnrepository.com/artifact/software.amazon.awssdk/dynamodb-enhanced -->
|
|
34
|
+
<dependency>
|
|
35
|
+
<groupId>software.amazon.awssdk</groupId>
|
|
36
|
+
<artifactId>dynamodb-enhanced</artifactId>
|
|
37
|
+
<version>${awssdk.version}</version>
|
|
38
|
+
</dependency>
|
|
39
|
+
<!-- https://mvnrepository.com/artifact/com.amazonaws/dynamodb-lock-client -->
|
|
40
|
+
<dependency>
|
|
41
|
+
<groupId>com.amazonaws</groupId>
|
|
42
|
+
<artifactId>dynamodb-lock-client</artifactId>
|
|
43
|
+
<version>1.3.0</version>
|
|
44
|
+
</dependency>
|
|
35
45
|
<dependency>
|
|
36
46
|
<groupId>software.amazon.awssdk</groupId>
|
|
37
47
|
<artifactId>cloudwatch</artifactId>
|
|
@@ -265,7 +275,7 @@
|
|
|
265
275
|
<dependency>
|
|
266
276
|
<groupId>org.checkerframework</groupId>
|
|
267
277
|
<artifactId>checker-qual</artifactId>
|
|
268
|
-
<version>
|
|
278
|
+
<version>3.49.4</version>
|
|
269
279
|
</dependency>
|
|
270
280
|
<dependency>
|
|
271
281
|
<groupId>com.google.errorprone</groupId>
|
|
@@ -342,16 +352,6 @@
|
|
|
342
352
|
<artifactId>httpcore</artifactId>
|
|
343
353
|
<version>4.4.15</version>
|
|
344
354
|
</dependency>
|
|
345
|
-
<dependency>
|
|
346
|
-
<groupId>com.amazonaws</groupId>
|
|
347
|
-
<artifactId>aws-java-sdk-core</artifactId>
|
|
348
|
-
<version>${aws-java-sdk.version}</version>
|
|
349
|
-
</dependency>
|
|
350
|
-
<dependency>
|
|
351
|
-
<groupId>com.amazonaws</groupId>
|
|
352
|
-
<artifactId>aws-java-sdk-sts</artifactId>
|
|
353
|
-
<version>${aws-java-sdk.version}</version>
|
|
354
|
-
</dependency>
|
|
355
355
|
<dependency>
|
|
356
356
|
<groupId>com.amazon.ion</groupId>
|
|
357
357
|
<artifactId>ion-java</artifactId>
|
|
@@ -361,6 +361,12 @@
|
|
|
361
361
|
<groupId>software.amazon.glue</groupId>
|
|
362
362
|
<artifactId>schema-registry-serde</artifactId>
|
|
363
363
|
<version>1.1.19</version>
|
|
364
|
+
<exclusions>
|
|
365
|
+
<exclusion>
|
|
366
|
+
<groupId>com.amazonaws</groupId>
|
|
367
|
+
<artifactId>aws-java-sdk-sts</artifactId>
|
|
368
|
+
</exclusion>
|
|
369
|
+
</exclusions>
|
|
364
370
|
</dependency>
|
|
365
371
|
<dependency>
|
|
366
372
|
<groupId>joda-time</groupId>
|
|
@@ -400,7 +406,7 @@
|
|
|
400
406
|
<dependency>
|
|
401
407
|
<groupId>commons-beanutils</groupId>
|
|
402
408
|
<artifactId>commons-beanutils</artifactId>
|
|
403
|
-
<version>1.
|
|
409
|
+
<version>1.11.0</version>
|
|
404
410
|
</dependency>
|
|
405
411
|
<dependency>
|
|
406
412
|
<groupId>commons-collections</groupId>
|
data/spec/kcl_process_spec.rb
CHANGED
|
@@ -27,66 +27,124 @@ module Aws::KCLrb
|
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
29
|
|
|
30
|
-
def
|
|
31
|
-
checkpointer.checkpoint
|
|
30
|
+
def shutdown_requested(checkpointer)
|
|
31
|
+
checkpointer.checkpoint
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
class TestRecordProcessorV2 < Aws::KCLrb::V2::RecordProcessorBase
|
|
36
|
+
|
|
37
|
+
def init_processor(shard_id)
|
|
38
|
+
# no-op
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def shutdown_requested(shutdown_requested_input)
|
|
42
|
+
shutdown_requested_input.checkpointer.checkpoint
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def process_records(process_records_input)
|
|
46
|
+
last_seq = nil
|
|
47
|
+
records = process_records_input.records
|
|
48
|
+
records.each do |record|
|
|
49
|
+
last_seq = record['sequenceNumber']
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
if last_seq
|
|
53
|
+
checkpointer = process_records_input.checkpointer
|
|
54
|
+
begin
|
|
55
|
+
checkpointer.checkpoint(last_seq)
|
|
56
|
+
rescue CheckpointError => cpe
|
|
57
|
+
if cpe.value == 'ThrottlingException'
|
|
58
|
+
checkpointer.checkpoint(last_seq)
|
|
59
|
+
else
|
|
60
|
+
raise
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
end
|
|
67
|
+
|
|
35
68
|
describe KCLProcess do
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
69
|
+
|
|
70
|
+
input_specs = [
|
|
71
|
+
{:method => :init_processor, :action => 'initialize', :input => '{"action":"initialize","shardId":"shard-000001","sequenceNumber":123456789}'},
|
|
72
|
+
{:method => :process_records, :action => 'processRecords', :input => '{"action":"processRecords","records":[],"millisBehindLatest":0}'},
|
|
73
|
+
{:method => :shutdown, :action => 'shutdownRequested', :input => '{"action":"shutdownRequested","reason":"TERMINATE"}'}
|
|
74
|
+
]
|
|
75
|
+
versions = [1, 2]
|
|
76
|
+
|
|
77
|
+
describe "#action_response" do
|
|
78
|
+
|
|
79
|
+
before(:each) do
|
|
80
|
+
@processor = double(RecordProcessorBase)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
input_specs.each do |input_spec|
|
|
84
|
+
versions.each do |version|
|
|
85
|
+
it "V#{version} should respond to #{input_spec[:method]} action by invoking the corresponding processor's method and write a status message to the output IO" do
|
|
86
|
+
allow(@processor).to receive(:version).and_return(version)
|
|
87
|
+
action_response_common_test(input_spec[:method], input_spec[:action], input_spec[:input], version)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def action_response_common_test(method, action, input, version)
|
|
93
|
+
if method == :shutdown
|
|
94
|
+
expected_shutdown_input = version == 1 ? Aws::KCLrb::CheckpointerImpl : Aws::KCLrb::V2::ShutdownRequestedInput
|
|
95
|
+
allow(@processor).to receive(:shutdown_requested).with(expected_shutdown_input).and_return(nil)
|
|
96
|
+
else
|
|
97
|
+
expect(@processor).to receive(method).once
|
|
98
|
+
end
|
|
99
|
+
input = StringIO.new(input)
|
|
48
100
|
output = StringIO.new
|
|
49
101
|
error = StringIO.new
|
|
50
|
-
driver = KCLProcess.new(processor, input, output, error)
|
|
102
|
+
driver = KCLProcess.new(@processor, input, output, error)
|
|
51
103
|
driver.run
|
|
52
104
|
|
|
53
|
-
expected_output = %Q[{"action":"status","responseFor":"#{
|
|
54
|
-
expect(
|
|
55
|
-
expect(
|
|
56
|
-
expect(
|
|
105
|
+
expected_output = %Q[{"action":"status","responseFor":"#{action}"}]
|
|
106
|
+
expect(output.string.gsub(/\s+/, "")).to eq(expected_output.gsub(/\s+/, ""))
|
|
107
|
+
expect(error.string).to eq("")
|
|
108
|
+
expect(input.eof?).to eq(true)
|
|
57
109
|
end
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
110
|
+
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
describe "#run" do
|
|
114
|
+
versions.each do |version|
|
|
115
|
+
it "V#{version} should process a normal stream of actions and produce expected output" do
|
|
116
|
+
input_string = <<-INPUT
|
|
117
|
+
{"action":"initialize","shardId":"shardId-123","sequenceNumber":123}
|
|
118
|
+
{"action":"processRecords","records":[{"data":"bWVvdw==","partitionKey":"cat","sequenceNumber":"456"}],"millisBehindLatest":"0"}
|
|
62
119
|
{"action":"checkpoint","sequenceNumber":"456","error":"ThrottlingException"}
|
|
63
120
|
{"action":"checkpoint","sequenceNumber":"456"}
|
|
64
|
-
{"action":"
|
|
121
|
+
{"action":"shutdownRequested","reason":"TERMINATE"}
|
|
65
122
|
{"action":"checkpoint","sequenceNumber":"456"}
|
|
66
|
-
|
|
123
|
+
INPUT
|
|
67
124
|
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
125
|
+
# NOTE: The first checkpoint is expected to fail
|
|
126
|
+
# with a ThrottlingException and hence the
|
|
127
|
+
# retry.
|
|
128
|
+
expected_output_string = <<-OUTPUT
|
|
72
129
|
{"action":"status","responseFor":"initialize"}
|
|
73
130
|
{"action":"checkpoint","sequenceNumber":"456"}
|
|
74
131
|
{"action":"checkpoint","sequenceNumber":"456"}
|
|
75
132
|
{"action":"status","responseFor":"processRecords"}
|
|
76
133
|
{"action":"checkpoint","sequenceNumber":null}
|
|
77
|
-
{"action":"status","responseFor":"
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
134
|
+
{"action":"status","responseFor":"shutdownRequested"}
|
|
135
|
+
OUTPUT
|
|
136
|
+
processor = version == 1 ? TestRecordProcessor.new : TestRecordProcessorV2.new
|
|
137
|
+
input = StringIO.new(input_string)
|
|
138
|
+
output = StringIO.new
|
|
139
|
+
error = StringIO.new
|
|
140
|
+
driver = KCLProcess.new(processor, input, output, error)
|
|
141
|
+
driver.run
|
|
85
142
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
143
|
+
# outputs should be same modulo some whitespaces
|
|
144
|
+
expect( output.string.gsub(/\s+/, "") ).to eq(expected_output_string.gsub(/\s+/, ""))
|
|
145
|
+
expect( error.string ).to eq("")
|
|
146
|
+
expect( input.eof? ).to eq(true)
|
|
147
|
+
end
|
|
90
148
|
end
|
|
91
149
|
end
|
|
92
150
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: aws-kclrb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Amazon Web Services
|
|
@@ -24,6 +24,34 @@ dependencies:
|
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '1.0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: rspec
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: simplecov
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - ">="
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '0'
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - ">="
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '0'
|
|
27
55
|
description: A ruby interface for the Amazon Kinesis Client Library MultiLangDaemon
|
|
28
56
|
email:
|
|
29
57
|
executables: []
|