rdkafka 0.13.0 → 0.14.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/workflows/ci.yml +58 -0
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +130 -111
- data/{LICENSE → MIT-LICENSE} +2 -1
- data/README.md +19 -20
- data/certs/cert_chain.pem +26 -0
- data/docker-compose.yml +16 -15
- data/ext/README.md +1 -1
- data/ext/Rakefile +1 -1
- data/lib/rdkafka/abstract_handle.rb +40 -26
- data/lib/rdkafka/admin.rb +6 -7
- data/lib/rdkafka/bindings.rb +8 -5
- data/lib/rdkafka/config.rb +30 -17
- data/lib/rdkafka/consumer/headers.rb +2 -4
- data/lib/rdkafka/consumer/topic_partition_list.rb +3 -1
- data/lib/rdkafka/consumer.rb +92 -53
- data/lib/rdkafka/helpers/time.rb +14 -0
- data/lib/rdkafka/metadata.rb +22 -1
- data/lib/rdkafka/native_kafka.rb +6 -1
- data/lib/rdkafka/producer.rb +85 -7
- data/lib/rdkafka/version.rb +3 -3
- data/lib/rdkafka.rb +10 -1
- data/rdkafka.gemspec +17 -3
- data/renovate.json +6 -0
- data/spec/rdkafka/abstract_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/create_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_topic_handle_spec.rb +0 -2
- data/spec/rdkafka/admin/delete_topic_report_spec.rb +0 -2
- data/spec/rdkafka/admin_spec.rb +1 -2
- data/spec/rdkafka/bindings_spec.rb +0 -1
- data/spec/rdkafka/callbacks_spec.rb +0 -2
- data/spec/rdkafka/config_spec.rb +0 -2
- data/spec/rdkafka/consumer/headers_spec.rb +0 -2
- data/spec/rdkafka/consumer/message_spec.rb +0 -2
- data/spec/rdkafka/consumer/partition_spec.rb +0 -2
- data/spec/rdkafka/consumer/topic_partition_list_spec.rb +19 -2
- data/spec/rdkafka/consumer_spec.rb +143 -39
- data/spec/rdkafka/error_spec.rb +0 -2
- data/spec/rdkafka/metadata_spec.rb +2 -3
- data/spec/rdkafka/native_kafka_spec.rb +2 -3
- data/spec/rdkafka/producer/delivery_handle_spec.rb +0 -2
- data/spec/rdkafka/producer/delivery_report_spec.rb +0 -2
- data/spec/rdkafka/producer_spec.rb +157 -1
- data.tar.gz.sig +0 -0
- metadata +52 -13
- metadata.gz.sig +0 -0
- data/.semaphore/semaphore.yml +0 -27
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 59f5b664693b87cd66340f821027e45a86c70c0f652c91a0ec6826ba69e02969
|
4
|
+
data.tar.gz: 79e734db66b41d1618a581686df428de75912d06d1e41fe9e2340e63ace3d0da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cf9821c62c9f958196d78cbe2d8f50b860d06084017cfc3fbf3ffd6c289278d8a8b54f3b40d16d3d3e6c1d71f1112de897cce67cfc09071cb8fa5a0895bd5ed7
|
7
|
+
data.tar.gz: a8e9fab48d749ca62fcdfeda7c2a3d2f85cff7849338cedbe43bbcb8d9300979c0ec44c59a49de8f4efd2ee9a3d7feb3a1fa718f2311c89c6b5aedd99c6d870c
|
checksums.yaml.gz.sig
ADDED
Binary file
|
@@ -0,0 +1,58 @@
|
|
1
|
+
name: ci
|
2
|
+
|
3
|
+
concurrency:
|
4
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
5
|
+
cancel-in-progress: true
|
6
|
+
|
7
|
+
on:
|
8
|
+
pull_request:
|
9
|
+
push:
|
10
|
+
schedule:
|
11
|
+
- cron: '0 1 * * *'
|
12
|
+
|
13
|
+
env:
|
14
|
+
BUNDLE_RETRY: 6
|
15
|
+
BUNDLE_JOBS: 4
|
16
|
+
|
17
|
+
jobs:
|
18
|
+
specs:
|
19
|
+
timeout-minutes: 30
|
20
|
+
runs-on: ubuntu-latest
|
21
|
+
strategy:
|
22
|
+
fail-fast: false
|
23
|
+
matrix:
|
24
|
+
ruby:
|
25
|
+
- '3.3.0-preview2'
|
26
|
+
- '3.2'
|
27
|
+
- '3.1'
|
28
|
+
- '3.1.0'
|
29
|
+
- '3.0'
|
30
|
+
- '3.0.0'
|
31
|
+
- '2.7'
|
32
|
+
- '2.7.0'
|
33
|
+
include:
|
34
|
+
- ruby: '3.2'
|
35
|
+
coverage: 'true'
|
36
|
+
steps:
|
37
|
+
- uses: actions/checkout@v4
|
38
|
+
- name: Install package dependencies
|
39
|
+
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
40
|
+
|
41
|
+
- name: Start Kafka with docker-compose
|
42
|
+
run: |
|
43
|
+
docker-compose up -d || (sleep 5 && docker-compose up -d)
|
44
|
+
|
45
|
+
- name: Set up Ruby
|
46
|
+
uses: ruby/setup-ruby@v1
|
47
|
+
with:
|
48
|
+
ruby-version: ${{matrix.ruby}}
|
49
|
+
bundler-cache: true
|
50
|
+
|
51
|
+
- name: Run all specs
|
52
|
+
env:
|
53
|
+
GITHUB_COVERAGE: ${{matrix.coverage}}
|
54
|
+
|
55
|
+
run: |
|
56
|
+
bundle install --path vendor/bundle
|
57
|
+
cd ext && bundle exec rake && cd ..
|
58
|
+
bundle exec rspec
|
data/.gitignore
CHANGED
data/.rspec
CHANGED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rdkafka-ruby
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
3.2.2
|
data/CHANGELOG.md
CHANGED
@@ -1,122 +1,141 @@
|
|
1
|
-
#
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
#
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
1
|
+
# Rdkafka Changelog
|
2
|
+
|
3
|
+
## 0.14.0 (2023-11-21)
|
4
|
+
- [Enhancement] Add `raise_response_error` flag to the `Rdkafka::AbstractHandle`.
|
5
|
+
- [Enhancement] Allow for setting `statistics_callback` as nil to reset predefined settings configured by a different gem (mensfeld)
|
6
|
+
- [Enhancement] Get consumer position (thijsc & mensfeld)
|
7
|
+
- [Enhancement] Provide `#purge` to remove any outstanding requests from the producer (mensfeld)
|
8
|
+
- [Enhancement] Update `librdkafka` to `2.2.0` (mensfeld)
|
9
|
+
- [Enhancement] Introduce producer partitions count metadata cache (mensfeld)
|
10
|
+
- [Enhancement] Increase metadata timeout request from `250 ms` to `2000 ms` default to allow for remote cluster operations via `rdkafka-ruby` (mensfeld)
|
11
|
+
- [Enhancement] Introduce `#name` for producers and consumers (mensfeld)
|
12
|
+
- [Enhancement] Include backtrace in non-raised binded errors (mensfeld)
|
13
|
+
- [Fix] Reference to Opaque is not released when Admin, Consumer or Producer is closed (mensfeld)
|
14
|
+
- [Fix] Trigger `#poll` on native kafka creation to handle oauthbearer cb (mensfeld)
|
15
|
+
- [Fix] `#flush` does not handle the timeouts errors by making it return `true` if all flushed or `false` if failed. We do **not** raise an exception here to keep it backwards compatible (mensfeld)
|
16
|
+
- [Change] Remove support for Ruby 2.6 due to it being EOL and WeakMap incompatibilities (mensfeld)
|
17
|
+
- [Change] Update Kafka Docker with Confluent KRaft (mensfeld)
|
18
|
+
- [Change] Update librdkafka repo reference from edenhill to confluentinc (mensfeld)
|
19
|
+
|
20
|
+
## 0.13.0 (2023-07-24)
|
21
|
+
- Support cooperative sticky partition assignment in the rebalance callback (methodmissing)
|
22
|
+
- Support both string and symbol header keys (ColinDKelley)
|
23
|
+
- Handle tombstone messages properly (kgalieva)
|
24
|
+
- Add topic name to delivery report (maeve)
|
25
|
+
- Allow string partitioner config (mollyegibson)
|
26
|
+
- Fix documented type for DeliveryReport#error (jimmydo)
|
27
|
+
- Bump librdkafka to 2.0.2 (lmaia)
|
28
|
+
- Use finalizers to cleanly exit producer and admin (thijsc)
|
29
|
+
- Lock access to the native kafka client (thijsc)
|
30
|
+
- Fix potential race condition in multi-threaded producer (mensfeld)
|
31
|
+
- Fix leaking FFI resources in specs (mensfeld)
|
32
|
+
- Improve specs stability (mensfeld)
|
33
|
+
- Make metadata request timeout configurable (mensfeld)
|
34
|
+
- call_on_partitions_assigned and call_on_partitions_revoked only get a tpl passed in (thijsc)
|
35
|
+
|
36
|
+
## 0.12.0 (2022-06-17)
|
37
|
+
- Bumps librdkafka to 1.9.0
|
38
|
+
- Fix crash on empty partition key (mensfeld)
|
39
|
+
- Pass the delivery handle to the callback (gvisokinskas)
|
40
|
+
|
41
|
+
## 0.11.0 (2021-11-17)
|
42
|
+
- Upgrade librdkafka to 1.8.2
|
43
|
+
- Bump supported minimum Ruby version to 2.6
|
44
|
+
- Better homebrew path detection
|
45
|
+
|
46
|
+
## 0.10.0 (2021-09-07)
|
47
|
+
- Upgrade librdkafka to 1.5.0
|
48
|
+
- Add error callback config
|
49
|
+
|
50
|
+
## 0.9.0 (2021-06-23)
|
51
|
+
- Fixes for Ruby 3.0
|
52
|
+
- Allow any callable object for callbacks (gremerritt)
|
53
|
+
- Reduce memory allocations in Rdkafka::Producer#produce (jturkel)
|
54
|
+
- Use queue as log callback to avoid unsafe calls from trap context (breunigs)
|
55
|
+
- Allow passing in topic configuration on create_topic (dezka)
|
56
|
+
- Add each_batch method to consumer (mgrosso)
|
57
|
+
|
58
|
+
## 0.8.1 (2020-12-07)
|
59
|
+
- Fix topic_flag behaviour and add tests for Metadata (geoff2k)
|
60
|
+
- Add topic admin interface (geoff2k)
|
61
|
+
- Raise an exception if @native_kafka is nil (geoff2k)
|
62
|
+
- Option to use zstd compression (jasonmartens)
|
63
|
+
|
64
|
+
## 0.8.0 (2020-06-02)
|
65
|
+
- Upgrade librdkafka to 1.4.0
|
66
|
+
- Integrate librdkafka metadata API and add partition_key (by Adithya-copart)
|
67
|
+
- Ruby 2.7 compatibility fix (by Geoff Thé)A
|
68
|
+
- Add error to delivery report (by Alex Stanovsky)
|
69
|
+
- Don't override CPPFLAGS and LDFLAGS if already set on Mac (by Hiroshi Hatake)
|
70
|
+
- Allow use of Rake 13.x and up (by Tomasz Pajor)
|
71
|
+
|
72
|
+
## 0.7.0 (2019-09-21)
|
73
|
+
- Bump librdkafka to 1.2.0 (by rob-as)
|
74
|
+
- Allow customizing the wait time for delivery report availability (by mensfeld)
|
75
|
+
|
76
|
+
## 0.6.0 (2019-07-23)
|
77
|
+
- Bump librdkafka to 1.1.0 (by Chris Gaffney)
|
78
|
+
- Implement seek (by breunigs)
|
79
|
+
|
80
|
+
## 0.5.0 (2019-04-11)
|
81
|
+
- Bump librdkafka to 1.0.0 (by breunigs)
|
82
|
+
- Add cluster and member information (by dmexe)
|
83
|
+
- Support message headers for consumer & producer (by dmexe)
|
84
|
+
- Add consumer rebalance listener (by dmexe)
|
85
|
+
- Implement pause/resume partitions (by dmexe)
|
86
|
+
|
87
|
+
## 0.4.2 (2019-01-12)
|
88
|
+
- Delivery callback for producer
|
89
|
+
- Document list param of commit method
|
90
|
+
- Use default Homebrew openssl location if present
|
91
|
+
- Consumer lag handles empty topics
|
92
|
+
- End iteration in consumer when it is closed
|
93
|
+
- Add support for storing message offsets
|
94
|
+
- Add missing runtime dependency to rake
|
95
|
+
|
96
|
+
## 0.4.1 (2018-10-19)
|
97
|
+
- Bump librdkafka to 0.11.6
|
98
|
+
|
99
|
+
## 0.4.0 (2018-09-24)
|
100
|
+
- Improvements in librdkafka archive download
|
101
|
+
- Add global statistics callback
|
102
|
+
- Use Time for timestamps, potentially breaking change if you
|
84
103
|
rely on the previous behavior where it returns an integer with
|
85
104
|
the number of milliseconds.
|
86
|
-
|
87
|
-
|
105
|
+
- Bump librdkafka to 0.11.5
|
106
|
+
- Implement TopicPartitionList in Ruby so we don't have to keep
|
88
107
|
track of native objects.
|
89
|
-
|
90
|
-
|
108
|
+
- Support committing a topic partition list
|
109
|
+
- Add consumer assignment method
|
91
110
|
|
92
|
-
|
93
|
-
|
94
|
-
|
111
|
+
## 0.3.5 (2018-01-17)
|
112
|
+
- Fix crash when not waiting for delivery handles
|
113
|
+
- Run specs on Ruby 2.5
|
95
114
|
|
96
|
-
|
97
|
-
|
115
|
+
## 0.3.4 (2017-12-05)
|
116
|
+
- Bump librdkafka to 0.11.3
|
98
117
|
|
99
|
-
|
100
|
-
|
118
|
+
## 0.3.3 (2017-10-27)
|
119
|
+
- Fix bug that prevent display of `RdkafkaError` message
|
101
120
|
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
121
|
+
## 0.3.2 (2017-10-25)
|
122
|
+
- `add_topic` now supports using a partition count
|
123
|
+
- Add way to make errors clearer with an extra message
|
124
|
+
- Show topics in subscribe error message
|
125
|
+
- Show partition and topic in query watermark offsets error message
|
107
126
|
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
127
|
+
## 0.3.1 (2017-10-23)
|
128
|
+
- Bump librdkafka to 0.11.1
|
129
|
+
- Officially support ranges in `add_topic` for topic partition list.
|
130
|
+
- Add consumer lag calculator
|
112
131
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
132
|
+
## 0.3.0 (2017-10-17)
|
133
|
+
- Move both add topic methods to one `add_topic` in `TopicPartitionList`
|
134
|
+
- Add committed offsets to consumer
|
135
|
+
- Add query watermark offset to consumer
|
117
136
|
|
118
|
-
|
119
|
-
|
137
|
+
## 0.2.0 (2017-10-13)
|
138
|
+
- Some refactoring and add inline documentation
|
120
139
|
|
121
|
-
|
122
|
-
|
140
|
+
## 0.1.x (2017-09-10)
|
141
|
+
- Initial working version including producing and consuming
|
data/{LICENSE → MIT-LICENSE}
RENAMED
@@ -1,6 +1,7 @@
|
|
1
1
|
The MIT License (MIT)
|
2
2
|
|
3
|
-
Copyright (c) 2017 Thijs Cadier
|
3
|
+
Copyright (c) 2017-2023 Thijs Cadier
|
4
|
+
2023, Maciej Mensfeld
|
4
5
|
|
5
6
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
7
|
of this software and associated documentation files (the "Software"), to deal
|
data/README.md
CHANGED
@@ -1,26 +1,25 @@
|
|
1
1
|
# Rdkafka
|
2
2
|
|
3
|
-
[![Build Status](https://
|
3
|
+
[![Build Status](https://github.com/karafka/rdkafka-ruby/actions/workflows/ci.yml/badge.svg)](https://github.com/karafka/rdkafka-ruby/actions/workflows/ci.yml)
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/rdkafka.svg)](https://badge.fury.io/rb/rdkafka)
|
5
|
-
[![
|
5
|
+
[![Join the chat at https://slack.karafka.io](https://raw.githubusercontent.com/karafka/misc/master/slack.svg)](https://slack.karafka.io)
|
6
|
+
|
7
|
+
> [!NOTE]
|
8
|
+
> The `rdkafka-ruby` gem was created and developed by [AppSignal](https://www.appsignal.com/). Their impactful contributions have significantly shaped the Ruby Kafka and Karafka ecosystems. For robust monitoring, we highly recommend AppSignal.
|
9
|
+
|
10
|
+
---
|
6
11
|
|
7
12
|
The `rdkafka` gem is a modern Kafka client library for Ruby based on
|
8
|
-
[librdkafka](https://github.com/
|
13
|
+
[librdkafka](https://github.com/confluentinc/librdkafka/).
|
9
14
|
It wraps the production-ready C client using the [ffi](https://github.com/ffi/ffi)
|
10
|
-
gem and targets Kafka 1.0+ and Ruby versions
|
11
|
-
active maintenance. We remove Ruby version from our CI builds
|
15
|
+
gem and targets Kafka 1.0+ and Ruby versions under security or
|
16
|
+
active maintenance. We remove a Ruby version from our CI builds when they
|
12
17
|
become EOL.
|
13
18
|
|
14
|
-
`rdkafka` was written because
|
15
|
-
Kafka that supports modern Kafka at [AppSignal](https://appsignal.com).
|
16
|
-
We run it in production on very high traffic systems.
|
17
|
-
|
18
|
-
This gem only provides a high-level Kafka consumer. If you are running
|
19
|
-
an older version of Kafka and/or need the legacy simple consumer we
|
20
|
-
suggest using the [Hermann](https://github.com/reiseburo/hermann) gem.
|
19
|
+
`rdkafka` was written because of the need for a reliable Ruby client for Kafka that supports modern Kafka at [AppSignal](https://appsignal.com). AppSignal runs it in production on very high-traffic systems.
|
21
20
|
|
22
21
|
The most important pieces of a Kafka client are implemented. We're
|
23
|
-
working towards feature completeness
|
22
|
+
working towards feature completeness. You can track that here:
|
24
23
|
https://github.com/appsignal/rdkafka-ruby/milestone/1
|
25
24
|
|
26
25
|
## Table of content
|
@@ -39,7 +38,7 @@ https://github.com/appsignal/rdkafka-ruby/milestone/1
|
|
39
38
|
## Installation
|
40
39
|
|
41
40
|
This gem downloads and compiles librdkafka when it is installed. If you
|
42
|
-
have any problems installing the gem please open an issue.
|
41
|
+
If you have any problems installing the gem, please open an issue.
|
43
42
|
|
44
43
|
## Usage
|
45
44
|
|
@@ -65,9 +64,9 @@ end
|
|
65
64
|
|
66
65
|
### Producing messages
|
67
66
|
|
68
|
-
Produce a number of messages, put the delivery handles in an array and
|
67
|
+
Produce a number of messages, put the delivery handles in an array, and
|
69
68
|
wait for them before exiting. This way the messages will be batched and
|
70
|
-
sent to Kafka
|
69
|
+
efficiently sent to Kafka.
|
71
70
|
|
72
71
|
```ruby
|
73
72
|
config = {:"bootstrap.servers" => "localhost:9092"}
|
@@ -92,7 +91,7 @@ released until it `#close` is explicitly called, so be sure to call
|
|
92
91
|
|
93
92
|
## Higher level libraries
|
94
93
|
|
95
|
-
Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher
|
94
|
+
Currently, there are two actively developed frameworks based on rdkafka-ruby, that provide higher-level API that can be used to work with Kafka messages and one library for publishing messages.
|
96
95
|
|
97
96
|
### Message processing frameworks
|
98
97
|
|
@@ -105,7 +104,7 @@ Currently, there are two actively developed frameworks based on rdkafka-ruby, th
|
|
105
104
|
|
106
105
|
## Development
|
107
106
|
|
108
|
-
A Docker Compose file is included to run Kafka
|
107
|
+
A Docker Compose file is included to run Kafka. To run
|
109
108
|
that:
|
110
109
|
|
111
110
|
```
|
@@ -123,7 +122,7 @@ DEBUG_PRODUCER=true bundle exec rspec
|
|
123
122
|
DEBUG_CONSUMER=true bundle exec rspec
|
124
123
|
```
|
125
124
|
|
126
|
-
After running the tests you can bring the cluster down to start with a
|
125
|
+
After running the tests, you can bring the cluster down to start with a
|
127
126
|
clean slate:
|
128
127
|
|
129
128
|
```
|
@@ -132,7 +131,7 @@ docker-compose down
|
|
132
131
|
|
133
132
|
## Example
|
134
133
|
|
135
|
-
To see everything working run these in separate tabs:
|
134
|
+
To see everything working, run these in separate tabs:
|
136
135
|
|
137
136
|
```
|
138
137
|
bundle exec rake consume_messages
|
@@ -0,0 +1,26 @@
|
|
1
|
+
-----BEGIN CERTIFICATE-----
|
2
|
+
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
3
|
+
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
4
|
+
MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
|
5
|
+
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
6
|
+
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
|
7
|
+
8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
|
8
|
+
sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
|
9
|
+
wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
|
10
|
+
b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
|
11
|
+
ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
|
12
|
+
5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
|
13
|
+
YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
|
14
|
+
kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
|
15
|
+
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
|
16
|
+
1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
17
|
+
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
|
18
|
+
bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
|
19
|
+
3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
|
20
|
+
kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
|
21
|
+
IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
|
22
|
+
fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
|
23
|
+
qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
|
24
|
+
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
25
|
+
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
26
|
+
-----END CERTIFICATE-----
|
data/docker-compose.yml
CHANGED
@@ -1,24 +1,25 @@
|
|
1
|
-
---
|
2
|
-
|
3
1
|
version: '2'
|
4
2
|
|
5
3
|
services:
|
6
|
-
zookeeper:
|
7
|
-
image: confluentinc/cp-zookeeper:5.2.6
|
8
|
-
environment:
|
9
|
-
ZOOKEEPER_CLIENT_PORT: 2181
|
10
|
-
ZOOKEEPER_TICK_TIME: 2000
|
11
|
-
|
12
4
|
kafka:
|
13
|
-
|
14
|
-
|
15
|
-
|
5
|
+
container_name: kafka
|
6
|
+
image: confluentinc/cp-kafka:7.5.2
|
7
|
+
|
16
8
|
ports:
|
17
9
|
- 9092:9092
|
10
|
+
|
18
11
|
environment:
|
19
|
-
|
20
|
-
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
|
21
|
-
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092,PLAINTEXT_HOST://localhost:9092
|
22
|
-
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
|
12
|
+
CLUSTER_ID: kafka-docker-cluster-1
|
23
13
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
24
14
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
15
|
+
KAFKA_PROCESS_ROLES: broker,controller
|
16
|
+
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
|
17
|
+
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
|
18
|
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
|
19
|
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
|
20
|
+
KAFKA_BROKER_ID: 1
|
21
|
+
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
|
22
|
+
ALLOW_PLAINTEXT_LISTENER: 'yes'
|
23
|
+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
24
|
+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
25
|
+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
data/ext/README.md
CHANGED
@@ -5,7 +5,7 @@ this gem is installed.
|
|
5
5
|
|
6
6
|
To update the `librdkafka` version follow the following steps:
|
7
7
|
|
8
|
-
* Go to https://github.com/
|
8
|
+
* Go to https://github.com/confluentinc/librdkafka/releases to get the new
|
9
9
|
version number and asset checksum for `tar.gz`.
|
10
10
|
* Change the version in `lib/rdkafka/version.rb`
|
11
11
|
* Change the `sha256` in `lib/rdkafka/version.rb`
|
data/ext/Rakefile
CHANGED
@@ -17,7 +17,7 @@ task :default => :clean do
|
|
17
17
|
end
|
18
18
|
|
19
19
|
recipe.files << {
|
20
|
-
:url => "https://codeload.github.com/
|
20
|
+
:url => "https://codeload.github.com/confluentinc/librdkafka/tar.gz/v#{Rdkafka::LIBRDKAFKA_VERSION}",
|
21
21
|
:sha256 => Rdkafka::LIBRDKAFKA_SOURCE_SHA256
|
22
22
|
}
|
23
23
|
recipe.configure_options = ["--host=#{recipe.host}"]
|
@@ -1,28 +1,37 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "ffi"
|
4
|
-
|
5
3
|
module Rdkafka
|
4
|
+
# This class serves as an abstract base class to represent handles within the Rdkafka module.
|
5
|
+
# As a subclass of `FFI::Struct`, this class provides a blueprint for other specific handle
|
6
|
+
# classes to inherit from, ensuring they adhere to a particular structure and behavior.
|
7
|
+
#
|
8
|
+
# Subclasses must define their own layout, and the layout must start with:
|
9
|
+
#
|
10
|
+
# layout :pending, :bool,
|
11
|
+
# :response, :int
|
6
12
|
class AbstractHandle < FFI::Struct
|
7
|
-
|
8
|
-
#
|
9
|
-
# layout :pending, :bool,
|
10
|
-
# :response, :int
|
13
|
+
include Helpers::Time
|
11
14
|
|
15
|
+
# Registry for registering all the handles.
|
12
16
|
REGISTRY = {}
|
13
17
|
|
14
|
-
|
15
|
-
|
16
|
-
|
18
|
+
class << self
|
19
|
+
# Adds handle to the register
|
20
|
+
#
|
21
|
+
# @param handle [AbstractHandle] any handle we want to register
|
22
|
+
def register(handle)
|
23
|
+
address = handle.to_ptr.address
|
24
|
+
REGISTRY[address] = handle
|
25
|
+
end
|
17
26
|
|
18
|
-
|
19
|
-
|
20
|
-
|
27
|
+
# Removes handle from the register based on the handle address
|
28
|
+
#
|
29
|
+
# @param address [Integer] address of the registered handle we want to remove
|
30
|
+
def remove(address)
|
31
|
+
REGISTRY.delete(address)
|
32
|
+
end
|
21
33
|
end
|
22
34
|
|
23
|
-
def self.remove(address)
|
24
|
-
REGISTRY.delete(address)
|
25
|
-
end
|
26
35
|
|
27
36
|
# Whether the handle is still pending.
|
28
37
|
#
|
@@ -32,29 +41,34 @@ module Rdkafka
|
|
32
41
|
end
|
33
42
|
|
34
43
|
# Wait for the operation to complete or raise an error if this takes longer than the timeout.
|
35
|
-
# If there is a timeout this does not mean the operation failed, rdkafka might still be working
|
36
|
-
# In this case it is possible to call wait again.
|
44
|
+
# If there is a timeout this does not mean the operation failed, rdkafka might still be working
|
45
|
+
# on the operation. In this case it is possible to call wait again.
|
46
|
+
#
|
47
|
+
# @param max_wait_timeout [Numeric, nil] Amount of time to wait before timing out.
|
48
|
+
# If this is nil it does not time out.
|
49
|
+
# @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the
|
50
|
+
# operation has completed
|
51
|
+
# @param raise_response_error [Boolean] should we raise error when waiting finishes
|
37
52
|
#
|
38
|
-
# @
|
39
|
-
# @param wait_timeout [Numeric] Amount of time we should wait before we recheck if the operation has completed
|
53
|
+
# @return [Object] Operation-specific result
|
40
54
|
#
|
41
55
|
# @raise [RdkafkaError] When the operation failed
|
42
56
|
# @raise [WaitTimeoutError] When the timeout has been reached and the handle is still pending
|
43
|
-
|
44
|
-
# @return [Object] Operation-specific result
|
45
|
-
def wait(max_wait_timeout: 60, wait_timeout: 0.1)
|
57
|
+
def wait(max_wait_timeout: 60, wait_timeout: 0.1, raise_response_error: true)
|
46
58
|
timeout = if max_wait_timeout
|
47
|
-
|
59
|
+
monotonic_now + max_wait_timeout
|
48
60
|
else
|
49
61
|
nil
|
50
62
|
end
|
51
63
|
loop do
|
52
64
|
if pending?
|
53
|
-
if timeout && timeout <=
|
54
|
-
raise WaitTimeoutError.new(
|
65
|
+
if timeout && timeout <= monotonic_now
|
66
|
+
raise WaitTimeoutError.new(
|
67
|
+
"Waiting for #{operation_name} timed out after #{max_wait_timeout} seconds"
|
68
|
+
)
|
55
69
|
end
|
56
70
|
sleep wait_timeout
|
57
|
-
elsif self[:response] != 0
|
71
|
+
elsif self[:response] != 0 && raise_response_error
|
58
72
|
raise_error
|
59
73
|
else
|
60
74
|
return create_result
|