rdkafka 0.22.0.beta1-arm64-darwin → 0.22.2-arm64-darwin
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci_linux_x86_64_gnu.yml +66 -44
- data/.github/workflows/ci_linux_x86_64_musl.yml +51 -62
- data/.github/workflows/ci_macos_arm64.yml +23 -45
- data/.github/workflows/push_linux_x86_64_gnu.yml +2 -1
- data/.github/workflows/push_linux_x86_64_musl.yml +3 -1
- data/.github/workflows/push_macos_arm64.yml +1 -1
- data/.github/workflows/push_ruby.yml +1 -1
- data/.ruby-version +1 -1
- data/CHANGELOG.md +12 -2
- data/README.md +2 -2
- data/ext/librdkafka.dylib +0 -0
- data/lib/rdkafka/bindings.rb +6 -7
- data/lib/rdkafka/producer.rb +11 -6
- data/lib/rdkafka/version.rb +1 -1
- data/rdkafka.gemspec +0 -2
- data/spec/rdkafka/admin_spec.rb +202 -1
- data/spec/rdkafka/bindings_spec.rb +0 -24
- data/spec/rdkafka/producer_spec.rb +295 -2
- metadata +2 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 9f41b963f3f92bafd7bd74d9bbb8c41dc94a7300f367e343053f3a3d6664ecd5
|
|
4
|
+
data.tar.gz: c0d09c53795ea421306c322d04b0d7678b69e47ca13c71857facd026ddebdbe5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 13548a65cfd73faccb3f9db6352bca0e18b50a8102cc4180797741be78a5d4d6f3c5c21ab29a9344d0b96da7ff4c26dc0bea09ad0fe3079f6426db852a121ee9
|
|
7
|
+
data.tar.gz: 28bd25f43c673a2378501e49feaa3d546e7371f280414bf592cb617eef7050542a82af58f2274235eb048f0d66a2c5321859df2e1eef28e81b70e6812e8b9616
|
|
@@ -1,3 +1,37 @@
|
|
|
1
|
+
# CI Strategy: Comprehensive Testing of Build and Precompiled Flows
|
|
2
|
+
#
|
|
3
|
+
# This workflow tests both compilation-from-source and precompiled binary distribution
|
|
4
|
+
# strategies across multiple Ubuntu and Ruby versions to ensure broad compatibility and
|
|
5
|
+
# reliability.
|
|
6
|
+
#
|
|
7
|
+
# WHY WE TEST BOTH UBUNTU 22.04 AND 24.04:
|
|
8
|
+
# - Different system library versions (OpenSSL, zlib, libsasl2, libzstd, etc.)
|
|
9
|
+
# - Different GCC compiler versions that affect native extension compilation
|
|
10
|
+
# - Different glibc versions that can impact binary compatibility
|
|
11
|
+
# - Real-world deployment scenarios where users run on various Ubuntu LTS versions
|
|
12
|
+
# - Different Ruby versions
|
|
13
|
+
#
|
|
14
|
+
# COMPILATION FLOW (build_install + specs_install):
|
|
15
|
+
# - Tests that librdkafka compiles correctly from source on each Ubuntu version
|
|
16
|
+
# - Validates that mini_portile2 can successfully build native dependencies
|
|
17
|
+
# - Ensures Ruby native extensions link properly with system libraries
|
|
18
|
+
# - Verifies that the same codebase works across different toolchain versions
|
|
19
|
+
#
|
|
20
|
+
# PRECOMPILED FLOW (build_precompiled + specs_precompiled):
|
|
21
|
+
# - Tests our precompiled static libraries work on different Ubuntu versions
|
|
22
|
+
# - Validates that statically-linked binaries are truly portable across environments
|
|
23
|
+
# - Ensures precompiled libraries don't have unexpected system dependencies
|
|
24
|
+
# - Verifies that removing build tools doesn't break precompiled binary usage
|
|
25
|
+
#
|
|
26
|
+
# ARTIFACT ISOLATION:
|
|
27
|
+
# - Each Ubuntu version gets separate artifacts (rdkafka-built-gem-22.04, etc.)
|
|
28
|
+
# - Prevents cross-contamination of OS-specific compiled extensions
|
|
29
|
+
# - Ensures test accuracy by matching build and test environments
|
|
30
|
+
#
|
|
31
|
+
# This comprehensive approach catches issues that single-platform testing would miss,
|
|
32
|
+
# such as system library incompatibilities, compiler-specific bugs, or static linking
|
|
33
|
+
# problems that only manifest on specific Ubuntu versions.
|
|
34
|
+
|
|
1
35
|
name: CI Linux x86_64 GNU
|
|
2
36
|
|
|
3
37
|
concurrency:
|
|
@@ -20,41 +54,8 @@ env:
|
|
|
20
54
|
BUNDLE_JOBS: 4
|
|
21
55
|
|
|
22
56
|
jobs:
|
|
23
|
-
build_install:
|
|
24
|
-
timeout-minutes: 30
|
|
25
|
-
runs-on: ubuntu-latest
|
|
26
|
-
steps:
|
|
27
|
-
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
28
|
-
with:
|
|
29
|
-
fetch-depth: 0
|
|
30
|
-
- name: Install package dependencies
|
|
31
|
-
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
|
32
|
-
- name: Set up Ruby
|
|
33
|
-
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
|
|
34
|
-
with:
|
|
35
|
-
ruby-version: '3.4' # Use one Ruby version for building
|
|
36
|
-
bundler-cache: false
|
|
37
|
-
- name: Build gem with mini_portile
|
|
38
|
-
run: |
|
|
39
|
-
set -e
|
|
40
|
-
bundle install
|
|
41
|
-
cd ext && bundle exec rake
|
|
42
|
-
cd ..
|
|
43
|
-
- name: Upload built gem and bundle
|
|
44
|
-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
45
|
-
with:
|
|
46
|
-
name: rdkafka-built-gem
|
|
47
|
-
path: |
|
|
48
|
-
vendor/bundle/
|
|
49
|
-
.bundle/
|
|
50
|
-
ext/
|
|
51
|
-
lib/
|
|
52
|
-
retention-days: 1
|
|
53
|
-
|
|
54
57
|
specs_install:
|
|
55
58
|
timeout-minutes: 30
|
|
56
|
-
runs-on: ubuntu-latest
|
|
57
|
-
needs: build_install
|
|
58
59
|
strategy:
|
|
59
60
|
fail-fast: false
|
|
60
61
|
matrix:
|
|
@@ -65,20 +66,17 @@ jobs:
|
|
|
65
66
|
- '3.2'
|
|
66
67
|
- '3.1'
|
|
67
68
|
- 'jruby-10.0'
|
|
69
|
+
ubuntu-version: ['22.04', '24.04']
|
|
68
70
|
include:
|
|
69
71
|
- ruby: '3.4'
|
|
70
72
|
coverage: 'true'
|
|
71
73
|
- ruby: 'jruby-10.0'
|
|
72
74
|
continue-on-error: true
|
|
75
|
+
runs-on: ubuntu-${{ matrix.ubuntu-version }}
|
|
73
76
|
steps:
|
|
74
77
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
75
78
|
with:
|
|
76
79
|
fetch-depth: 0
|
|
77
|
-
- name: Download built gem
|
|
78
|
-
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
79
|
-
with:
|
|
80
|
-
name: rdkafka-built-gem
|
|
81
|
-
path: ./
|
|
82
80
|
- name: Set up Ruby
|
|
83
81
|
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
|
|
84
82
|
with:
|
|
@@ -108,12 +106,17 @@ jobs:
|
|
|
108
106
|
echo "Sleeping 2 seconds..."
|
|
109
107
|
sleep 2
|
|
110
108
|
done
|
|
111
|
-
- name: Install
|
|
109
|
+
- name: Install dependencies
|
|
112
110
|
env:
|
|
113
111
|
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
|
|
114
112
|
run: |
|
|
115
113
|
# Only install gems that aren't Ruby-version specific
|
|
116
114
|
bundle install
|
|
115
|
+
- name: Build gem with mini_portile
|
|
116
|
+
run: |
|
|
117
|
+
set -e
|
|
118
|
+
cd ext && bundle exec rake
|
|
119
|
+
cd ..
|
|
117
120
|
- name: Run all specs
|
|
118
121
|
env:
|
|
119
122
|
GITHUB_COVERAGE: ${{matrix.coverage}}
|
|
@@ -124,7 +127,10 @@ jobs:
|
|
|
124
127
|
|
|
125
128
|
build_precompiled:
|
|
126
129
|
timeout-minutes: 30
|
|
127
|
-
|
|
130
|
+
# We precompile on older Ubuntu and check compatibility by running specs since we aim to
|
|
131
|
+
# release only one precompiled version for all supported Ubuntu versions
|
|
132
|
+
# This is why we do not want Renovate to update it automatically
|
|
133
|
+
runs-on: ubuntu-22.04 # renovate: ignore
|
|
128
134
|
steps:
|
|
129
135
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
130
136
|
with:
|
|
@@ -135,15 +141,29 @@ jobs:
|
|
|
135
141
|
sudo apt-get install -y --no-install-recommends \
|
|
136
142
|
build-essential \
|
|
137
143
|
gcc \
|
|
144
|
+
g++ \
|
|
138
145
|
make \
|
|
139
|
-
patch \
|
|
140
146
|
tar \
|
|
147
|
+
gzip \
|
|
141
148
|
wget \
|
|
149
|
+
curl \
|
|
150
|
+
file \
|
|
151
|
+
pkg-config \
|
|
152
|
+
autoconf \
|
|
153
|
+
automake \
|
|
154
|
+
libtool \
|
|
155
|
+
python3 \
|
|
156
|
+
git \
|
|
142
157
|
ca-certificates \
|
|
158
|
+
patch \
|
|
143
159
|
libsasl2-dev \
|
|
144
160
|
libssl-dev \
|
|
145
161
|
zlib1g-dev \
|
|
146
|
-
libzstd-dev
|
|
162
|
+
libzstd-dev \
|
|
163
|
+
bison \
|
|
164
|
+
flex \
|
|
165
|
+
perl \
|
|
166
|
+
binutils-dev
|
|
147
167
|
- name: Cache build-tmp directory
|
|
148
168
|
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
|
149
169
|
with:
|
|
@@ -162,7 +182,6 @@ jobs:
|
|
|
162
182
|
|
|
163
183
|
specs_precompiled:
|
|
164
184
|
timeout-minutes: 30
|
|
165
|
-
runs-on: ubuntu-latest
|
|
166
185
|
needs: build_precompiled
|
|
167
186
|
strategy:
|
|
168
187
|
fail-fast: false
|
|
@@ -173,9 +192,13 @@ jobs:
|
|
|
173
192
|
- '3.3'
|
|
174
193
|
- '3.2'
|
|
175
194
|
- '3.1'
|
|
195
|
+
ubuntu-version: ['22.04', '24.04']
|
|
176
196
|
include:
|
|
177
197
|
- ruby: '3.4'
|
|
198
|
+
ubuntu-version: '24.04'
|
|
178
199
|
coverage: 'true'
|
|
200
|
+
|
|
201
|
+
runs-on: ubuntu-${{ matrix.ubuntu-version }}
|
|
179
202
|
steps:
|
|
180
203
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
181
204
|
with:
|
|
@@ -199,7 +222,6 @@ jobs:
|
|
|
199
222
|
|
|
200
223
|
echo "=== Container status ==="
|
|
201
224
|
docker compose ps kafka
|
|
202
|
-
|
|
203
225
|
for i in {1..30}; do
|
|
204
226
|
echo "=== Attempt $i/30 ==="
|
|
205
227
|
|
|
@@ -1,3 +1,26 @@
|
|
|
1
|
+
# Why We Build and Run Without Caching Native Extensions
|
|
2
|
+
#
|
|
3
|
+
# We intentionally compile the native librdkafka library fresh in each test job
|
|
4
|
+
# rather than caching or pre-building it for several reasons:
|
|
5
|
+
#
|
|
6
|
+
# 1. Architecture Compatibility
|
|
7
|
+
# - Pre-built native libraries (.so files) are architecture-specific
|
|
8
|
+
# - Can cause "Exec format error" when build/runtime environments differ
|
|
9
|
+
# - Building in the same container guarantees compatibility
|
|
10
|
+
#
|
|
11
|
+
# 2. Container Image Variations
|
|
12
|
+
# - Different Ruby Alpine images may have subtle differences in:
|
|
13
|
+
# * Base system libraries, compiler toolchains, musl libc versions
|
|
14
|
+
# - These differences can cause pre-built libraries to fail at runtime
|
|
15
|
+
#
|
|
16
|
+
# 3. Simplicity and Reliability
|
|
17
|
+
# - Single source of truth: everything builds and runs in same environment
|
|
18
|
+
# - No artifact management complexity or potential upload/download failures
|
|
19
|
+
# - Easier debugging when issues are contained in one job
|
|
20
|
+
#
|
|
21
|
+
# Trade-offs: Slightly longer CI times (~2-3 min per job) but much more reliable
|
|
22
|
+
# than dealing with architecture mismatches and artifact corruption issues.
|
|
23
|
+
|
|
1
24
|
name: CI Linux x86_64 musl
|
|
2
25
|
|
|
3
26
|
concurrency:
|
|
@@ -20,70 +43,39 @@ env:
|
|
|
20
43
|
BUNDLE_JOBS: 4
|
|
21
44
|
|
|
22
45
|
jobs:
|
|
23
|
-
build_install:
|
|
24
|
-
timeout-minutes: 30
|
|
25
|
-
runs-on: ubuntu-latest
|
|
26
|
-
container:
|
|
27
|
-
image: alpine:3.22@sha256:8a1f59ffb675680d47db6337b49d22281a139e9d709335b492be023728e11715
|
|
28
|
-
steps:
|
|
29
|
-
- name: Install dependencies
|
|
30
|
-
run: |
|
|
31
|
-
apk add --no-cache git curl ca-certificates build-base linux-headers \
|
|
32
|
-
pkgconf perl autoconf automake libtool bison flex file \
|
|
33
|
-
ruby ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev cyrus-sasl-dev
|
|
34
|
-
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
35
|
-
with:
|
|
36
|
-
fetch-depth: 0
|
|
37
|
-
- name: Configure git safe directory
|
|
38
|
-
run: git config --global --add safe.directory /__w/karafka-rdkafka/karafka-rdkafka
|
|
39
|
-
- name: Build gem with mini_portile
|
|
40
|
-
run: |
|
|
41
|
-
set -e
|
|
42
|
-
bundle config set --local path 'vendor/bundle'
|
|
43
|
-
bundle install
|
|
44
|
-
cd ext && bundle exec rake
|
|
45
|
-
cd ..
|
|
46
|
-
- name: Upload built gem and bundle
|
|
47
|
-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
48
|
-
with:
|
|
49
|
-
name: rdkafka-built-gem-musl
|
|
50
|
-
path: |
|
|
51
|
-
vendor/bundle/
|
|
52
|
-
.bundle/
|
|
53
|
-
ext/
|
|
54
|
-
lib/
|
|
55
|
-
retention-days: 1
|
|
56
|
-
|
|
57
46
|
specs_install:
|
|
58
47
|
timeout-minutes: 30
|
|
59
48
|
runs-on: ubuntu-latest
|
|
60
|
-
needs: build_install
|
|
61
49
|
strategy:
|
|
62
50
|
fail-fast: false
|
|
63
51
|
matrix:
|
|
64
|
-
ruby:
|
|
65
|
-
- '3.4'
|
|
66
|
-
- '3.3'
|
|
67
|
-
- '3.2'
|
|
68
|
-
- '3.1'
|
|
69
52
|
include:
|
|
53
|
+
- ruby: '3.1'
|
|
54
|
+
alpine_version: '3.21'
|
|
55
|
+
- ruby: '3.2'
|
|
56
|
+
alpine_version: '3.21'
|
|
57
|
+
- ruby: '3.2'
|
|
58
|
+
alpine_version: '3.22'
|
|
59
|
+
- ruby: '3.3'
|
|
60
|
+
alpine_version: '3.21'
|
|
61
|
+
- ruby: '3.3'
|
|
62
|
+
alpine_version: '3.22'
|
|
63
|
+
- ruby: '3.4'
|
|
64
|
+
alpine_version: '3.21'
|
|
65
|
+
coverage: 'true'
|
|
70
66
|
- ruby: '3.4'
|
|
67
|
+
alpine_version: '3.22'
|
|
71
68
|
coverage: 'true'
|
|
72
69
|
steps:
|
|
73
70
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
74
71
|
with:
|
|
75
72
|
fetch-depth: 0
|
|
76
|
-
|
|
77
|
-
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
78
|
-
with:
|
|
79
|
-
name: rdkafka-built-gem-musl
|
|
80
|
-
path: ./
|
|
73
|
+
|
|
81
74
|
- name: Start Kafka with Docker Compose
|
|
82
75
|
run: |
|
|
83
76
|
docker compose up -d
|
|
84
77
|
echo "Waiting for Kafka to be ready..."
|
|
85
78
|
sleep 10
|
|
86
|
-
|
|
87
79
|
for i in {1..30}; do
|
|
88
80
|
if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then
|
|
89
81
|
echo "Kafka topics command succeeded!"
|
|
@@ -91,39 +83,36 @@ jobs:
|
|
|
91
83
|
fi
|
|
92
84
|
sleep 2
|
|
93
85
|
done
|
|
86
|
+
|
|
94
87
|
- name: Run all specs
|
|
95
88
|
env:
|
|
96
89
|
GITHUB_COVERAGE: ${{ matrix.coverage }}
|
|
97
|
-
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
|
|
98
90
|
run: |
|
|
99
91
|
docker run --rm \
|
|
100
92
|
--network host \
|
|
101
93
|
-v "${{ github.workspace }}:/workspace" \
|
|
102
94
|
-w /workspace \
|
|
103
95
|
-e "GITHUB_COVERAGE=${{ matrix.coverage }}" \
|
|
104
|
-
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
cyrus-sasl-login \
|
|
109
|
-
cyrus-sasl-crammd5 \
|
|
110
|
-
|
|
111
|
-
cyrus-sasl-gssapiv2 \
|
|
112
|
-
cyrus-sasl-scram \
|
|
113
|
-
krb5-libs \
|
|
114
|
-
openssl \
|
|
115
|
-
zlib \
|
|
116
|
-
zlib-dev \
|
|
117
|
-
zstd-libs && \
|
|
96
|
+
ruby:${{ matrix.ruby }}-alpine${{ matrix.alpine_version }} \
|
|
97
|
+
sh -c 'apk add --no-cache git curl ca-certificates build-base linux-headers \
|
|
98
|
+
pkgconf perl autoconf automake libtool bison flex file \
|
|
99
|
+
ruby-dev ruby-bundler bash zstd-dev zlib zlib-dev openssl-dev \
|
|
100
|
+
cyrus-sasl-dev cyrus-sasl cyrus-sasl-login \
|
|
101
|
+
cyrus-sasl-crammd5 cyrus-sasl-digestmd5 cyrus-sasl-gssapiv2 cyrus-sasl-scram \
|
|
102
|
+
krb5-libs openssl zlib zstd-libs && \
|
|
118
103
|
git config --global --add safe.directory /workspace && \
|
|
119
104
|
bundle config set --local path vendor/bundle && \
|
|
120
105
|
bundle install && \
|
|
106
|
+
cd ext && bundle exec rake && \
|
|
107
|
+
cd .. && \
|
|
121
108
|
bundle exec ruby -S rspec'
|
|
109
|
+
|
|
122
110
|
build_precompiled:
|
|
123
111
|
timeout-minutes: 45
|
|
124
112
|
runs-on: ubuntu-latest
|
|
125
113
|
container:
|
|
126
|
-
|
|
114
|
+
# Similar to GNU, we build on the oldest for ABI compatibility
|
|
115
|
+
image: alpine:3.18@sha256:de0eb0b3f2a47ba1eb89389859a9bd88b28e82f5826b6969ad604979713c2d4f # renovate: ignore
|
|
127
116
|
steps:
|
|
128
117
|
- name: Install dependencies
|
|
129
118
|
run: |
|
|
@@ -22,44 +22,8 @@ env:
|
|
|
22
22
|
CONFLUENT_VERSION: "8.0.0"
|
|
23
23
|
|
|
24
24
|
jobs:
|
|
25
|
-
build_install:
|
|
26
|
-
timeout-minutes: 30
|
|
27
|
-
runs-on: macos-latest
|
|
28
|
-
steps:
|
|
29
|
-
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
30
|
-
with:
|
|
31
|
-
fetch-depth: 0
|
|
32
|
-
- name: Install Bash 4+ and Kerberos
|
|
33
|
-
run: |
|
|
34
|
-
brew install bash
|
|
35
|
-
brew list krb5 &>/dev/null || brew install krb5
|
|
36
|
-
echo "/opt/homebrew/bin" >> $GITHUB_PATH
|
|
37
|
-
- name: Set up Ruby
|
|
38
|
-
uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
|
|
39
|
-
with:
|
|
40
|
-
ruby-version: '3.4' # Use one Ruby version for building
|
|
41
|
-
bundler-cache: false
|
|
42
|
-
- name: Build gem with mini_portile
|
|
43
|
-
run: |
|
|
44
|
-
set -e
|
|
45
|
-
bundle install
|
|
46
|
-
cd ext && bundle exec rake
|
|
47
|
-
cd ..
|
|
48
|
-
- name: Upload built gem and bundle
|
|
49
|
-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
|
50
|
-
with:
|
|
51
|
-
name: rdkafka-built-gem-macos
|
|
52
|
-
path: |
|
|
53
|
-
vendor/bundle/
|
|
54
|
-
.bundle/
|
|
55
|
-
ext/
|
|
56
|
-
lib/
|
|
57
|
-
retention-days: 1
|
|
58
|
-
|
|
59
25
|
specs_install:
|
|
60
26
|
timeout-minutes: 30
|
|
61
|
-
runs-on: macos-latest
|
|
62
|
-
needs: build_install
|
|
63
27
|
strategy:
|
|
64
28
|
fail-fast: false
|
|
65
29
|
matrix:
|
|
@@ -69,18 +33,21 @@ jobs:
|
|
|
69
33
|
- '3.3'
|
|
70
34
|
- '3.2'
|
|
71
35
|
- '3.1'
|
|
36
|
+
macos-version:
|
|
37
|
+
- 'macos-14' # macOS 14 Sonoma (ARM64)
|
|
38
|
+
- 'macos-15' # macOS 15 Sequoia (ARM64)
|
|
72
39
|
include:
|
|
73
40
|
- ruby: '3.4'
|
|
41
|
+
macos-version: 'macos-15'
|
|
74
42
|
coverage: 'true'
|
|
43
|
+
exclude:
|
|
44
|
+
- ruby: '3.5.0-preview1'
|
|
45
|
+
macos-version: 'macos-14'
|
|
46
|
+
runs-on: ${{ matrix.macos-version }}
|
|
75
47
|
steps:
|
|
76
48
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
77
49
|
with:
|
|
78
50
|
fetch-depth: 0
|
|
79
|
-
- name: Download built gem
|
|
80
|
-
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
|
81
|
-
with:
|
|
82
|
-
name: rdkafka-built-gem-macos
|
|
83
|
-
path: ./
|
|
84
51
|
- name: Install Bash 4+ and Kerberos
|
|
85
52
|
run: |
|
|
86
53
|
brew install bash
|
|
@@ -153,12 +120,16 @@ jobs:
|
|
|
153
120
|
[ $i -eq 30 ] && { echo "❌ Kafka failed to start"; exit 1; }
|
|
154
121
|
sleep 2
|
|
155
122
|
done
|
|
156
|
-
- name: Install
|
|
123
|
+
- name: Install dependencies
|
|
157
124
|
env:
|
|
158
125
|
RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext
|
|
159
126
|
run: |
|
|
160
|
-
# Only install gems that aren't Ruby-version specific
|
|
161
127
|
bundle install
|
|
128
|
+
- name: Build gem with mini_portile
|
|
129
|
+
run: |
|
|
130
|
+
set -e
|
|
131
|
+
cd ext && bundle exec rake
|
|
132
|
+
cd ..
|
|
162
133
|
- name: Run all specs
|
|
163
134
|
env:
|
|
164
135
|
GITHUB_COVERAGE: ${{matrix.coverage}}
|
|
@@ -168,7 +139,7 @@ jobs:
|
|
|
168
139
|
|
|
169
140
|
build_precompiled:
|
|
170
141
|
timeout-minutes: 45
|
|
171
|
-
runs-on: macos-
|
|
142
|
+
runs-on: macos-14
|
|
172
143
|
steps:
|
|
173
144
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
174
145
|
with:
|
|
@@ -196,7 +167,6 @@ jobs:
|
|
|
196
167
|
|
|
197
168
|
specs_precompiled:
|
|
198
169
|
timeout-minutes: 30
|
|
199
|
-
runs-on: macos-latest
|
|
200
170
|
needs: build_precompiled
|
|
201
171
|
strategy:
|
|
202
172
|
fail-fast: false
|
|
@@ -207,9 +177,17 @@ jobs:
|
|
|
207
177
|
- '3.3'
|
|
208
178
|
- '3.2'
|
|
209
179
|
- '3.1'
|
|
180
|
+
macos-version:
|
|
181
|
+
- 'macos-14'
|
|
182
|
+
- 'macos-15'
|
|
210
183
|
include:
|
|
211
184
|
- ruby: '3.4'
|
|
185
|
+
macos-version: 'macos-15'
|
|
212
186
|
coverage: 'true'
|
|
187
|
+
exclude:
|
|
188
|
+
- ruby: '3.5.0-preview1'
|
|
189
|
+
macos-version: 'macos-14'
|
|
190
|
+
runs-on: ${{ matrix.macos-version }}
|
|
213
191
|
steps:
|
|
214
192
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
215
193
|
with:
|
|
@@ -16,7 +16,8 @@ jobs:
|
|
|
16
16
|
push:
|
|
17
17
|
if: github.repository_owner == 'karafka'
|
|
18
18
|
timeout-minutes: 30
|
|
19
|
-
|
|
19
|
+
# Same as CI, we build on the oldest possible for ABI compatibility
|
|
20
|
+
runs-on: ubuntu-22.04 # renovate: ignore
|
|
20
21
|
environment: deployment
|
|
21
22
|
permissions:
|
|
22
23
|
contents: write
|
|
@@ -15,7 +15,8 @@ jobs:
|
|
|
15
15
|
runs-on: ubuntu-latest
|
|
16
16
|
environment: deployment
|
|
17
17
|
container:
|
|
18
|
-
|
|
18
|
+
# Same as CI, we build on the oldest possible for ABI compatibility
|
|
19
|
+
image: alpine:3.18@sha256:de0eb0b3f2a47ba1eb89389859a9bd88b28e82f5826b6969ad604979713c2d4f # renovate: ignore
|
|
19
20
|
steps:
|
|
20
21
|
- name: Install dependencies
|
|
21
22
|
run: |
|
|
@@ -44,6 +45,7 @@ jobs:
|
|
|
44
45
|
name: librdkafka-precompiled-musl
|
|
45
46
|
path: ext/
|
|
46
47
|
retention-days: 1
|
|
48
|
+
|
|
47
49
|
push:
|
|
48
50
|
if: github.repository_owner == 'karafka'
|
|
49
51
|
timeout-minutes: 30
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
3.4.
|
|
1
|
+
3.4.5
|
data/CHANGELOG.md
CHANGED
|
@@ -1,21 +1,31 @@
|
|
|
1
1
|
# Rdkafka Changelog
|
|
2
2
|
|
|
3
|
-
## 0.22.
|
|
3
|
+
## 0.22.2 (2025-07-21)
|
|
4
|
+
- [Enhancement] Drastically increase number of platforms in the integration suite
|
|
5
|
+
- [Fix] Support Ubuntu `22.04` and older Alpine precompiled versions
|
|
6
|
+
- [Fix] FFI::DynamicLibrary.load_library': Could not open library
|
|
7
|
+
|
|
8
|
+
## 0.22.1 (2025-07-17)
|
|
9
|
+
- [Fix] Fix `Rakefile` being available in the precompiled versions causing build failures.
|
|
10
|
+
|
|
11
|
+
## 0.22.0 (2025-07-17)
|
|
4
12
|
- **[Feature]** Add precompiled `x86_64-linux-gnu` setup.
|
|
5
13
|
- **[Feature]** Add precompiled `x86_64-linux-musl` setup.
|
|
6
14
|
- **[Feature]** Add precompiled `macos_arm64` setup.
|
|
7
15
|
- [Fix] Fix a case where using empty key on the `musl` architecture would cause a segfault.
|
|
16
|
+
- [Fix] Fix for null pointer reference bypass on empty string being too wide causing segfault.
|
|
8
17
|
- [Enhancement] Allow for producing to non-existing topics with `key` and `partition_key` present.
|
|
9
18
|
- [Enhancement] Replace TTL-based partition count cache with a global cache that reuses `librdkafka` statistics data when possible.
|
|
10
19
|
- [Enhancement] Support producing and consuming of headers with mulitple values (KIP-82).
|
|
11
20
|
- [Enhancement] Allow native Kafka customization poll time.
|
|
12
21
|
- [Enhancement] Roll out experimental jruby support.
|
|
13
22
|
- [Enhancement] Run all specs on each of the platforms with and without precompilation.
|
|
23
|
+
- [Enhancement] Support transactional id in the ACL API.
|
|
14
24
|
- [Fix] Fix issue where post-closed producer C topics refs would not be cleaned.
|
|
15
25
|
- [Fix] Fiber causes Segmentation Fault.
|
|
16
26
|
- [Change] Move to trusted-publishers and remove signing since no longer needed.
|
|
17
27
|
|
|
18
|
-
**Note**: Precompiled extensions are a new feature in this release. While they significantly improve installation speed and reduce build dependencies, they should be thoroughly tested in your staging environment before deploying to production. If you encounter any issues with precompiled extensions, you can fall back to building from sources.
|
|
28
|
+
**Note**: Precompiled extensions are a new feature in this release. While they significantly improve installation speed and reduce build dependencies, they should be thoroughly tested in your staging environment before deploying to production. If you encounter any issues with precompiled extensions, you can fall back to building from sources. For more information, see the [Native Extensions documentation](https://karafka.io/docs/Development-Native-Extensions/).
|
|
19
29
|
|
|
20
30
|
## 0.21.0 (2025-02-13)
|
|
21
31
|
- [Enhancement] Bump librdkafka to `2.8.0`
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Rdkafka
|
|
2
2
|
|
|
3
|
-
[](https://github.com/karafka/rdkafka-ruby/actions/workflows/ci_linux_x86_64_gnu.yml)
|
|
4
4
|
[](https://badge.fury.io/rb/rdkafka)
|
|
5
5
|
[](https://slack.karafka.io)
|
|
6
6
|
|
|
@@ -163,7 +163,7 @@ bundle exec rake produce_messages
|
|
|
163
163
|
|
|
164
164
|
| rdkafka-ruby | librdkafka | patches |
|
|
165
165
|
|-|-|-|
|
|
166
|
-
| 0.22.x (2025-
|
|
166
|
+
| 0.22.x (2025-07-17) | 2.8.0 (2025-01-07) | yes |
|
|
167
167
|
| 0.21.x (2025-02-13) | 2.8.0 (2025-01-07) | yes |
|
|
168
168
|
| 0.20.0 (2025-01-07) | 2.6.1 (2024-11-18) | yes |
|
|
169
169
|
| 0.19.0 (2024-10-01) | 2.5.3 (2024-09-02) | yes |
|
data/ext/librdkafka.dylib
CHANGED
|
Binary file
|
data/lib/rdkafka/bindings.rb
CHANGED
|
@@ -384,18 +384,16 @@ module Rdkafka
|
|
|
384
384
|
hsh[name] = method_name
|
|
385
385
|
end
|
|
386
386
|
|
|
387
|
-
def self.partitioner(str, partition_count,
|
|
387
|
+
def self.partitioner(topic_ptr, str, partition_count, partitioner = "consistent_random")
|
|
388
388
|
# Return RD_KAFKA_PARTITION_UA(unassigned partition) when partition count is nil/zero.
|
|
389
389
|
return -1 unless partition_count&.nonzero?
|
|
390
|
-
# musl architecture crashes with empty string
|
|
391
|
-
return 0 if str.empty?
|
|
392
390
|
|
|
393
|
-
str_ptr = FFI::MemoryPointer.from_string(str)
|
|
394
|
-
method_name = PARTITIONERS.fetch(
|
|
395
|
-
raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{
|
|
391
|
+
str_ptr = str.empty? ? FFI::MemoryPointer::NULL : FFI::MemoryPointer.from_string(str)
|
|
392
|
+
method_name = PARTITIONERS.fetch(partitioner) do
|
|
393
|
+
raise Rdkafka::Config::ConfigError.new("Unknown partitioner: #{partitioner}")
|
|
396
394
|
end
|
|
397
395
|
|
|
398
|
-
public_send(method_name,
|
|
396
|
+
public_send(method_name, topic_ptr, str_ptr, str.size, partition_count, nil, nil)
|
|
399
397
|
end
|
|
400
398
|
|
|
401
399
|
# Create Topics
|
|
@@ -513,6 +511,7 @@ module Rdkafka
|
|
|
513
511
|
RD_KAFKA_RESOURCE_TOPIC = 2
|
|
514
512
|
RD_KAFKA_RESOURCE_GROUP = 3
|
|
515
513
|
RD_KAFKA_RESOURCE_BROKER = 4
|
|
514
|
+
RD_KAFKA_RESOURCE_TRANSACTIONAL_ID = 5
|
|
516
515
|
|
|
517
516
|
# rd_kafka_ResourcePatternType_t - https://github.com/confluentinc/librdkafka/blob/292d2a66b9921b783f08147807992e603c7af059/src/rdkafka.h#L7320
|
|
518
517
|
|
data/lib/rdkafka/producer.rb
CHANGED
|
@@ -51,13 +51,13 @@ module Rdkafka
|
|
|
51
51
|
|
|
52
52
|
# @private
|
|
53
53
|
# @param native_kafka [NativeKafka]
|
|
54
|
-
# @param
|
|
54
|
+
# @param partitioner [String, nil] name of the partitioner we want to use or nil to use
|
|
55
55
|
# the "consistent_random" default
|
|
56
|
-
def initialize(native_kafka,
|
|
56
|
+
def initialize(native_kafka, partitioner)
|
|
57
57
|
@topics_refs_map = {}
|
|
58
58
|
@topics_configs = {}
|
|
59
59
|
@native_kafka = native_kafka
|
|
60
|
-
@
|
|
60
|
+
@partitioner = partitioner || "consistent_random"
|
|
61
61
|
|
|
62
62
|
# Makes sure, that native kafka gets closed before it gets GCed by Ruby
|
|
63
63
|
ObjectSpace.define_finalizer(self, native_kafka.finalizer)
|
|
@@ -275,7 +275,8 @@ module Rdkafka
|
|
|
275
275
|
timestamp: nil,
|
|
276
276
|
headers: nil,
|
|
277
277
|
label: nil,
|
|
278
|
-
topic_config: EMPTY_HASH
|
|
278
|
+
topic_config: EMPTY_HASH,
|
|
279
|
+
partitioner: @partitioner
|
|
279
280
|
)
|
|
280
281
|
closed_producer_check(__method__)
|
|
281
282
|
|
|
@@ -307,10 +308,14 @@ module Rdkafka
|
|
|
307
308
|
|
|
308
309
|
# Check if there are no overrides for the partitioner and use the default one only when
|
|
309
310
|
# no per-topic is present.
|
|
310
|
-
|
|
311
|
+
selected_partitioner = @topics_configs.dig(topic, topic_config_hash, :partitioner) || partitioner
|
|
311
312
|
|
|
312
313
|
# If the topic is not present, set to -1
|
|
313
|
-
partition = Rdkafka::Bindings.partitioner(
|
|
314
|
+
partition = Rdkafka::Bindings.partitioner(
|
|
315
|
+
topic_ref,
|
|
316
|
+
partition_key,
|
|
317
|
+
partition_count,
|
|
318
|
+
selected_partitioner) if partition_count.positive?
|
|
314
319
|
end
|
|
315
320
|
|
|
316
321
|
# If partition is nil, use -1 to let librdafka set the partition randomly or
|
data/lib/rdkafka/version.rb
CHANGED
data/rdkafka.gemspec
CHANGED
|
@@ -9,13 +9,11 @@ Gem::Specification.new do |gem|
|
|
|
9
9
|
gem.summary = "The rdkafka gem is a modern Kafka client library for Ruby based on librdkafka. It wraps the production-ready C client using the ffi gem and targets Kafka 1.0+ and Ruby 2.7+."
|
|
10
10
|
gem.license = 'MIT'
|
|
11
11
|
|
|
12
|
-
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
|
13
12
|
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
|
14
13
|
gem.name = 'rdkafka'
|
|
15
14
|
gem.require_paths = ['lib']
|
|
16
15
|
gem.version = Rdkafka::VERSION
|
|
17
16
|
gem.required_ruby_version = '>= 3.1'
|
|
18
|
-
gem.extensions = %w(ext/Rakefile)
|
|
19
17
|
|
|
20
18
|
if ENV['RUBY_PLATFORM']
|
|
21
19
|
gem.platform = ENV['RUBY_PLATFORM']
|
data/spec/rdkafka/admin_spec.rb
CHANGED
|
@@ -513,7 +513,7 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
|
513
513
|
end
|
|
514
514
|
end
|
|
515
515
|
|
|
516
|
-
describe "#ACL tests" do
|
|
516
|
+
describe "#ACL tests for topic resource" do
|
|
517
517
|
let(:non_existing_resource_name) {"non-existing-topic"}
|
|
518
518
|
before do
|
|
519
519
|
#create topic for testing acl
|
|
@@ -615,6 +615,207 @@ expect(ex.broker_message).to match(/Topic name.*is invalid: .* contains one or m
|
|
|
615
615
|
end
|
|
616
616
|
end
|
|
617
617
|
|
|
618
|
+
describe "#ACL tests for transactional_id" do
|
|
619
|
+
let(:transactional_id_resource_name) {"test-transactional-id"}
|
|
620
|
+
let(:non_existing_transactional_id) {"non-existing-transactional-id"}
|
|
621
|
+
let(:transactional_id_resource_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_TRANSACTIONAL_ID }
|
|
622
|
+
let(:transactional_id_resource_pattern_type) { Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_LITERAL }
|
|
623
|
+
let(:transactional_id_principal) { "User:test-user" }
|
|
624
|
+
let(:transactional_id_host) { "*" }
|
|
625
|
+
let(:transactional_id_operation) { Rdkafka::Bindings::RD_KAFKA_ACL_OPERATION_WRITE }
|
|
626
|
+
let(:transactional_id_permission_type) { Rdkafka::Bindings::RD_KAFKA_ACL_PERMISSION_TYPE_ALLOW }
|
|
627
|
+
|
|
628
|
+
after do
|
|
629
|
+
# Clean up any ACLs that might have been created during tests
|
|
630
|
+
begin
|
|
631
|
+
delete_acl_handle = admin.delete_acl(
|
|
632
|
+
resource_type: transactional_id_resource_type,
|
|
633
|
+
resource_name: nil,
|
|
634
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
635
|
+
principal: transactional_id_principal,
|
|
636
|
+
host: transactional_id_host,
|
|
637
|
+
operation: transactional_id_operation,
|
|
638
|
+
permission_type: transactional_id_permission_type
|
|
639
|
+
)
|
|
640
|
+
delete_acl_handle.wait(max_wait_timeout: 15.0)
|
|
641
|
+
rescue
|
|
642
|
+
# Ignore cleanup errors
|
|
643
|
+
end
|
|
644
|
+
end
|
|
645
|
+
|
|
646
|
+
describe "#create_acl" do
|
|
647
|
+
it "creates acl for a transactional_id" do
|
|
648
|
+
create_acl_handle = admin.create_acl(
|
|
649
|
+
resource_type: transactional_id_resource_type,
|
|
650
|
+
resource_name: transactional_id_resource_name,
|
|
651
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
652
|
+
principal: transactional_id_principal,
|
|
653
|
+
host: transactional_id_host,
|
|
654
|
+
operation: transactional_id_operation,
|
|
655
|
+
permission_type: transactional_id_permission_type
|
|
656
|
+
)
|
|
657
|
+
create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
|
|
658
|
+
expect(create_acl_report.rdkafka_response).to eq(0)
|
|
659
|
+
expect(create_acl_report.rdkafka_response_string).to eq("")
|
|
660
|
+
end
|
|
661
|
+
|
|
662
|
+
it "creates acl for a non-existing transactional_id" do
|
|
663
|
+
# ACL creation for transactional_ids that don't exist will still get created successfully
|
|
664
|
+
create_acl_handle = admin.create_acl(
|
|
665
|
+
resource_type: transactional_id_resource_type,
|
|
666
|
+
resource_name: non_existing_transactional_id,
|
|
667
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
668
|
+
principal: transactional_id_principal,
|
|
669
|
+
host: transactional_id_host,
|
|
670
|
+
operation: transactional_id_operation,
|
|
671
|
+
permission_type: transactional_id_permission_type
|
|
672
|
+
)
|
|
673
|
+
create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
|
|
674
|
+
expect(create_acl_report.rdkafka_response).to eq(0)
|
|
675
|
+
expect(create_acl_report.rdkafka_response_string).to eq("")
|
|
676
|
+
|
|
677
|
+
# Clean up the ACL that was created for the non-existing transactional_id
|
|
678
|
+
delete_acl_handle = admin.delete_acl(
|
|
679
|
+
resource_type: transactional_id_resource_type,
|
|
680
|
+
resource_name: non_existing_transactional_id,
|
|
681
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
682
|
+
principal: transactional_id_principal,
|
|
683
|
+
host: transactional_id_host,
|
|
684
|
+
operation: transactional_id_operation,
|
|
685
|
+
permission_type: transactional_id_permission_type
|
|
686
|
+
)
|
|
687
|
+
delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
|
|
688
|
+
expect(delete_acl_handle[:response]).to eq(0)
|
|
689
|
+
expect(delete_acl_report.deleted_acls.size).to eq(1)
|
|
690
|
+
end
|
|
691
|
+
end
|
|
692
|
+
|
|
693
|
+
describe "#describe_acl" do
|
|
694
|
+
it "describes acl of a transactional_id that does not exist" do
|
|
695
|
+
describe_acl_handle = admin.describe_acl(
|
|
696
|
+
resource_type: transactional_id_resource_type,
|
|
697
|
+
resource_name: non_existing_transactional_id,
|
|
698
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
699
|
+
principal: transactional_id_principal,
|
|
700
|
+
host: transactional_id_host,
|
|
701
|
+
operation: transactional_id_operation,
|
|
702
|
+
permission_type: transactional_id_permission_type
|
|
703
|
+
)
|
|
704
|
+
describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
|
|
705
|
+
expect(describe_acl_handle[:response]).to eq(0)
|
|
706
|
+
expect(describe_acl_report.acls.size).to eq(0)
|
|
707
|
+
end
|
|
708
|
+
|
|
709
|
+
it "creates acls and describes the newly created transactional_id acls" do
|
|
710
|
+
# Create first ACL
|
|
711
|
+
create_acl_handle = admin.create_acl(
|
|
712
|
+
resource_type: transactional_id_resource_type,
|
|
713
|
+
resource_name: "test_transactional_id_1",
|
|
714
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
715
|
+
principal: transactional_id_principal,
|
|
716
|
+
host: transactional_id_host,
|
|
717
|
+
operation: transactional_id_operation,
|
|
718
|
+
permission_type: transactional_id_permission_type
|
|
719
|
+
)
|
|
720
|
+
create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
|
|
721
|
+
expect(create_acl_report.rdkafka_response).to eq(0)
|
|
722
|
+
expect(create_acl_report.rdkafka_response_string).to eq("")
|
|
723
|
+
|
|
724
|
+
# Create second ACL
|
|
725
|
+
create_acl_handle = admin.create_acl(
|
|
726
|
+
resource_type: transactional_id_resource_type,
|
|
727
|
+
resource_name: "test_transactional_id_2",
|
|
728
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
729
|
+
principal: transactional_id_principal,
|
|
730
|
+
host: transactional_id_host,
|
|
731
|
+
operation: transactional_id_operation,
|
|
732
|
+
permission_type: transactional_id_permission_type
|
|
733
|
+
)
|
|
734
|
+
create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
|
|
735
|
+
expect(create_acl_report.rdkafka_response).to eq(0)
|
|
736
|
+
expect(create_acl_report.rdkafka_response_string).to eq("")
|
|
737
|
+
|
|
738
|
+
# Since we create and immediately check, this is slow on loaded CIs, hence we wait
|
|
739
|
+
sleep(2)
|
|
740
|
+
|
|
741
|
+
# Describe ACLs - filter by transactional_id resource type
|
|
742
|
+
describe_acl_handle = admin.describe_acl(
|
|
743
|
+
resource_type: transactional_id_resource_type,
|
|
744
|
+
resource_name: nil,
|
|
745
|
+
resource_pattern_type: Rdkafka::Bindings::RD_KAFKA_RESOURCE_PATTERN_ANY,
|
|
746
|
+
principal: transactional_id_principal,
|
|
747
|
+
host: transactional_id_host,
|
|
748
|
+
operation: transactional_id_operation,
|
|
749
|
+
permission_type: transactional_id_permission_type
|
|
750
|
+
)
|
|
751
|
+
describe_acl_report = describe_acl_handle.wait(max_wait_timeout: 15.0)
|
|
752
|
+
expect(describe_acl_handle[:response]).to eq(0)
|
|
753
|
+
expect(describe_acl_report.acls.length).to eq(2)
|
|
754
|
+
end
|
|
755
|
+
end
|
|
756
|
+
|
|
757
|
+
describe "#delete_acl" do
|
|
758
|
+
it "deletes acl of a transactional_id that does not exist" do
|
|
759
|
+
delete_acl_handle = admin.delete_acl(
|
|
760
|
+
resource_type: transactional_id_resource_type,
|
|
761
|
+
resource_name: non_existing_transactional_id,
|
|
762
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
763
|
+
principal: transactional_id_principal,
|
|
764
|
+
host: transactional_id_host,
|
|
765
|
+
operation: transactional_id_operation,
|
|
766
|
+
permission_type: transactional_id_permission_type
|
|
767
|
+
)
|
|
768
|
+
delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
|
|
769
|
+
expect(delete_acl_handle[:response]).to eq(0)
|
|
770
|
+
expect(delete_acl_report.deleted_acls.size).to eq(0)
|
|
771
|
+
end
|
|
772
|
+
|
|
773
|
+
it "creates transactional_id acls and deletes the newly created acls" do
|
|
774
|
+
# Create first ACL
|
|
775
|
+
create_acl_handle = admin.create_acl(
|
|
776
|
+
resource_type: transactional_id_resource_type,
|
|
777
|
+
resource_name: "test_transactional_id_1",
|
|
778
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
779
|
+
principal: transactional_id_principal,
|
|
780
|
+
host: transactional_id_host,
|
|
781
|
+
operation: transactional_id_operation,
|
|
782
|
+
permission_type: transactional_id_permission_type
|
|
783
|
+
)
|
|
784
|
+
create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
|
|
785
|
+
expect(create_acl_report.rdkafka_response).to eq(0)
|
|
786
|
+
expect(create_acl_report.rdkafka_response_string).to eq("")
|
|
787
|
+
|
|
788
|
+
# Create second ACL
|
|
789
|
+
create_acl_handle = admin.create_acl(
|
|
790
|
+
resource_type: transactional_id_resource_type,
|
|
791
|
+
resource_name: "test_transactional_id_2",
|
|
792
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
793
|
+
principal: transactional_id_principal,
|
|
794
|
+
host: transactional_id_host,
|
|
795
|
+
operation: transactional_id_operation,
|
|
796
|
+
permission_type: transactional_id_permission_type
|
|
797
|
+
)
|
|
798
|
+
create_acl_report = create_acl_handle.wait(max_wait_timeout: 15.0)
|
|
799
|
+
expect(create_acl_report.rdkafka_response).to eq(0)
|
|
800
|
+
expect(create_acl_report.rdkafka_response_string).to eq("")
|
|
801
|
+
|
|
802
|
+
# Delete ACLs - resource_name nil to delete all ACLs with any resource name and matching all other filters
|
|
803
|
+
delete_acl_handle = admin.delete_acl(
|
|
804
|
+
resource_type: transactional_id_resource_type,
|
|
805
|
+
resource_name: nil,
|
|
806
|
+
resource_pattern_type: transactional_id_resource_pattern_type,
|
|
807
|
+
principal: transactional_id_principal,
|
|
808
|
+
host: transactional_id_host,
|
|
809
|
+
operation: transactional_id_operation,
|
|
810
|
+
permission_type: transactional_id_permission_type
|
|
811
|
+
)
|
|
812
|
+
delete_acl_report = delete_acl_handle.wait(max_wait_timeout: 15.0)
|
|
813
|
+
expect(delete_acl_handle[:response]).to eq(0)
|
|
814
|
+
expect(delete_acl_report.deleted_acls.length).to eq(2)
|
|
815
|
+
end
|
|
816
|
+
end
|
|
817
|
+
end
|
|
818
|
+
|
|
618
819
|
describe('Group tests') do
|
|
619
820
|
describe "#delete_group" do
|
|
620
821
|
describe("with an existing group") do
|
|
@@ -77,30 +77,6 @@ describe Rdkafka::Bindings do
|
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
-
describe "partitioner" do
|
|
81
|
-
let(:partition_key) { ('a'..'z').to_a.shuffle.take(15).join('') }
|
|
82
|
-
let(:partition_count) { rand(50) + 1 }
|
|
83
|
-
|
|
84
|
-
it "should return the same partition for a similar string and the same partition count" do
|
|
85
|
-
result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
|
|
86
|
-
result_2 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
|
|
87
|
-
expect(result_1).to eq(result_2)
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
it "should match the old partitioner" do
|
|
91
|
-
result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count)
|
|
92
|
-
result_2 = (Zlib.crc32(partition_key) % partition_count)
|
|
93
|
-
expect(result_1).to eq(result_2)
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
it "should return the partition calculated by the specified partitioner" do
|
|
97
|
-
result_1 = Rdkafka::Bindings.partitioner(partition_key, partition_count, "murmur2")
|
|
98
|
-
ptr = FFI::MemoryPointer.from_string(partition_key)
|
|
99
|
-
result_2 = Rdkafka::Bindings.rd_kafka_msg_partitioner_murmur2(nil, ptr, partition_key.size, partition_count, nil, nil)
|
|
100
|
-
expect(result_1).to eq(result_2)
|
|
101
|
-
end
|
|
102
|
-
end
|
|
103
|
-
|
|
104
80
|
describe "stats callback" do
|
|
105
81
|
context "without a stats callback" do
|
|
106
82
|
it "should do nothing" do
|
|
@@ -340,7 +340,7 @@ describe Rdkafka::Producer do
|
|
|
340
340
|
)
|
|
341
341
|
end
|
|
342
342
|
|
|
343
|
-
expect(messages[0].partition).to
|
|
343
|
+
expect(messages[0].partition).to be >= 0
|
|
344
344
|
expect(messages[0].key).to eq 'a'
|
|
345
345
|
end
|
|
346
346
|
|
|
@@ -920,7 +920,6 @@ describe Rdkafka::Producer do
|
|
|
920
920
|
end
|
|
921
921
|
end
|
|
922
922
|
|
|
923
|
-
|
|
924
923
|
describe 'with active statistics callback' do
|
|
925
924
|
let(:producer) do
|
|
926
925
|
rdkafka_producer_config('statistics.interval.ms': 1_000).producer
|
|
@@ -1049,4 +1048,298 @@ describe Rdkafka::Producer do
|
|
|
1049
1048
|
end
|
|
1050
1049
|
end
|
|
1051
1050
|
end
|
|
1051
|
+
|
|
1052
|
+
let(:producer) { rdkafka_producer_config.producer }
|
|
1053
|
+
let(:all_partitioners) { %w(random consistent consistent_random murmur2 murmur2_random fnv1a fnv1a_random) }
|
|
1054
|
+
|
|
1055
|
+
describe "partitioner behavior through producer API" do
|
|
1056
|
+
context "testing all partitioners with same key" do
|
|
1057
|
+
it "should not return partition 0 for all partitioners" do
|
|
1058
|
+
test_key = "test-key-123"
|
|
1059
|
+
results = {}
|
|
1060
|
+
|
|
1061
|
+
all_partitioners.each do |partitioner|
|
|
1062
|
+
handle = producer.produce(
|
|
1063
|
+
topic: "partitioner_test_topic",
|
|
1064
|
+
payload: "test payload",
|
|
1065
|
+
partition_key: test_key,
|
|
1066
|
+
partitioner: partitioner
|
|
1067
|
+
)
|
|
1068
|
+
|
|
1069
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1070
|
+
results[partitioner] = report.partition
|
|
1071
|
+
end
|
|
1072
|
+
|
|
1073
|
+
# Should not all be the same partition (especially not all 0)
|
|
1074
|
+
unique_partitions = results.values.uniq
|
|
1075
|
+
expect(unique_partitions.size).to be > 1
|
|
1076
|
+
end
|
|
1077
|
+
end
|
|
1078
|
+
|
|
1079
|
+
context "empty string partition key" do
|
|
1080
|
+
it "should produce message with empty partition key without crashing and go to partition 0 for all partitioners" do
|
|
1081
|
+
all_partitioners.each do |partitioner|
|
|
1082
|
+
handle = producer.produce(
|
|
1083
|
+
topic: "partitioner_test_topic",
|
|
1084
|
+
payload: "test payload",
|
|
1085
|
+
key: "test-key",
|
|
1086
|
+
partition_key: "",
|
|
1087
|
+
partitioner: partitioner
|
|
1088
|
+
)
|
|
1089
|
+
|
|
1090
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1091
|
+
expect(report.partition).to be >= 0
|
|
1092
|
+
end
|
|
1093
|
+
end
|
|
1094
|
+
end
|
|
1095
|
+
|
|
1096
|
+
context "nil partition key" do
|
|
1097
|
+
it "should handle nil partition key gracefully" do
|
|
1098
|
+
handle = producer.produce(
|
|
1099
|
+
topic: "partitioner_test_topic",
|
|
1100
|
+
payload: "test payload",
|
|
1101
|
+
key: "test-key",
|
|
1102
|
+
partition_key: nil
|
|
1103
|
+
)
|
|
1104
|
+
|
|
1105
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1106
|
+
expect(report.partition).to be >= 0
|
|
1107
|
+
expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1108
|
+
end
|
|
1109
|
+
end
|
|
1110
|
+
|
|
1111
|
+
context "various key types and lengths with different partitioners" do
|
|
1112
|
+
it "should handle very short keys with all partitioners" do
|
|
1113
|
+
all_partitioners.each do |partitioner|
|
|
1114
|
+
handle = producer.produce(
|
|
1115
|
+
topic: "partitioner_test_topic",
|
|
1116
|
+
payload: "test payload",
|
|
1117
|
+
partition_key: "a",
|
|
1118
|
+
partitioner: partitioner
|
|
1119
|
+
)
|
|
1120
|
+
|
|
1121
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1122
|
+
expect(report.partition).to be >= 0
|
|
1123
|
+
expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1124
|
+
end
|
|
1125
|
+
end
|
|
1126
|
+
|
|
1127
|
+
it "should handle very long keys with all partitioners" do
|
|
1128
|
+
long_key = "a" * 1000
|
|
1129
|
+
|
|
1130
|
+
all_partitioners.each do |partitioner|
|
|
1131
|
+
handle = producer.produce(
|
|
1132
|
+
topic: "partitioner_test_topic",
|
|
1133
|
+
payload: "test payload",
|
|
1134
|
+
partition_key: long_key,
|
|
1135
|
+
partitioner: partitioner
|
|
1136
|
+
)
|
|
1137
|
+
|
|
1138
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1139
|
+
expect(report.partition).to be >= 0
|
|
1140
|
+
expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1141
|
+
end
|
|
1142
|
+
end
|
|
1143
|
+
|
|
1144
|
+
it "should handle unicode keys with all partitioners" do
|
|
1145
|
+
unicode_key = "测试键值🚀"
|
|
1146
|
+
|
|
1147
|
+
all_partitioners.each do |partitioner|
|
|
1148
|
+
handle = producer.produce(
|
|
1149
|
+
topic: "partitioner_test_topic",
|
|
1150
|
+
payload: "test payload",
|
|
1151
|
+
partition_key: unicode_key,
|
|
1152
|
+
partitioner: partitioner
|
|
1153
|
+
)
|
|
1154
|
+
|
|
1155
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1156
|
+
expect(report.partition).to be >= 0
|
|
1157
|
+
expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1158
|
+
end
|
|
1159
|
+
end
|
|
1160
|
+
end
|
|
1161
|
+
|
|
1162
|
+
context "consistency testing for deterministic partitioners" do
|
|
1163
|
+
%w(consistent murmur2 fnv1a).each do |partitioner|
|
|
1164
|
+
it "should consistently route same partition key to same partition with #{partitioner}" do
|
|
1165
|
+
partition_key = "consistent-test-key"
|
|
1166
|
+
|
|
1167
|
+
# Produce multiple messages with same partition key
|
|
1168
|
+
reports = 5.times.map do
|
|
1169
|
+
handle = producer.produce(
|
|
1170
|
+
topic: "partitioner_test_topic",
|
|
1171
|
+
payload: "test payload #{Time.now.to_f}",
|
|
1172
|
+
partition_key: partition_key,
|
|
1173
|
+
partitioner: partitioner
|
|
1174
|
+
)
|
|
1175
|
+
handle.wait(max_wait_timeout: 5)
|
|
1176
|
+
end
|
|
1177
|
+
|
|
1178
|
+
# All should go to same partition
|
|
1179
|
+
partitions = reports.map(&:partition).uniq
|
|
1180
|
+
expect(partitions.size).to eq(1)
|
|
1181
|
+
end
|
|
1182
|
+
end
|
|
1183
|
+
end
|
|
1184
|
+
|
|
1185
|
+
context "randomness testing for random partitioners" do
|
|
1186
|
+
%w(random consistent_random murmur2_random fnv1a_random).each do |partitioner|
|
|
1187
|
+
it "should potentially distribute across partitions with #{partitioner}" do
|
|
1188
|
+
# Note: random partitioners might still return same value by chance
|
|
1189
|
+
partition_key = "random-test-key"
|
|
1190
|
+
|
|
1191
|
+
reports = 10.times.map do
|
|
1192
|
+
handle = producer.produce(
|
|
1193
|
+
topic: "partitioner_test_topic",
|
|
1194
|
+
payload: "test payload #{Time.now.to_f}",
|
|
1195
|
+
partition_key: partition_key,
|
|
1196
|
+
partitioner: partitioner
|
|
1197
|
+
)
|
|
1198
|
+
handle.wait(max_wait_timeout: 5)
|
|
1199
|
+
end
|
|
1200
|
+
|
|
1201
|
+
partitions = reports.map(&:partition)
|
|
1202
|
+
|
|
1203
|
+
# Just ensure they're valid partitions
|
|
1204
|
+
partitions.each do |partition|
|
|
1205
|
+
expect(partition).to be >= 0
|
|
1206
|
+
expect(partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1207
|
+
end
|
|
1208
|
+
end
|
|
1209
|
+
end
|
|
1210
|
+
end
|
|
1211
|
+
|
|
1212
|
+
context "comparing different partitioners with same key" do
|
|
1213
|
+
it "should route different partition keys to potentially different partitions" do
|
|
1214
|
+
keys = ["key1", "key2", "key3", "key4", "key5"]
|
|
1215
|
+
|
|
1216
|
+
all_partitioners.each do |partitioner|
|
|
1217
|
+
reports = keys.map do |key|
|
|
1218
|
+
handle = producer.produce(
|
|
1219
|
+
topic: "partitioner_test_topic",
|
|
1220
|
+
payload: "test payload",
|
|
1221
|
+
partition_key: key,
|
|
1222
|
+
partitioner: partitioner
|
|
1223
|
+
)
|
|
1224
|
+
handle.wait(max_wait_timeout: 5)
|
|
1225
|
+
end
|
|
1226
|
+
|
|
1227
|
+
partitions = reports.map(&:partition).uniq
|
|
1228
|
+
|
|
1229
|
+
# Should distribute across multiple partitions for most partitioners
|
|
1230
|
+
# (though some might hash all keys to same partition by chance)
|
|
1231
|
+
expect(partitions.all? { |p| p >= 0 && p < producer.partition_count("partitioner_test_topic") }).to be true
|
|
1232
|
+
end
|
|
1233
|
+
end
|
|
1234
|
+
end
|
|
1235
|
+
|
|
1236
|
+
context "partition key vs regular key behavior" do
|
|
1237
|
+
it "should use partition key for partitioning when both key and partition_key are provided" do
|
|
1238
|
+
# Use keys that would hash to different partitions
|
|
1239
|
+
regular_key = "regular-key-123"
|
|
1240
|
+
partition_key = "partition-key-456"
|
|
1241
|
+
|
|
1242
|
+
# Message with both keys
|
|
1243
|
+
handle1 = producer.produce(
|
|
1244
|
+
topic: "partitioner_test_topic",
|
|
1245
|
+
payload: "test payload 1",
|
|
1246
|
+
key: regular_key,
|
|
1247
|
+
partition_key: partition_key
|
|
1248
|
+
)
|
|
1249
|
+
|
|
1250
|
+
# Message with only partition key (should go to same partition)
|
|
1251
|
+
handle2 = producer.produce(
|
|
1252
|
+
topic: "partitioner_test_topic",
|
|
1253
|
+
payload: "test payload 2",
|
|
1254
|
+
partition_key: partition_key
|
|
1255
|
+
)
|
|
1256
|
+
|
|
1257
|
+
# Message with only regular key (should go to different partition)
|
|
1258
|
+
handle3 = producer.produce(
|
|
1259
|
+
topic: "partitioner_test_topic",
|
|
1260
|
+
payload: "test payload 3",
|
|
1261
|
+
key: regular_key
|
|
1262
|
+
)
|
|
1263
|
+
|
|
1264
|
+
report1 = handle1.wait(max_wait_timeout: 5)
|
|
1265
|
+
report2 = handle2.wait(max_wait_timeout: 5)
|
|
1266
|
+
report3 = handle3.wait(max_wait_timeout: 5)
|
|
1267
|
+
|
|
1268
|
+
# Messages 1 and 2 should go to same partition (both use partition_key)
|
|
1269
|
+
expect(report1.partition).to eq(report2.partition)
|
|
1270
|
+
|
|
1271
|
+
# Message 3 should potentially go to different partition (uses regular key)
|
|
1272
|
+
expect(report3.partition).not_to eq(report1.partition)
|
|
1273
|
+
end
|
|
1274
|
+
end
|
|
1275
|
+
|
|
1276
|
+
context "edge case combinations with different partitioners" do
|
|
1277
|
+
it "should handle nil partition key with all partitioners" do
|
|
1278
|
+
all_partitioners.each do |partitioner|
|
|
1279
|
+
handle = producer.produce(
|
|
1280
|
+
topic: "partitioner_test_topic",
|
|
1281
|
+
payload: "test payload",
|
|
1282
|
+
key: "test-key",
|
|
1283
|
+
partition_key: nil,
|
|
1284
|
+
partitioner: partitioner
|
|
1285
|
+
)
|
|
1286
|
+
|
|
1287
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1288
|
+
expect(report.partition).to be >= 0
|
|
1289
|
+
expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1290
|
+
end
|
|
1291
|
+
end
|
|
1292
|
+
|
|
1293
|
+
it "should handle whitespace-only partition key with all partitioners" do
|
|
1294
|
+
all_partitioners.each do |partitioner|
|
|
1295
|
+
handle = producer.produce(
|
|
1296
|
+
topic: "partitioner_test_topic",
|
|
1297
|
+
payload: "test payload",
|
|
1298
|
+
partition_key: " ",
|
|
1299
|
+
partitioner: partitioner
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1303
|
+
expect(report.partition).to be >= 0
|
|
1304
|
+
expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1305
|
+
end
|
|
1306
|
+
end
|
|
1307
|
+
|
|
1308
|
+
it "should handle newline characters in partition key with all partitioners" do
|
|
1309
|
+
all_partitioners.each do |partitioner|
|
|
1310
|
+
handle = producer.produce(
|
|
1311
|
+
topic: "partitioner_test_topic",
|
|
1312
|
+
payload: "test payload",
|
|
1313
|
+
partition_key: "key\nwith\nnewlines",
|
|
1314
|
+
partitioner: partitioner
|
|
1315
|
+
)
|
|
1316
|
+
|
|
1317
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1318
|
+
expect(report.partition).to be >= 0
|
|
1319
|
+
expect(report.partition).to be < producer.partition_count("partitioner_test_topic")
|
|
1320
|
+
end
|
|
1321
|
+
end
|
|
1322
|
+
end
|
|
1323
|
+
|
|
1324
|
+
context "debugging partitioner issues" do
|
|
1325
|
+
it "should show if all partitioners return 0 (indicating a problem)" do
|
|
1326
|
+
test_key = "debug-test-key"
|
|
1327
|
+
zero_count = 0
|
|
1328
|
+
|
|
1329
|
+
all_partitioners.each do |partitioner|
|
|
1330
|
+
handle = producer.produce(
|
|
1331
|
+
topic: "partitioner_test_topic",
|
|
1332
|
+
payload: "debug payload",
|
|
1333
|
+
partition_key: test_key,
|
|
1334
|
+
partitioner: partitioner
|
|
1335
|
+
)
|
|
1336
|
+
|
|
1337
|
+
report = handle.wait(max_wait_timeout: 5)
|
|
1338
|
+
zero_count += 1 if report.partition == 0
|
|
1339
|
+
end
|
|
1340
|
+
|
|
1341
|
+
expect(zero_count).to be < all_partitioners.size
|
|
1342
|
+
end
|
|
1343
|
+
end
|
|
1344
|
+
end
|
|
1052
1345
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rdkafka
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.22.
|
|
4
|
+
version: 0.22.2
|
|
5
5
|
platform: arm64-darwin
|
|
6
6
|
authors:
|
|
7
7
|
- Thijs Cadier
|
|
@@ -140,8 +140,7 @@ description: Modern Kafka client library for Ruby based on librdkafka
|
|
|
140
140
|
email:
|
|
141
141
|
- contact@karafka.io
|
|
142
142
|
executables: []
|
|
143
|
-
extensions:
|
|
144
|
-
- ext/Rakefile
|
|
143
|
+
extensions: []
|
|
145
144
|
extra_rdoc_files: []
|
|
146
145
|
files:
|
|
147
146
|
- ".github/CODEOWNERS"
|