waterdrop 2.8.16 → 2.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/.rubocop.yml +2 -41
- data/.ruby-version +1 -1
- data/.yard-lint.yml +1 -1
- data/CHANGELOG.md +10 -0
- data/Gemfile +2 -1
- data/Gemfile.lint +2 -2
- data/Gemfile.lint.lock +11 -26
- data/Gemfile.lock +43 -66
- data/Rakefile +12 -0
- data/bin/integrations +5 -3
- data/config/locales/errors.yml +1 -0
- data/docker-compose.oauth.yml +3 -3
- data/docker-compose.yml +1 -1
- data/lib/waterdrop/clients/rdkafka.rb +6 -1
- data/lib/waterdrop/config.rb +23 -6
- data/lib/waterdrop/contracts/config.rb +3 -0
- data/lib/waterdrop/instrumentation/callbacks/statistics.rb +4 -2
- data/lib/waterdrop/producer/variant.rb +2 -2
- data/lib/waterdrop/version.rb +1 -1
- data/package-lock.json +3 -3
- data/renovate.json +2 -2
- data/waterdrop.gemspec +2 -2
- metadata +4 -14
- data/.github/CODEOWNERS +0 -3
- data/.github/FUNDING.yml +0 -1
- data/.github/ISSUE_TEMPLATE/bug_report.md +0 -43
- data/.github/ISSUE_TEMPLATE/feature_request.md +0 -20
- data/.github/workflows/ci.yml +0 -322
- data/.github/workflows/push.yml +0 -35
- data/.github/workflows/trigger-wiki-refresh.yml +0 -30
- data/.github/workflows/verify-action-pins.yml +0 -16
- data/.rspec +0 -2
- data/log/.gitkeep +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 38a7f384d48b06062104341df54cf2532c6e338549dd370507d8b2e820721c73
|
|
4
|
+
data.tar.gz: 2b66e9f62a94a4fb1baafc8492f9bcb9d51cee3bebc9de937da2860576f82bfb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 941854a59acf37d5f19d49186cd0950ca699a441653995a1f09cec740fd44c98880364f954aada62377a2313a61e1e164d20e38601384a9f77c3671156150d5b
|
|
7
|
+
data.tar.gz: 70fdbebdff760f2fba63fc001cfe4c151a25c64b9e9ce61f5fbc4f17b9c13b392f06a04cff41d02ae32edd015d2ca801b9fd744f7804417aff2a16bde6d4a75e
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -1,17 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
plugins:
|
|
4
|
-
- rubocop-
|
|
5
|
-
- rubocop-factory_bot
|
|
4
|
+
- rubocop-minitest
|
|
6
5
|
- rubocop-performance
|
|
7
|
-
- rubocop-rspec
|
|
8
|
-
- rubocop-rspec_rails
|
|
9
6
|
- rubocop-thread_safety
|
|
10
7
|
|
|
11
8
|
inherit_gem:
|
|
12
9
|
standard: config/base.yml
|
|
13
10
|
standard-performance: config/base.yml
|
|
14
|
-
standard-
|
|
11
|
+
standard-minitest: config/base.yml
|
|
15
12
|
|
|
16
13
|
AllCops:
|
|
17
14
|
NewCops: enable
|
|
@@ -23,16 +20,6 @@ AllCops:
|
|
|
23
20
|
- "**/Rakefile"
|
|
24
21
|
- Gemfile.lint
|
|
25
22
|
|
|
26
|
-
# Disabled departments - not a Rails project, no Capybara/FactoryBot
|
|
27
|
-
Capybara:
|
|
28
|
-
Enabled: false
|
|
29
|
-
|
|
30
|
-
FactoryBot:
|
|
31
|
-
Enabled: false
|
|
32
|
-
|
|
33
|
-
RSpecRails:
|
|
34
|
-
Enabled: false
|
|
35
|
-
|
|
36
23
|
# Layout
|
|
37
24
|
Layout/LineLength:
|
|
38
25
|
Max: 100
|
|
@@ -45,32 +32,6 @@ Lint/RescueException:
|
|
|
45
32
|
Exclude:
|
|
46
33
|
- lib/waterdrop/producer/transactions.rb
|
|
47
34
|
|
|
48
|
-
# RSpec
|
|
49
|
-
RSpec:
|
|
50
|
-
Exclude:
|
|
51
|
-
- spec/integrations/**/*
|
|
52
|
-
|
|
53
|
-
RSpec/ExampleLength:
|
|
54
|
-
Enabled: false
|
|
55
|
-
|
|
56
|
-
RSpec/IndexedLet:
|
|
57
|
-
Enabled: false
|
|
58
|
-
|
|
59
|
-
RSpec/MultipleExpectations:
|
|
60
|
-
Enabled: false
|
|
61
|
-
|
|
62
|
-
RSpec/MultipleMemoizedHelpers:
|
|
63
|
-
Max: 20
|
|
64
|
-
|
|
65
|
-
RSpec/NestedGroups:
|
|
66
|
-
Max: 4
|
|
67
|
-
|
|
68
|
-
RSpec/NoExpectationExample:
|
|
69
|
-
Enabled: false
|
|
70
|
-
|
|
71
|
-
RSpec/SpecFilePathFormat:
|
|
72
|
-
Enabled: false
|
|
73
|
-
|
|
74
35
|
# Style
|
|
75
36
|
Style/SlicingWithRange:
|
|
76
37
|
Exclude:
|
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.0.
|
|
1
|
+
4.0.2
|
data/.yard-lint.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,15 @@
|
|
|
1
1
|
# WaterDrop changelog
|
|
2
2
|
|
|
3
|
+
## 2.9.0 (2026-04-08)
|
|
4
|
+
- [Fix] Use `delete` in the variant ensure block to avoid leaving stale nil entries in `Fiber.current.waterdrop_clients` and prevent memory leaks in long-running processes (#836).
|
|
5
|
+
- [Fix] Exclude test files, `.github/`, and `log/` directories from gem releases to reduce package size.
|
|
6
|
+
- **[Breaking]** Switch default polling mode from `:thread` to `:fd`. If you experience any issues, you can revert to the previous behavior by setting `config.polling.mode = :thread`. The `:thread` mode will be deprecated in 2.10 and removed in 2.11.
|
|
7
|
+
- **[Breaking]** Statistics decorator now only decorates keys used by the built-in Datadog metrics listener (`tx`, `txretries`, `txerrs`, `rxerrs`) and skips unused subtrees (`topics`, broker window stats). This reduces decoration cost by ~425x on large clusters (10 brokers, 20 topics, 2000 partitions). Users who rely on other `_d` or `_fd` keys in custom instrumentation should provide a custom decorator via `config.statistics_decorator`.
|
|
8
|
+
- [Feature] Add `config.statistics_decorator` setting to allow full control over the `StatisticsDecorator` instance used for statistics decoration. Users can provide a custom decorator with different `only_keys` and `excluded_keys` to match their instrumentation needs.
|
|
9
|
+
- [Change] Upscale default timeout values 3x closer to librdkafka defaults to prevent intermediate timeouts during node recovery (`message.timeout.ms`: 50s → 150s, `transaction.timeout.ms`: 55s → 165s, `max_wait_timeout`: 60s → 180s).
|
|
10
|
+
|
|
11
|
+
**Upgrade Notes**: https://karafka.io/docs/Upgrades-WaterDrop-2.9/
|
|
12
|
+
|
|
3
13
|
## 2.8.16 (2026-02-25)
|
|
4
14
|
- [Feature] Add FD-based polling mode (`config.polling.mode = :fd`) as an alternative to the default thread-based polling. FD mode uses a single Ruby thread with IO.select for efficient multiplexing, providing 39-54% higher throughput, lower memory usage, and fewer threads compared to the default thread mode.
|
|
5
15
|
- [Feature] Add `config.polling.fd.max_time` setting (default: 100ms) to control maximum polling time per producer per cycle. This enables per-producer priority differentiation.
|
data/Gemfile
CHANGED
data/Gemfile.lint
CHANGED
data/Gemfile.lint.lock
CHANGED
|
@@ -27,23 +27,14 @@ GEM
|
|
|
27
27
|
rubocop-ast (1.49.0)
|
|
28
28
|
parser (>= 3.3.7.2)
|
|
29
29
|
prism (~> 1.7)
|
|
30
|
-
rubocop-
|
|
30
|
+
rubocop-minitest (0.39.1)
|
|
31
31
|
lint_roller (~> 1.1)
|
|
32
|
-
rubocop (
|
|
33
|
-
|
|
34
|
-
lint_roller (~> 1.1)
|
|
35
|
-
rubocop (~> 1.72, >= 1.72.1)
|
|
32
|
+
rubocop (>= 1.75.0, < 2.0)
|
|
33
|
+
rubocop-ast (>= 1.38.0, < 2.0)
|
|
36
34
|
rubocop-performance (1.26.1)
|
|
37
35
|
lint_roller (~> 1.1)
|
|
38
36
|
rubocop (>= 1.75.0, < 2.0)
|
|
39
37
|
rubocop-ast (>= 1.47.1, < 2.0)
|
|
40
|
-
rubocop-rspec (3.9.0)
|
|
41
|
-
lint_roller (~> 1.1)
|
|
42
|
-
rubocop (~> 1.81)
|
|
43
|
-
rubocop-rspec_rails (2.32.0)
|
|
44
|
-
lint_roller (~> 1.1)
|
|
45
|
-
rubocop (~> 1.72, >= 1.72.1)
|
|
46
|
-
rubocop-rspec (~> 3.5)
|
|
47
38
|
rubocop-thread_safety (0.7.3)
|
|
48
39
|
lint_roller (~> 1.1)
|
|
49
40
|
rubocop (~> 1.72, >= 1.72.1)
|
|
@@ -58,15 +49,12 @@ GEM
|
|
|
58
49
|
standard-custom (1.0.2)
|
|
59
50
|
lint_roller (~> 1.0)
|
|
60
51
|
rubocop (~> 1.50)
|
|
52
|
+
standard-minitest (1.0.0)
|
|
53
|
+
lint_roller (~> 1.0)
|
|
54
|
+
rubocop-minitest
|
|
61
55
|
standard-performance (1.9.0)
|
|
62
56
|
lint_roller (~> 1.1)
|
|
63
57
|
rubocop-performance (~> 1.26.0)
|
|
64
|
-
standard-rspec (0.3.1)
|
|
65
|
-
lint_roller (>= 1.0)
|
|
66
|
-
rubocop-capybara (~> 2.22)
|
|
67
|
-
rubocop-factory_bot (~> 2.27)
|
|
68
|
-
rubocop-rspec (~> 3.5)
|
|
69
|
-
rubocop-rspec_rails (~> 2.31)
|
|
70
58
|
unicode-display_width (3.2.0)
|
|
71
59
|
unicode-emoji (~> 4.1)
|
|
72
60
|
unicode-emoji (4.2.0)
|
|
@@ -81,12 +69,12 @@ PLATFORMS
|
|
|
81
69
|
x86_64-linux
|
|
82
70
|
|
|
83
71
|
DEPENDENCIES
|
|
72
|
+
rubocop-minitest
|
|
84
73
|
rubocop-performance
|
|
85
|
-
rubocop-rspec
|
|
86
74
|
rubocop-thread_safety
|
|
87
75
|
standard
|
|
76
|
+
standard-minitest
|
|
88
77
|
standard-performance
|
|
89
|
-
standard-rspec
|
|
90
78
|
yard-lint
|
|
91
79
|
|
|
92
80
|
CHECKSUMS
|
|
@@ -102,17 +90,14 @@ CHECKSUMS
|
|
|
102
90
|
regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4
|
|
103
91
|
rubocop (1.82.1) sha256=09f1a6a654a960eda767aebea33e47603080f8e9c9a3f019bf9b94c9cab5e273
|
|
104
92
|
rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd
|
|
105
|
-
rubocop-
|
|
106
|
-
rubocop-factory_bot (2.28.0) sha256=4b17fc02124444173317e131759d195b0d762844a71a29fe8139c1105d92f0cb
|
|
93
|
+
rubocop-minitest (0.39.1) sha256=998398d6da4026d297f0f9bf709a1eac5f2b6947c24431f94af08138510cf7ed
|
|
107
94
|
rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834
|
|
108
|
-
rubocop-rspec (3.9.0) sha256=8fa70a3619408237d789aeecfb9beef40576acc855173e60939d63332fdb55e2
|
|
109
|
-
rubocop-rspec_rails (2.32.0) sha256=4a0d641c72f6ebb957534f539d9d0a62c47abd8ce0d0aeee1ef4701e892a9100
|
|
110
95
|
rubocop-thread_safety (0.7.3) sha256=067cdd52fbf5deffc18995437e45b5194236eaff4f71de3375a1f6052e48f431
|
|
111
96
|
ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
|
|
112
97
|
standard (1.53.0) sha256=f3c9493385db7079d0abce6f7582f553122156997b81258cd361d3480eeacf9c
|
|
113
98
|
standard-custom (1.0.2) sha256=424adc84179a074f1a2a309bb9cf7cd6bfdb2b6541f20c6bf9436c0ba22a652b
|
|
99
|
+
standard-minitest (1.0.0) sha256=450caa86a64a6e6f6f186cc88601dbb49b6cfaa3b0dce77a73b50ba8cdc15b2a
|
|
114
100
|
standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2
|
|
115
|
-
standard-rspec (0.3.1) sha256=67bc957281cacf24f0d88235ca1bf28a8995265b1a60eb519cd0451858b56a22
|
|
116
101
|
unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
|
|
117
102
|
unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f
|
|
118
103
|
yard (0.9.38) sha256=721fb82afb10532aa49860655f6cc2eaa7130889df291b052e1e6b268283010f
|
|
@@ -120,4 +105,4 @@ CHECKSUMS
|
|
|
120
105
|
zeitwerk (2.7.4) sha256=2bef90f356bdafe9a6c2bd32bcd804f83a4f9b8bc27f3600fff051eb3edcec8b
|
|
121
106
|
|
|
122
107
|
BUNDLED WITH
|
|
123
|
-
|
|
108
|
+
2.6.9
|
data/Gemfile.lock
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
waterdrop (2.
|
|
5
|
-
karafka-core (>= 2.
|
|
4
|
+
waterdrop (2.9.0)
|
|
5
|
+
karafka-core (>= 2.5.12, < 3.0.0)
|
|
6
6
|
karafka-rdkafka (>= 0.24.0)
|
|
7
7
|
zeitwerk (~> 2.3)
|
|
8
8
|
|
|
@@ -12,49 +12,15 @@ GEM
|
|
|
12
12
|
byebug (13.0.0)
|
|
13
13
|
reline (>= 0.6.0)
|
|
14
14
|
connection_pool (3.0.2)
|
|
15
|
-
diff-lcs (1.6.2)
|
|
16
15
|
docile (1.4.1)
|
|
17
|
-
|
|
18
|
-
ffi (1.17.
|
|
19
|
-
ffi (1.17.3-aarch64-linux-musl)
|
|
20
|
-
ffi (1.17.3-arm-linux-gnu)
|
|
21
|
-
ffi (1.17.3-arm-linux-musl)
|
|
22
|
-
ffi (1.17.3-arm64-darwin)
|
|
23
|
-
ffi (1.17.3-x86-linux-gnu)
|
|
24
|
-
ffi (1.17.3-x86-linux-musl)
|
|
25
|
-
ffi (1.17.3-x86_64-darwin)
|
|
26
|
-
ffi (1.17.3-x86_64-linux-gnu)
|
|
27
|
-
ffi (1.17.3-x86_64-linux-musl)
|
|
16
|
+
drb (2.2.3)
|
|
17
|
+
ffi (1.17.4)
|
|
28
18
|
io-console (0.8.2)
|
|
29
|
-
json (2.
|
|
30
|
-
karafka-core (2.5.
|
|
19
|
+
json (2.19.3)
|
|
20
|
+
karafka-core (2.5.12)
|
|
31
21
|
karafka-rdkafka (>= 0.20.0)
|
|
32
22
|
logger (>= 1.6.0)
|
|
33
|
-
karafka-rdkafka (0.
|
|
34
|
-
ffi (~> 1.17.1)
|
|
35
|
-
json (> 2.0)
|
|
36
|
-
logger
|
|
37
|
-
mini_portile2 (~> 2.6)
|
|
38
|
-
rake (> 12)
|
|
39
|
-
karafka-rdkafka (0.24.0-aarch64-linux-gnu)
|
|
40
|
-
ffi (~> 1.17.1)
|
|
41
|
-
json (> 2.0)
|
|
42
|
-
logger
|
|
43
|
-
mini_portile2 (~> 2.6)
|
|
44
|
-
rake (> 12)
|
|
45
|
-
karafka-rdkafka (0.24.0-aarch64-linux-musl)
|
|
46
|
-
ffi (~> 1.17.1)
|
|
47
|
-
json (> 2.0)
|
|
48
|
-
logger
|
|
49
|
-
mini_portile2 (~> 2.6)
|
|
50
|
-
rake (> 12)
|
|
51
|
-
karafka-rdkafka (0.24.0-arm64-darwin)
|
|
52
|
-
ffi (~> 1.17.1)
|
|
53
|
-
json (> 2.0)
|
|
54
|
-
logger
|
|
55
|
-
mini_portile2 (~> 2.6)
|
|
56
|
-
rake (> 12)
|
|
57
|
-
karafka-rdkafka (0.24.0-x86_64-linux-gnu)
|
|
23
|
+
karafka-rdkafka (0.25.0)
|
|
58
24
|
ffi (~> 1.17.1)
|
|
59
25
|
json (> 2.0)
|
|
60
26
|
logger
|
|
@@ -62,23 +28,17 @@ GEM
|
|
|
62
28
|
rake (> 12)
|
|
63
29
|
logger (1.7.0)
|
|
64
30
|
mini_portile2 (2.8.9)
|
|
31
|
+
minitest (6.0.2)
|
|
32
|
+
drb (~> 2.0)
|
|
33
|
+
prism (~> 1.5)
|
|
34
|
+
mocha (3.1.0)
|
|
35
|
+
ruby2_keywords (>= 0.0.5)
|
|
65
36
|
ostruct (0.6.3)
|
|
37
|
+
prism (1.9.0)
|
|
66
38
|
rake (13.3.1)
|
|
67
39
|
reline (0.6.3)
|
|
68
40
|
io-console (~> 0.5)
|
|
69
|
-
|
|
70
|
-
rspec-core (~> 3.13.0)
|
|
71
|
-
rspec-expectations (~> 3.13.0)
|
|
72
|
-
rspec-mocks (~> 3.13.0)
|
|
73
|
-
rspec-core (3.13.6)
|
|
74
|
-
rspec-support (~> 3.13.0)
|
|
75
|
-
rspec-expectations (3.13.5)
|
|
76
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
77
|
-
rspec-support (~> 3.13.0)
|
|
78
|
-
rspec-mocks (3.13.6)
|
|
79
|
-
diff-lcs (>= 1.2.0, < 2.0)
|
|
80
|
-
rspec-support (~> 3.13.0)
|
|
81
|
-
rspec-support (3.13.6)
|
|
41
|
+
ruby2_keywords (0.0.5)
|
|
82
42
|
simplecov (0.22.0)
|
|
83
43
|
docile (~> 1.1)
|
|
84
44
|
simplecov-html (~> 0.11)
|
|
@@ -89,27 +49,44 @@ GEM
|
|
|
89
49
|
zeitwerk (2.7.5)
|
|
90
50
|
|
|
91
51
|
PLATFORMS
|
|
92
|
-
aarch64-linux-gnu
|
|
93
|
-
aarch64-linux-musl
|
|
94
|
-
arm-linux-gnu
|
|
95
|
-
arm-linux-musl
|
|
96
|
-
arm64-darwin
|
|
97
52
|
ruby
|
|
98
|
-
x86-linux-gnu
|
|
99
|
-
x86-linux-musl
|
|
100
|
-
x86_64-darwin
|
|
101
|
-
x86_64-linux-gnu
|
|
102
|
-
x86_64-linux-musl
|
|
103
53
|
|
|
104
54
|
DEPENDENCIES
|
|
105
55
|
byebug
|
|
106
56
|
connection_pool
|
|
57
|
+
minitest
|
|
58
|
+
mocha
|
|
107
59
|
ostruct
|
|
108
|
-
rspec
|
|
109
60
|
simplecov
|
|
110
61
|
warning
|
|
111
62
|
waterdrop!
|
|
112
63
|
zeitwerk (~> 2.7.0)
|
|
113
64
|
|
|
65
|
+
CHECKSUMS
|
|
66
|
+
byebug (13.0.0) sha256=d2263efe751941ca520fa29744b71972d39cbc41839496706f5d9b22e92ae05d
|
|
67
|
+
connection_pool (3.0.2) sha256=33fff5ba71a12d2aa26cb72b1db8bba2a1a01823559fb01d29eb74c286e62e0a
|
|
68
|
+
docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
|
|
69
|
+
drb (2.2.3) sha256=0b00d6fdb50995fe4a45dea13663493c841112e4068656854646f418fda13373
|
|
70
|
+
ffi (1.17.4) sha256=bcd1642e06f0d16fc9e09ac6d49c3a7298b9789bcb58127302f934e437d60acf
|
|
71
|
+
io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
|
|
72
|
+
json (2.19.3) sha256=289b0bb53052a1fa8c34ab33cc750b659ba14a5c45f3fcf4b18762dc67c78646
|
|
73
|
+
karafka-core (2.5.12) sha256=57cbb45a187fbe3df9b9a57af59dda7211f9969524b2afbb83792a64705860e1
|
|
74
|
+
karafka-rdkafka (0.25.0) sha256=67b316b942cf9ff7e9d7bbf9029e6f2d91eba97b4c9dc93b9f49fd207dfb80f8
|
|
75
|
+
logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
|
|
76
|
+
mini_portile2 (2.8.9) sha256=0cd7c7f824e010c072e33f68bc02d85a00aeb6fce05bb4819c03dfd3c140c289
|
|
77
|
+
minitest (6.0.2) sha256=db6e57956f6ecc6134683b4c87467d6dd792323c7f0eea7b93f66bd284adbc3d
|
|
78
|
+
mocha (3.1.0) sha256=75f42d69ebfb1f10b32489dff8f8431d37a418120ecdfc07afe3bc183d4e1d56
|
|
79
|
+
ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912
|
|
80
|
+
prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
|
|
81
|
+
rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
|
|
82
|
+
reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
|
|
83
|
+
ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef
|
|
84
|
+
simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
|
|
85
|
+
simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
|
|
86
|
+
simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
|
|
87
|
+
warning (1.5.0) sha256=0f12c49fea0c06757778eefdcc7771e4fd99308901e3d55c504d87afdd718c53
|
|
88
|
+
waterdrop (2.9.0)
|
|
89
|
+
zeitwerk (2.7.5) sha256=d8da92128c09ea6ec62c949011b00ed4a20242b255293dd66bf41545398f73dd
|
|
90
|
+
|
|
114
91
|
BUNDLED WITH
|
|
115
|
-
|
|
92
|
+
4.0.6
|
data/Rakefile
CHANGED
|
@@ -2,3 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
require "bundler/setup"
|
|
4
4
|
require "bundler/gem_tasks"
|
|
5
|
+
require "minitest/test_task"
|
|
6
|
+
|
|
7
|
+
Minitest::TestTask.create(:test) do |t|
|
|
8
|
+
t.libs << "test"
|
|
9
|
+
t.libs << "lib"
|
|
10
|
+
t.test_globs = ["test/**/*_test.rb"]
|
|
11
|
+
# Load test_helper before minitest/autorun so SimpleCov's at_exit hook is
|
|
12
|
+
# registered first and thus runs last (LIFO), collecting coverage AFTER tests
|
|
13
|
+
t.test_prelude = 'require "test_helper"; require "minitest/autorun"'
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
task default: :test
|
data/bin/integrations
CHANGED
|
@@ -49,7 +49,7 @@ class Scenario
|
|
|
49
49
|
|
|
50
50
|
# @return [String] integration spec name
|
|
51
51
|
def name
|
|
52
|
-
@path.gsub("#{ROOT_PATH}/
|
|
52
|
+
@path.gsub("#{ROOT_PATH}/test/integrations/", "")
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
# @return [Boolean] did this scenario finished or is it still running
|
|
@@ -142,6 +142,7 @@ class Scenario
|
|
|
142
142
|
# We copy the spec into a temp dir, not to pollute the spec location with logs, etc
|
|
143
143
|
temp_dir = Dir.mktmpdir
|
|
144
144
|
file_name = File.basename(@path)
|
|
145
|
+
spec_file_path = @path.sub("#{ROOT_PATH}/", "")
|
|
145
146
|
|
|
146
147
|
FileUtils.cp_r("#{scenario_dir}/.", temp_dir)
|
|
147
148
|
|
|
@@ -151,7 +152,8 @@ class Scenario
|
|
|
151
152
|
bundle install &&
|
|
152
153
|
BUNDLE_AUTO_INSTALL=true \
|
|
153
154
|
WATERDROP_GEM_DIR=#{ROOT_PATH} \
|
|
154
|
-
|
|
155
|
+
SPEC_FILE_PATH=#{spec_file_path} \
|
|
156
|
+
bundle exec ruby -r "#{ROOT_PATH}/test/support/spec_hash" #{file_name}
|
|
155
157
|
CMD
|
|
156
158
|
end
|
|
157
159
|
|
|
@@ -162,7 +164,7 @@ class Scenario
|
|
|
162
164
|
end
|
|
163
165
|
|
|
164
166
|
# Load all the specs
|
|
165
|
-
specs = Dir[ROOT_PATH.join("
|
|
167
|
+
specs = Dir[ROOT_PATH.join("test/integrations/**/*_spec.rb")]
|
|
166
168
|
|
|
167
169
|
FILTER_TYPE = (ARGV[0] == "--exclude") ? "exclude" : "include"
|
|
168
170
|
|
data/config/locales/errors.yml
CHANGED
|
@@ -26,6 +26,7 @@ en:
|
|
|
26
26
|
non_reloadable_errors_format: must be an array of symbols
|
|
27
27
|
oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
|
|
28
28
|
idle_disconnect_timeout_format: 'must be an integer that is equal to 0 or bigger than 30 000 (30 seconds)'
|
|
29
|
+
statistics_decorator_format: 'must respond to #call'
|
|
29
30
|
polling.mode_format: must be :thread or :fd
|
|
30
31
|
polling.poller_format: must be nil or a WaterDrop::Polling::Poller instance
|
|
31
32
|
polling.poller_poller_only_with_fd_mode: can only be set when polling.mode is :fd
|
data/docker-compose.oauth.yml
CHANGED
|
@@ -9,7 +9,7 @@ services:
|
|
|
9
9
|
KEYCLOAK_ADMIN: admin
|
|
10
10
|
KEYCLOAK_ADMIN_PASSWORD: admin
|
|
11
11
|
volumes:
|
|
12
|
-
- ./
|
|
12
|
+
- ./test/integrations/oauth_token_refresh/keycloak-realm.json:/opt/keycloak/data/import/realm.json
|
|
13
13
|
healthcheck:
|
|
14
14
|
test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080 && echo -e 'GET /realms/kafka HTTP/1.1\\r\\nHost: localhost\\r\\n\\r\\n' >&3 && timeout 1 cat <&3 | grep -q '200 OK'"]
|
|
15
15
|
interval: 10s
|
|
@@ -18,7 +18,7 @@ services:
|
|
|
18
18
|
start_period: 90s
|
|
19
19
|
|
|
20
20
|
kafka-oauth:
|
|
21
|
-
image: confluentinc/cp-kafka:8.
|
|
21
|
+
image: confluentinc/cp-kafka:8.2.0
|
|
22
22
|
container_name: kafka-oauth
|
|
23
23
|
depends_on:
|
|
24
24
|
keycloak:
|
|
@@ -53,4 +53,4 @@ services:
|
|
|
53
53
|
# Also allow HTTP URLs for JWKS endpoint (Kafka 3.0+ blocks non-HTTPS by default)
|
|
54
54
|
KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf -Dorg.apache.kafka.sasl.oauthbearer.allowed.urls=http://keycloak:8080/realms/kafka/protocol/openid-connect/certs"
|
|
55
55
|
volumes:
|
|
56
|
-
- ./
|
|
56
|
+
- ./test/integrations/oauth_token_refresh/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf:ro
|
data/docker-compose.yml
CHANGED
|
@@ -25,7 +25,12 @@ module WaterDrop
|
|
|
25
25
|
# Register statistics runner for this particular type of callbacks
|
|
26
26
|
::Karafka::Core::Instrumentation.statistics_callbacks.add(
|
|
27
27
|
producer.id,
|
|
28
|
-
Instrumentation::Callbacks::Statistics.new(
|
|
28
|
+
Instrumentation::Callbacks::Statistics.new(
|
|
29
|
+
producer.id,
|
|
30
|
+
client.name,
|
|
31
|
+
monitor,
|
|
32
|
+
producer.config.statistics_decorator
|
|
33
|
+
)
|
|
29
34
|
)
|
|
30
35
|
|
|
31
36
|
# Register error tracking callback
|
data/lib/waterdrop/config.rb
CHANGED
|
@@ -15,9 +15,9 @@ module WaterDrop
|
|
|
15
15
|
"statistics.interval.ms": 5_000,
|
|
16
16
|
# We set it to a value that is lower than `max_wait_timeout` to have a final verdict upon
|
|
17
17
|
# sync delivery
|
|
18
|
-
"message.timeout.ms":
|
|
18
|
+
"message.timeout.ms": 150_000,
|
|
19
19
|
# Must be more or equal to `message.timeout.ms` defaults
|
|
20
|
-
"transaction.timeout.ms":
|
|
20
|
+
"transaction.timeout.ms": 165_000,
|
|
21
21
|
# Lowers latency. Default in newer librdkafka but we want to make sure it is shipped to
|
|
22
22
|
# users despite what librdkafka they run on
|
|
23
23
|
"socket.nagle.disable": true
|
|
@@ -53,7 +53,7 @@ module WaterDrop
|
|
|
53
53
|
setting :max_payload_size, default: 1_000_012
|
|
54
54
|
# option [Integer] Wait that long for the delivery report or raise an error if this takes
|
|
55
55
|
# longer than the timeout ms.
|
|
56
|
-
setting :max_wait_timeout, default:
|
|
56
|
+
setting :max_wait_timeout, default: 180_000
|
|
57
57
|
# option [Boolean] should we upon detecting full librdkafka queue backoff and retry or should
|
|
58
58
|
# we raise an exception.
|
|
59
59
|
# When this is set to `true`, upon full queue, we won't raise an error. There will be error
|
|
@@ -116,6 +116,23 @@ module WaterDrop
|
|
|
116
116
|
# prevent overly aggressive disconnections.
|
|
117
117
|
setting :idle_disconnect_timeout, default: 0
|
|
118
118
|
|
|
119
|
+
# option [Karafka::Core::Monitoring::StatisticsDecorator] decorator instance used to compute
|
|
120
|
+
# deltas (_d) and freeze durations (_fd) on raw librdkafka statistics. The default is
|
|
121
|
+
# pre-configured with `only_keys` covering keys used by the built-in Datadog metrics
|
|
122
|
+
# listener and `excluded_keys` skipping subtrees not needed by producers (topics, broker
|
|
123
|
+
# window stats). Users who need additional decorated keys or full decoration can provide
|
|
124
|
+
# a custom decorator instance.
|
|
125
|
+
setting(
|
|
126
|
+
:statistics_decorator,
|
|
127
|
+
default: false,
|
|
128
|
+
constructor: lambda { |decorator|
|
|
129
|
+
decorator || ::Karafka::Core::Monitoring::StatisticsDecorator.new(
|
|
130
|
+
only_keys: %w[tx txretries txerrs rxerrs],
|
|
131
|
+
excluded_keys: %w[int_latency outbuf_latency rtt throttle req toppars topics]
|
|
132
|
+
)
|
|
133
|
+
}
|
|
134
|
+
)
|
|
135
|
+
|
|
119
136
|
# option [Boolean] should we send messages. Setting this to false can be really useful when
|
|
120
137
|
# testing and or developing because when set to false, won't actually ping Kafka but will
|
|
121
138
|
# run all the validations, etc
|
|
@@ -143,9 +160,9 @@ module WaterDrop
|
|
|
143
160
|
# Namespace for polling configuration
|
|
144
161
|
setting :polling do
|
|
145
162
|
# option [Symbol] Polling mode for handling producer callbacks
|
|
146
|
-
# :
|
|
147
|
-
# :
|
|
148
|
-
setting :mode, default: :
|
|
163
|
+
# :fd - uses a single global Ruby thread with IO.select-based multiplexing (default)
|
|
164
|
+
# :thread - uses librdkafka's native background polling threads
|
|
165
|
+
setting :mode, default: :fd
|
|
149
166
|
|
|
150
167
|
# option [WaterDrop::Polling::Poller, nil] Custom poller instance for isolation
|
|
151
168
|
# When nil (default), uses the global singleton poller shared by all FD-mode producers
|
|
@@ -33,10 +33,13 @@ module WaterDrop
|
|
|
33
33
|
required(:non_reloadable_errors) do |val|
|
|
34
34
|
val.is_a?(Array) && val.all?(Symbol)
|
|
35
35
|
end
|
|
36
|
+
|
|
36
37
|
required(:idle_disconnect_timeout) do |val|
|
|
37
38
|
val.is_a?(Integer) && (val.zero? || val >= 30_000)
|
|
38
39
|
end
|
|
39
40
|
|
|
41
|
+
required(:statistics_decorator) { |val| val.respond_to?(:call) }
|
|
42
|
+
|
|
40
43
|
nested(:oauth) do
|
|
41
44
|
required(:token_provider_listener) do |val|
|
|
42
45
|
val == false || val.respond_to?(:on_oauthbearer_token_refresh)
|
|
@@ -13,11 +13,13 @@ module WaterDrop
|
|
|
13
13
|
# @param producer_id [String]
|
|
14
14
|
# @param client_name [String] rdkafka client name
|
|
15
15
|
# @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
|
|
16
|
-
|
|
16
|
+
# @param statistics_decorator [Karafka::Core::Monitoring::StatisticsDecorator] decorator
|
|
17
|
+
# instance to use for computing deltas and freeze durations on raw librdkafka statistics
|
|
18
|
+
def initialize(producer_id, client_name, monitor, statistics_decorator)
|
|
17
19
|
@producer_id = producer_id
|
|
18
20
|
@client_name = client_name
|
|
19
21
|
@monitor = monitor
|
|
20
|
-
@statistics_decorator =
|
|
22
|
+
@statistics_decorator = statistics_decorator
|
|
21
23
|
end
|
|
22
24
|
|
|
23
25
|
# Emits decorated statistics to the monitor
|
|
@@ -81,7 +81,7 @@ module WaterDrop
|
|
|
81
81
|
#
|
|
82
82
|
# @producer.produce_async(*args, &block)
|
|
83
83
|
# ensure
|
|
84
|
-
# ref
|
|
84
|
+
# ref.delete(@producer.id)
|
|
85
85
|
# end
|
|
86
86
|
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
87
87
|
def #{method_name}(*args, &block)
|
|
@@ -90,7 +90,7 @@ module WaterDrop
|
|
|
90
90
|
|
|
91
91
|
@producer.#{method_name}(*args, &block)
|
|
92
92
|
ensure
|
|
93
|
-
ref
|
|
93
|
+
ref.delete(@producer.id)
|
|
94
94
|
end
|
|
95
95
|
RUBY
|
|
96
96
|
end
|
data/lib/waterdrop/version.rb
CHANGED
data/package-lock.json
CHANGED
|
@@ -217,9 +217,9 @@
|
|
|
217
217
|
}
|
|
218
218
|
},
|
|
219
219
|
"node_modules/picomatch": {
|
|
220
|
-
"version": "2.3.
|
|
221
|
-
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.
|
|
222
|
-
"integrity": "sha512-
|
|
220
|
+
"version": "2.3.2",
|
|
221
|
+
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz",
|
|
222
|
+
"integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==",
|
|
223
223
|
"dev": true,
|
|
224
224
|
"license": "MIT",
|
|
225
225
|
"engines": {
|
data/renovate.json
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
"Gemfile",
|
|
9
9
|
"Gemfile.lint",
|
|
10
10
|
"waterdrop.gemspec",
|
|
11
|
-
"
|
|
11
|
+
"test/integrations/**/Gemfile",
|
|
12
12
|
".github/workflows/**",
|
|
13
13
|
"docker-compose*.yml",
|
|
14
14
|
"package.json"
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
},
|
|
29
29
|
{
|
|
30
30
|
"matchFileNames": [
|
|
31
|
-
"
|
|
31
|
+
"test/integrations/**/Gemfile"
|
|
32
32
|
],
|
|
33
33
|
"groupName": "integration test dependencies",
|
|
34
34
|
"commitMessageTopic": "integration test dependencies"
|
data/waterdrop.gemspec
CHANGED
|
@@ -16,13 +16,13 @@ Gem::Specification.new do |spec|
|
|
|
16
16
|
spec.description = spec.summary
|
|
17
17
|
spec.licenses = %w[LGPL-3.0-only Commercial]
|
|
18
18
|
|
|
19
|
-
spec.add_dependency "karafka-core", ">= 2.
|
|
19
|
+
spec.add_dependency "karafka-core", ">= 2.5.12", "< 3.0.0"
|
|
20
20
|
spec.add_dependency "karafka-rdkafka", ">= 0.24.0"
|
|
21
21
|
spec.add_dependency "zeitwerk", "~> 2.3"
|
|
22
22
|
|
|
23
23
|
spec.required_ruby_version = ">= 3.2.0"
|
|
24
24
|
|
|
25
|
-
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
|
|
25
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match?(%r{^(spec|test|\.github|log)/}) }
|
|
26
26
|
spec.executables = []
|
|
27
27
|
spec.require_paths = %w[lib]
|
|
28
28
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: waterdrop
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.
|
|
4
|
+
version: 2.9.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Maciej Mensfeld
|
|
@@ -15,7 +15,7 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - ">="
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: 2.
|
|
18
|
+
version: 2.5.12
|
|
19
19
|
- - "<"
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
21
|
version: 3.0.0
|
|
@@ -25,7 +25,7 @@ dependencies:
|
|
|
25
25
|
requirements:
|
|
26
26
|
- - ">="
|
|
27
27
|
- !ruby/object:Gem::Version
|
|
28
|
-
version: 2.
|
|
28
|
+
version: 2.5.12
|
|
29
29
|
- - "<"
|
|
30
30
|
- !ruby/object:Gem::Version
|
|
31
31
|
version: 3.0.0
|
|
@@ -64,16 +64,7 @@ executables: []
|
|
|
64
64
|
extensions: []
|
|
65
65
|
extra_rdoc_files: []
|
|
66
66
|
files:
|
|
67
|
-
- ".github/CODEOWNERS"
|
|
68
|
-
- ".github/FUNDING.yml"
|
|
69
|
-
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
|
70
|
-
- ".github/ISSUE_TEMPLATE/feature_request.md"
|
|
71
|
-
- ".github/workflows/ci.yml"
|
|
72
|
-
- ".github/workflows/push.yml"
|
|
73
|
-
- ".github/workflows/trigger-wiki-refresh.yml"
|
|
74
|
-
- ".github/workflows/verify-action-pins.yml"
|
|
75
67
|
- ".gitignore"
|
|
76
|
-
- ".rspec"
|
|
77
68
|
- ".rubocop.yml"
|
|
78
69
|
- ".ruby-gemset"
|
|
79
70
|
- ".ruby-version"
|
|
@@ -137,7 +128,6 @@ files:
|
|
|
137
128
|
- lib/waterdrop/producer/transactions.rb
|
|
138
129
|
- lib/waterdrop/producer/variant.rb
|
|
139
130
|
- lib/waterdrop/version.rb
|
|
140
|
-
- log/.gitkeep
|
|
141
131
|
- package-lock.json
|
|
142
132
|
- package.json
|
|
143
133
|
- renovate.json
|
|
@@ -167,7 +157,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
167
157
|
- !ruby/object:Gem::Version
|
|
168
158
|
version: '0'
|
|
169
159
|
requirements: []
|
|
170
|
-
rubygems_version: 4.0.
|
|
160
|
+
rubygems_version: 4.0.6
|
|
171
161
|
specification_version: 4
|
|
172
162
|
summary: Kafka messaging made easy!
|
|
173
163
|
test_files: []
|
data/.github/CODEOWNERS
DELETED
data/.github/FUNDING.yml
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
custom: ['https://karafka.io/#become-pro']
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: Bug Report
|
|
3
|
-
about: Report an issue within the Karafka ecosystem you've discovered.
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
To make this process smoother for everyone involved, please read the following information before filling out the template.
|
|
7
|
-
|
|
8
|
-
Scope of the OSS Support
|
|
9
|
-
===========
|
|
10
|
-
|
|
11
|
-
We do not provide OSS support for outdated versions of Karafka and its components.
|
|
12
|
-
|
|
13
|
-
Please ensure that you are using a version that is still actively supported. We cannot assist with any no longer maintained versions unless you support us with our Pro offering (https://karafka.io/docs/Pro-Support/).
|
|
14
|
-
|
|
15
|
-
We acknowledge that understanding the specifics of your application and its configuration can be essential for resolving certain issues. However, due to the extensive time and resources such analysis can require, this may fall beyond our Open Source Support scope.
|
|
16
|
-
|
|
17
|
-
If Karafka or its components are critical to your infrastructure, we encourage you to consider our Pro Offering.
|
|
18
|
-
|
|
19
|
-
By backing us up, you can gain direct assistance and ensure your use case receives the dedicated attention it deserves.
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
Important Links to Read
|
|
23
|
-
===========
|
|
24
|
-
|
|
25
|
-
Please take a moment to review the following resources before submitting your report:
|
|
26
|
-
|
|
27
|
-
- Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
|
|
28
|
-
- Support Policy: https://karafka.io/docs/Support/
|
|
29
|
-
- Versions, Lifecycle, and EOL: https://karafka.io/docs/Versions-Lifecycle-and-EOL/
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
Bug Report Details
|
|
33
|
-
===========
|
|
34
|
-
|
|
35
|
-
Please provide all the details per our Issue Reporting Guide: https://karafka.io/docs/Support/#issue-reporting-guide
|
|
36
|
-
|
|
37
|
-
Failing to provide the required details may result in the issue being closed. Please include all necessary information to help us understand and resolve your issue effectively.
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
Additional Context
|
|
41
|
-
===========
|
|
42
|
-
|
|
43
|
-
Add any other context about the problem here.
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: Feature Request
|
|
3
|
-
about: Suggest new WaterDrop features or improvements to existing features.
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
## Is your feature request related to a problem? Please describe.
|
|
7
|
-
|
|
8
|
-
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
|
9
|
-
|
|
10
|
-
## Describe the solution you'd like
|
|
11
|
-
|
|
12
|
-
A clear and concise description of what you want to happen.
|
|
13
|
-
|
|
14
|
-
## Describe alternatives you've considered
|
|
15
|
-
|
|
16
|
-
A clear and concise description of any alternative solutions or features you've considered.
|
|
17
|
-
|
|
18
|
-
## Additional context
|
|
19
|
-
|
|
20
|
-
Add any other context or screenshots about the feature request here.
|
data/.github/workflows/ci.yml
DELETED
|
@@ -1,322 +0,0 @@
|
|
|
1
|
-
name: CI
|
|
2
|
-
|
|
3
|
-
concurrency:
|
|
4
|
-
group: ${{ github.workflow }}-${{ github.ref }}
|
|
5
|
-
cancel-in-progress: true
|
|
6
|
-
|
|
7
|
-
on:
|
|
8
|
-
pull_request:
|
|
9
|
-
branches: [ master ]
|
|
10
|
-
schedule:
|
|
11
|
-
- cron: '0 1 * * *'
|
|
12
|
-
|
|
13
|
-
permissions:
|
|
14
|
-
contents: read
|
|
15
|
-
|
|
16
|
-
jobs:
|
|
17
|
-
specs:
|
|
18
|
-
timeout-minutes: 15
|
|
19
|
-
runs-on: ubuntu-latest
|
|
20
|
-
strategy:
|
|
21
|
-
fail-fast: false
|
|
22
|
-
matrix:
|
|
23
|
-
ruby:
|
|
24
|
-
- '4.0'
|
|
25
|
-
- '3.4'
|
|
26
|
-
- '3.3'
|
|
27
|
-
- '3.2'
|
|
28
|
-
polling_mode:
|
|
29
|
-
- 'thread'
|
|
30
|
-
- 'fiber'
|
|
31
|
-
include:
|
|
32
|
-
- ruby: '3.4'
|
|
33
|
-
polling_mode: 'thread'
|
|
34
|
-
coverage: 'true'
|
|
35
|
-
steps:
|
|
36
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
37
|
-
with:
|
|
38
|
-
fetch-depth: 0
|
|
39
|
-
|
|
40
|
-
- name: Install package dependencies
|
|
41
|
-
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
|
42
|
-
|
|
43
|
-
- name: Set up Ruby
|
|
44
|
-
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
|
45
|
-
with:
|
|
46
|
-
ruby-version: ${{matrix.ruby}}
|
|
47
|
-
bundler-cache: true
|
|
48
|
-
bundler: 'latest'
|
|
49
|
-
self-hosted: false
|
|
50
|
-
|
|
51
|
-
- name: Run Kafka with Docker Compose
|
|
52
|
-
run: |
|
|
53
|
-
docker compose up -d || (sleep 5 && docker compose up -d)
|
|
54
|
-
|
|
55
|
-
- name: Wait for Kafka
|
|
56
|
-
run: sleep 15
|
|
57
|
-
|
|
58
|
-
- name: Install latest bundler
|
|
59
|
-
run: |
|
|
60
|
-
gem install bundler --no-document
|
|
61
|
-
gem update --system --no-document
|
|
62
|
-
bundle config set without 'tools benchmarks docs'
|
|
63
|
-
|
|
64
|
-
- name: Bundle install
|
|
65
|
-
run: |
|
|
66
|
-
bundle config set without development
|
|
67
|
-
bundle install --jobs 4 --retry 3
|
|
68
|
-
|
|
69
|
-
- name: Run all tests
|
|
70
|
-
env:
|
|
71
|
-
GITHUB_COVERAGE: ${{matrix.coverage}}
|
|
72
|
-
FD_POLLING: ${{ matrix.polling_mode == 'fiber' && 'true' || 'false' }}
|
|
73
|
-
run: bundle exec rspec
|
|
74
|
-
|
|
75
|
-
- name: Run integration tests
|
|
76
|
-
env:
|
|
77
|
-
FD_POLLING: ${{ matrix.polling_mode == 'fiber' && 'true' || 'false' }}
|
|
78
|
-
# Exclude oauth_token_refresh - requires separate Keycloak + OAuth Kafka infrastructure
|
|
79
|
-
run: ./bin/integrations --exclude oauth_token_refresh
|
|
80
|
-
|
|
81
|
-
- name: Check Kafka logs for unexpected warnings
|
|
82
|
-
run: bin/verify_kafka_warnings
|
|
83
|
-
|
|
84
|
-
- name: Check test topics naming convention
|
|
85
|
-
run: bin/verify_topics_naming
|
|
86
|
-
|
|
87
|
-
rubocop:
|
|
88
|
-
timeout-minutes: 5
|
|
89
|
-
runs-on: ubuntu-latest
|
|
90
|
-
env:
|
|
91
|
-
BUNDLE_GEMFILE: Gemfile.lint
|
|
92
|
-
steps:
|
|
93
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
94
|
-
with:
|
|
95
|
-
fetch-depth: 0
|
|
96
|
-
- name: Set up Ruby
|
|
97
|
-
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
|
98
|
-
with:
|
|
99
|
-
ruby-version: '4.0.1'
|
|
100
|
-
bundler-cache: true
|
|
101
|
-
- name: Run rubocop
|
|
102
|
-
run: bundle exec rubocop
|
|
103
|
-
|
|
104
|
-
yard-lint:
|
|
105
|
-
timeout-minutes: 5
|
|
106
|
-
runs-on: ubuntu-latest
|
|
107
|
-
env:
|
|
108
|
-
BUNDLE_GEMFILE: Gemfile.lint
|
|
109
|
-
steps:
|
|
110
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
111
|
-
with:
|
|
112
|
-
fetch-depth: 0
|
|
113
|
-
- name: Set up Ruby
|
|
114
|
-
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
|
115
|
-
with:
|
|
116
|
-
ruby-version: '4.0.1'
|
|
117
|
-
bundler-cache: true
|
|
118
|
-
- name: Run yard-lint
|
|
119
|
-
run: bundle exec yard-lint lib/
|
|
120
|
-
|
|
121
|
-
lostconf:
|
|
122
|
-
timeout-minutes: 5
|
|
123
|
-
runs-on: ubuntu-latest
|
|
124
|
-
steps:
|
|
125
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
126
|
-
with:
|
|
127
|
-
fetch-depth: 0
|
|
128
|
-
- name: Set up Node.js
|
|
129
|
-
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
|
|
130
|
-
with:
|
|
131
|
-
node-version: '20'
|
|
132
|
-
cache: 'npm'
|
|
133
|
-
- name: Install dependencies
|
|
134
|
-
run: npm ci
|
|
135
|
-
- name: Run lostconf
|
|
136
|
-
run: npx lostconf --fail-on-stale
|
|
137
|
-
|
|
138
|
-
oauth-integration:
|
|
139
|
-
timeout-minutes: 10
|
|
140
|
-
runs-on: ubuntu-latest
|
|
141
|
-
strategy:
|
|
142
|
-
fail-fast: false
|
|
143
|
-
matrix:
|
|
144
|
-
polling_mode:
|
|
145
|
-
- 'thread'
|
|
146
|
-
- 'fiber'
|
|
147
|
-
steps:
|
|
148
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
149
|
-
with:
|
|
150
|
-
fetch-depth: 0
|
|
151
|
-
|
|
152
|
-
- name: Set up Ruby
|
|
153
|
-
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
|
154
|
-
with:
|
|
155
|
-
ruby-version: '4.0.1'
|
|
156
|
-
bundler-cache: true
|
|
157
|
-
bundler: 'latest'
|
|
158
|
-
|
|
159
|
-
- name: Start OAuth infrastructure (Keycloak + Kafka with SASL/OAUTHBEARER)
|
|
160
|
-
run: docker compose -f docker-compose.oauth.yml up -d
|
|
161
|
-
|
|
162
|
-
- name: Wait for Keycloak to be ready
|
|
163
|
-
run: |
|
|
164
|
-
echo "Waiting for Keycloak to be ready..."
|
|
165
|
-
for i in {1..60}; do
|
|
166
|
-
if curl -sf http://localhost:8080/realms/kafka > /dev/null 2>&1; then
|
|
167
|
-
echo "Keycloak is ready!"
|
|
168
|
-
break
|
|
169
|
-
fi
|
|
170
|
-
echo "Waiting... ($i/60)"
|
|
171
|
-
sleep 2
|
|
172
|
-
done
|
|
173
|
-
|
|
174
|
-
- name: Wait for Kafka OAuth broker to be ready
|
|
175
|
-
run: |
|
|
176
|
-
echo "Waiting for Kafka OAuth broker..."
|
|
177
|
-
sleep 10
|
|
178
|
-
|
|
179
|
-
- name: Install dependencies
|
|
180
|
-
run: |
|
|
181
|
-
gem install bundler --no-document
|
|
182
|
-
bundle config set without 'tools benchmarks docs development'
|
|
183
|
-
bundle install --jobs 4 --retry 3
|
|
184
|
-
|
|
185
|
-
- name: Run OAuth integration test
|
|
186
|
-
env:
|
|
187
|
-
FD_POLLING: ${{ matrix.polling_mode == 'fiber' && 'true' || 'false' }}
|
|
188
|
-
run: ./bin/integrations oauth_token_refresh
|
|
189
|
-
|
|
190
|
-
- name: Show Keycloak logs on failure
|
|
191
|
-
if: failure()
|
|
192
|
-
run: docker compose -f docker-compose.oauth.yml logs keycloak
|
|
193
|
-
|
|
194
|
-
- name: Show Kafka OAuth logs on failure
|
|
195
|
-
if: failure()
|
|
196
|
-
run: docker compose -f docker-compose.oauth.yml logs kafka-oauth
|
|
197
|
-
|
|
198
|
-
specs-macos:
|
|
199
|
-
timeout-minutes: 30
|
|
200
|
-
runs-on: macos-15
|
|
201
|
-
env:
|
|
202
|
-
CONFLUENT_VERSION: "8.0.0"
|
|
203
|
-
strategy:
|
|
204
|
-
fail-fast: false
|
|
205
|
-
matrix:
|
|
206
|
-
polling_mode:
|
|
207
|
-
- 'thread'
|
|
208
|
-
- 'fiber'
|
|
209
|
-
steps:
|
|
210
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
211
|
-
with:
|
|
212
|
-
fetch-depth: 0
|
|
213
|
-
|
|
214
|
-
- name: Install Bash 4+
|
|
215
|
-
run: |
|
|
216
|
-
brew install bash
|
|
217
|
-
echo "/opt/homebrew/bin" >> $GITHUB_PATH
|
|
218
|
-
|
|
219
|
-
- name: Set up Ruby
|
|
220
|
-
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
|
221
|
-
with:
|
|
222
|
-
ruby-version: '4.0.1'
|
|
223
|
-
bundler-cache: true
|
|
224
|
-
bundler: 'latest'
|
|
225
|
-
|
|
226
|
-
- name: Install and Start Confluent Community Kafka (KRaft)
|
|
227
|
-
run: |
|
|
228
|
-
brew install openjdk@17
|
|
229
|
-
export PATH="/opt/homebrew/opt/openjdk@17/bin:$PATH"
|
|
230
|
-
export JAVA_HOME="/opt/homebrew/opt/openjdk@17"
|
|
231
|
-
|
|
232
|
-
curl -O "https://packages.confluent.io/archive/8.0/confluent-community-${CONFLUENT_VERSION}.tar.gz"
|
|
233
|
-
tar -xzf "confluent-community-${CONFLUENT_VERSION}.tar.gz"
|
|
234
|
-
|
|
235
|
-
export CONFLUENT_HOME="$(pwd)/confluent-${CONFLUENT_VERSION}"
|
|
236
|
-
export PATH="$CONFLUENT_HOME/bin:$PATH"
|
|
237
|
-
cd "$CONFLUENT_HOME"
|
|
238
|
-
|
|
239
|
-
KRAFT_CONFIG=""
|
|
240
|
-
for config in "etc/kafka/kraft/server.properties" "config/kraft/server.properties" "etc/kafka/server.properties"; do
|
|
241
|
-
if [ -f "$config" ]; then
|
|
242
|
-
KRAFT_CONFIG="$config"
|
|
243
|
-
echo "Found config: $KRAFT_CONFIG"
|
|
244
|
-
break
|
|
245
|
-
fi
|
|
246
|
-
done
|
|
247
|
-
|
|
248
|
-
if [ -z "$KRAFT_CONFIG" ]; then
|
|
249
|
-
echo "❌ No server config found"
|
|
250
|
-
exit 1
|
|
251
|
-
fi
|
|
252
|
-
|
|
253
|
-
cat >> "$KRAFT_CONFIG" << 'EOF'
|
|
254
|
-
|
|
255
|
-
process.roles=broker,controller
|
|
256
|
-
node.id=1
|
|
257
|
-
controller.quorum.voters=1@127.0.0.1:9093
|
|
258
|
-
listeners=PLAINTEXT://127.0.0.1:9092,CONTROLLER://127.0.0.1:9093
|
|
259
|
-
advertised.listeners=PLAINTEXT://127.0.0.1:9092
|
|
260
|
-
controller.listener.names=CONTROLLER
|
|
261
|
-
inter.broker.listener.name=PLAINTEXT
|
|
262
|
-
log.dirs=/tmp/kraft-combined-logs
|
|
263
|
-
authorizer.class.name=org.apache.kafka.metadata.authorizer.StandardAuthorizer
|
|
264
|
-
super.users=User:ANONYMOUS
|
|
265
|
-
allow.everyone.if.no.acl.found=true
|
|
266
|
-
EOF
|
|
267
|
-
|
|
268
|
-
CLUSTER_ID=$(bin/kafka-storage random-uuid)
|
|
269
|
-
bin/kafka-storage format -t "$CLUSTER_ID" -c "$KRAFT_CONFIG"
|
|
270
|
-
bin/kafka-server-start "$KRAFT_CONFIG" &
|
|
271
|
-
|
|
272
|
-
sleep 20
|
|
273
|
-
|
|
274
|
-
for i in {1..30}; do
|
|
275
|
-
if bin/kafka-topics --bootstrap-server 127.0.0.1:9092 --list >/dev/null 2>&1; then
|
|
276
|
-
echo "✅ Confluent Community ${CONFLUENT_VERSION} (KRaft) is ready!"
|
|
277
|
-
break
|
|
278
|
-
fi
|
|
279
|
-
[ $i -eq 30 ] && { echo "❌ Kafka failed to start"; exit 1; }
|
|
280
|
-
sleep 2
|
|
281
|
-
done
|
|
282
|
-
|
|
283
|
-
- name: Install latest bundler
|
|
284
|
-
run: |
|
|
285
|
-
gem install bundler --no-document
|
|
286
|
-
gem update --system --no-document
|
|
287
|
-
bundle config set without 'tools benchmarks docs'
|
|
288
|
-
|
|
289
|
-
- name: Bundle install
|
|
290
|
-
run: |
|
|
291
|
-
bundle config set without development
|
|
292
|
-
bundle install --jobs 4 --retry 3
|
|
293
|
-
|
|
294
|
-
- name: Run all tests
|
|
295
|
-
env:
|
|
296
|
-
FD_POLLING: ${{ matrix.polling_mode == 'fiber' && 'true' || 'false' }}
|
|
297
|
-
run: bundle exec rspec
|
|
298
|
-
|
|
299
|
-
- name: Run integration tests
|
|
300
|
-
env:
|
|
301
|
-
FD_POLLING: ${{ matrix.polling_mode == 'fiber' && 'true' || 'false' }}
|
|
302
|
-
run: ./bin/integrations --exclude oauth_token_refresh
|
|
303
|
-
|
|
304
|
-
ci-success:
|
|
305
|
-
name: CI Success
|
|
306
|
-
runs-on: ubuntu-latest
|
|
307
|
-
if: always()
|
|
308
|
-
needs:
|
|
309
|
-
- rubocop
|
|
310
|
-
- specs
|
|
311
|
-
- specs-macos
|
|
312
|
-
- yard-lint
|
|
313
|
-
- lostconf
|
|
314
|
-
- oauth-integration
|
|
315
|
-
steps:
|
|
316
|
-
- name: Check all jobs passed
|
|
317
|
-
if: |
|
|
318
|
-
contains(needs.*.result, 'failure') ||
|
|
319
|
-
contains(needs.*.result, 'cancelled') ||
|
|
320
|
-
contains(needs.*.result, 'skipped')
|
|
321
|
-
run: exit 1
|
|
322
|
-
- run: echo "All CI checks passed!"
|
data/.github/workflows/push.yml
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
name: Push Gem
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
push:
|
|
5
|
-
tags:
|
|
6
|
-
- v*
|
|
7
|
-
|
|
8
|
-
permissions:
|
|
9
|
-
contents: read
|
|
10
|
-
|
|
11
|
-
jobs:
|
|
12
|
-
push:
|
|
13
|
-
if: github.repository_owner == 'karafka'
|
|
14
|
-
runs-on: ubuntu-latest
|
|
15
|
-
environment: deployment
|
|
16
|
-
|
|
17
|
-
permissions:
|
|
18
|
-
contents: write
|
|
19
|
-
id-token: write
|
|
20
|
-
|
|
21
|
-
steps:
|
|
22
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
23
|
-
with:
|
|
24
|
-
fetch-depth: 0
|
|
25
|
-
|
|
26
|
-
- name: Set up Ruby
|
|
27
|
-
uses: ruby/setup-ruby@09a7688d3b55cf0e976497ff046b70949eeaccfd # v1.288.0
|
|
28
|
-
with:
|
|
29
|
-
bundler-cache: false
|
|
30
|
-
|
|
31
|
-
- name: Bundle install
|
|
32
|
-
run: |
|
|
33
|
-
bundle install --jobs 4 --retry 3
|
|
34
|
-
|
|
35
|
-
- uses: rubygems/release-gem@1c162a739e8b4cb21a676e97b087e8268d8fc40b # v1.1.2
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
name: Trigger Wiki Refresh
|
|
2
|
-
|
|
3
|
-
on:
|
|
4
|
-
release:
|
|
5
|
-
types: [published]
|
|
6
|
-
push:
|
|
7
|
-
branches: [master]
|
|
8
|
-
|
|
9
|
-
jobs:
|
|
10
|
-
trigger-wiki-refresh:
|
|
11
|
-
runs-on: ubuntu-latest
|
|
12
|
-
environment: wiki-trigger
|
|
13
|
-
if: github.repository_owner == 'karafka'
|
|
14
|
-
steps:
|
|
15
|
-
- name: Trigger wiki refresh
|
|
16
|
-
uses: peter-evans/repository-dispatch@28959ce8df70de7be546dd1250a005dd32156697 # v4.0.1
|
|
17
|
-
with:
|
|
18
|
-
token: ${{ secrets.WIKI_REPO_TOKEN }}
|
|
19
|
-
repository: karafka/wiki
|
|
20
|
-
event-type: sync-trigger
|
|
21
|
-
client-payload: |
|
|
22
|
-
{
|
|
23
|
-
"repository": "${{ github.repository }}",
|
|
24
|
-
"event_name": "${{ github.event_name }}",
|
|
25
|
-
"release_tag": "${{ github.event.release.tag_name || '' }}",
|
|
26
|
-
"release_name": "${{ github.event.release.name || '' }}",
|
|
27
|
-
"commit_sha": "${{ github.sha }}",
|
|
28
|
-
"commit_message": "Trigger Wiki Refresh",
|
|
29
|
-
"triggered_by": "${{ github.actor }}"
|
|
30
|
-
}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
name: Verify Action Pins
|
|
2
|
-
on:
|
|
3
|
-
pull_request:
|
|
4
|
-
paths:
|
|
5
|
-
- '.github/workflows/**'
|
|
6
|
-
jobs:
|
|
7
|
-
verify_action_pins:
|
|
8
|
-
runs-on: ubuntu-latest
|
|
9
|
-
steps:
|
|
10
|
-
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
11
|
-
- name: Check SHA pins
|
|
12
|
-
run: |
|
|
13
|
-
if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
|
|
14
|
-
echo "::error::Actions should use SHA pins, not tags or branch names"
|
|
15
|
-
exit 1
|
|
16
|
-
fi
|
data/.rspec
DELETED
data/log/.gitkeep
DELETED
|
File without changes
|