karafka-core 2.6.1 → 2.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +10 -10
- data/.github/workflows/push.yml +3 -3
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/.gitignore +1 -0
- data/.ruby-version +1 -1
- data/.yard-lint.yml +29 -1
- data/CHANGELOG.md +27 -3
- data/Gemfile.lint.lock +8 -7
- data/Gemfile.lock +9 -15
- data/lib/karafka/core/configurable/injector.rb +89 -0
- data/lib/karafka/core/configurable/node.rb +30 -6
- data/lib/karafka/core/contractable/contract.rb +37 -9
- data/lib/karafka/core/helpers/minitest_locator.rb +2 -2
- data/lib/karafka/core/helpers/rspec_locator.rb +2 -2
- data/lib/karafka/core/instrumentation/callbacks_manager.rb +17 -13
- data/lib/karafka/core/monitoring/notifications.rb +20 -9
- data/lib/karafka/core/monitoring/statistics_decorator.rb +53 -30
- data/lib/karafka/core/monitoring.rb +0 -1
- data/lib/karafka/core/patches/rdkafka/bindings.rb +21 -2
- data/lib/karafka/core/taggable/tags.rb +5 -1
- data/lib/karafka/core/version.rb +2 -3
- data/lib/karafka-core.rb +4 -2
- data/package-lock.json +3 -3
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 70aa3940193aebd6b4f232950a2b765ee57025aea6b3d3d7c72234429f904de4
|
|
4
|
+
data.tar.gz: e4a550006b517eea864370ef12534dacdb6fddae40f3adf7308981fa99a1e7a1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: '05908e99fd999ee007bcb3efd46a6a3b949d88e40d980c5ca4e8aa90a3c312755717dc8feef7a7de42c5c7c235e3ac623e26b91e9a3456568e4d552a2d8782ea'
|
|
7
|
+
data.tar.gz: 61456523b5822db6c78ad36c839ed6e3bdcbd9bc18d56cebfb95947443296aad94b917b3bef7798f093b471d703ae9d5a7f43d1b09d726ea655e0b2a0b091a3c
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -29,7 +29,7 @@ jobs:
|
|
|
29
29
|
- ruby: '4.0'
|
|
30
30
|
coverage: 'true'
|
|
31
31
|
steps:
|
|
32
|
-
- uses: actions/checkout@
|
|
32
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
33
33
|
with:
|
|
34
34
|
fetch-depth: 0
|
|
35
35
|
|
|
@@ -37,7 +37,7 @@ jobs:
|
|
|
37
37
|
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
|
38
38
|
|
|
39
39
|
- name: Set up Ruby
|
|
40
|
-
uses: ruby/setup-ruby@
|
|
40
|
+
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
41
41
|
with:
|
|
42
42
|
ruby-version: ${{matrix.ruby}}
|
|
43
43
|
bundler: 'latest'
|
|
@@ -65,13 +65,13 @@ jobs:
|
|
|
65
65
|
env:
|
|
66
66
|
BUNDLE_GEMFILE: Gemfile.lint
|
|
67
67
|
steps:
|
|
68
|
-
- uses: actions/checkout@
|
|
68
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
69
69
|
with:
|
|
70
70
|
fetch-depth: 0
|
|
71
71
|
- name: Set up Ruby
|
|
72
|
-
uses: ruby/setup-ruby@
|
|
72
|
+
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
73
73
|
with:
|
|
74
|
-
ruby-version: '4.0.
|
|
74
|
+
ruby-version: '4.0.6'
|
|
75
75
|
bundler-cache: true
|
|
76
76
|
- name: Run rubocop
|
|
77
77
|
run: bundle exec rubocop
|
|
@@ -82,13 +82,13 @@ jobs:
|
|
|
82
82
|
env:
|
|
83
83
|
BUNDLE_GEMFILE: Gemfile.lint
|
|
84
84
|
steps:
|
|
85
|
-
- uses: actions/checkout@
|
|
85
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
86
86
|
with:
|
|
87
87
|
fetch-depth: 0
|
|
88
88
|
- name: Set up Ruby
|
|
89
|
-
uses: ruby/setup-ruby@
|
|
89
|
+
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
90
90
|
with:
|
|
91
|
-
ruby-version: '4.0.
|
|
91
|
+
ruby-version: '4.0.6'
|
|
92
92
|
bundler-cache: true
|
|
93
93
|
- name: Run yard-lint
|
|
94
94
|
run: bundle exec yard-lint lib/
|
|
@@ -97,11 +97,11 @@ jobs:
|
|
|
97
97
|
timeout-minutes: 5
|
|
98
98
|
runs-on: ubuntu-latest
|
|
99
99
|
steps:
|
|
100
|
-
- uses: actions/checkout@
|
|
100
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
101
101
|
with:
|
|
102
102
|
fetch-depth: 0
|
|
103
103
|
- name: Set up Node.js
|
|
104
|
-
uses: actions/setup-node@
|
|
104
|
+
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
105
105
|
with:
|
|
106
106
|
node-version: '20'
|
|
107
107
|
cache: 'npm'
|
data/.github/workflows/push.yml
CHANGED
|
@@ -19,12 +19,12 @@ jobs:
|
|
|
19
19
|
id-token: write
|
|
20
20
|
|
|
21
21
|
steps:
|
|
22
|
-
- uses: actions/checkout@
|
|
22
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
23
23
|
with:
|
|
24
24
|
fetch-depth: 0
|
|
25
25
|
|
|
26
26
|
- name: Set up Ruby
|
|
27
|
-
uses: ruby/setup-ruby@
|
|
27
|
+
uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
|
|
28
28
|
with:
|
|
29
29
|
bundler-cache: false
|
|
30
30
|
|
|
@@ -32,4 +32,4 @@ jobs:
|
|
|
32
32
|
run: |
|
|
33
33
|
bundle install --jobs 4 --retry 3
|
|
34
34
|
|
|
35
|
-
- uses: rubygems/release-gem@
|
|
35
|
+
- uses: rubygems/release-gem@7f9650160c1a4e7989fdc9855807bdbd421d8b6b # v1.4.1
|
|
@@ -7,7 +7,7 @@ jobs:
|
|
|
7
7
|
verify_action_pins:
|
|
8
8
|
runs-on: ubuntu-latest
|
|
9
9
|
steps:
|
|
10
|
-
- uses: actions/checkout@
|
|
10
|
+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
11
11
|
- name: Check SHA pins
|
|
12
12
|
run: |
|
|
13
13
|
if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
|
data/.gitignore
CHANGED
data/.ruby-version
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
4.0.
|
|
1
|
+
4.0.6
|
data/.yard-lint.yml
CHANGED
|
@@ -62,6 +62,34 @@ Documentation/BlankLineBeforeDefinition:
|
|
|
62
62
|
SingleBlankLine: true
|
|
63
63
|
OrphanedDocs: true
|
|
64
64
|
|
|
65
|
+
Documentation/DuplicateNamespaceComment:
|
|
66
|
+
Description: Detects namespaces documented with a YARD comment in more than one file.
|
|
67
|
+
Enabled: true
|
|
68
|
+
Severity: error
|
|
69
|
+
|
|
70
|
+
Documentation/UnderfilledLines:
|
|
71
|
+
Description: Detects documentation prose that wraps too early and wastes horizontal space.
|
|
72
|
+
Enabled: true
|
|
73
|
+
Severity: error
|
|
74
|
+
# Aligned with RuboCop's Layout/LineLength (Max: 100) so documentation prose
|
|
75
|
+
# wraps to the same width as code.
|
|
76
|
+
MaxLength: 100
|
|
77
|
+
|
|
78
|
+
Documentation/LineLength:
|
|
79
|
+
Description: Detects documentation lines that exceed the maximum length.
|
|
80
|
+
Enabled: true
|
|
81
|
+
Severity: error
|
|
82
|
+
# Aligned with RuboCop's Layout/LineLength (Max: 100).
|
|
83
|
+
MaxLength: 100
|
|
84
|
+
|
|
85
|
+
Documentation/TextSubstitution:
|
|
86
|
+
Description: Detects em/en-dashes in documentation and replaces them with hyphens.
|
|
87
|
+
Enabled: true
|
|
88
|
+
Severity: error
|
|
89
|
+
Substitutions:
|
|
90
|
+
"—": "-" # em-dash (U+2014)
|
|
91
|
+
"–": "-" # en-dash (U+2013)
|
|
92
|
+
|
|
65
93
|
# Tags validators
|
|
66
94
|
Tags/Order:
|
|
67
95
|
Description: Enforces consistent ordering of YARD tags.
|
|
@@ -142,7 +170,7 @@ Tags/OptionTags:
|
|
|
142
170
|
Tags/ExampleSyntax:
|
|
143
171
|
Description: Validates Ruby syntax in @example tags.
|
|
144
172
|
Enabled: true
|
|
145
|
-
Severity:
|
|
173
|
+
Severity: error
|
|
146
174
|
|
|
147
175
|
Tags/RedundantParamDescription:
|
|
148
176
|
Description: Detects meaningless parameter descriptions that add no value.
|
data/CHANGELOG.md
CHANGED
|
@@ -1,11 +1,34 @@
|
|
|
1
1
|
# Karafka Core Changelog
|
|
2
2
|
|
|
3
|
+
## 2.6.3 (2026-08-31)
|
|
4
|
+
- [Enhancement] Add `Configurable::Injector`, a reusable base for injecting default values into a config hash without overwriting user-provided settings, with support for layering extra defaults on top.
|
|
5
|
+
|
|
6
|
+
## 2.6.2 (2026-06-29)
|
|
7
|
+
- [Enhancement] Document that a leaf's `default` value is shared by reference across all config instances, so a mutable default (e.g. `default: []`) mutated in place is visible everywhere. Assign it inside a `configure` block if you need a per-instance copy.
|
|
8
|
+
- [Fix] `require "pathname"` explicitly in `lib/karafka-core.rb` (with the other top-level requires). `Karafka::Core.gem_root` returns a `Pathname`, but the gem never required `pathname` -- it only worked because Bundler (or another gem) happened to load it first. In an environment where nothing else loads it, `gem_root` raised `NameError: uninitialized constant Pathname`.
|
|
9
|
+
- [Enhancement] Document that a `virtual` rule must return a freshly built `Array` of `[path, message]` error pairs on each call; a memoized, shared or frozen array is unsupported.
|
|
10
|
+
- [Fix] `Configurable::Node#register` raises the documented "already registered" `ArgumentError` for a name already used by an unread lazy-with-constructor setting. The duplicate guard only checked `@configs_refs`, but a lazy setting with a constructor is absent from it until first read, so `register` silently overwrote it; it now checks the defined children.
|
|
11
|
+
- [Fix] `Contractable::Contract.nested` now pops its path in an `ensure`. If the block raised while the contract was being defined and the caller rescued it, the path stayed on the nesting stack and was prefixed onto every rule defined afterwards.
|
|
12
|
+
- [Fix] `Contract#call` no longer raises `NoMethodError` when validating a non-Hash root with a 1-key or 2-key rule path; it reports the path as missing, consistent with the 3+-key path (and the non-Hash intermediate handling added in 2.6.1).
|
|
13
|
+
- [Fix] Honor `excluded_keys` containing `"cgrp"` in `StatisticsDecorator` `only_keys` mode. The `cgrp` branch of the structure-aware fast path lacked the exclusion guard that the `brokers` and `topics` branches have, so excluding the consumer-group subtree still decorated it (inconsistent with the full-decoration path).
|
|
14
|
+
- [Fix] Guard the patched rdkafka error callback against a null client pointer. librdkafka can invoke the error callback with a NULL `rd_kafka_t` (e.g. very early in client construction); calling `rd_kafka_name` on it dereferenced the null pointer and could segfault the process. Mirrors the upstream `ErrorCallback`.
|
|
15
|
+
- [Fix] Resolve fatal errors in the patched rdkafka error callback. `ERR__FATAL` is only a generic marker, so the callback now fetches the real underlying error code and description via `RdkafkaError.build_fatal` (`rd_kafka_fatal_error`) instead of reporting the generic fatal code. Mirrors the upstream `ErrorCallback`.
|
|
16
|
+
- [Fix] A lazy setting declared without a constructor (`setting(:x, lazy: true)`) no longer raises when read; it now behaves like a regular setting backed by its default.
|
|
17
|
+
- [Fix] `Contract#call` no longer raises `NoMethodError` when a virtual rule returns `false`. A virtual rule now signals "no errors" with any non-Array result (`true`/`false`/`nil`); only an `Array` of error pairs is collected. Previously a `false` return reached `false.each` (a `nil` return was already tolerated).
|
|
18
|
+
- [Fix] Make `CallbacksManager` thread-safe under concurrent `add`/`delete` during dispatch. A callback registered or removed while callbacks were firing could be lost — kept firing after removal, or never fired after being added. librdkafka fires these from a background thread, so it was reachable in practice.
|
|
19
|
+
- [Fix] Make `Notifications` subscriptions thread-safe: a listener that unsubscribes itself (or another) from within its own handler no longer causes the next listener to be skipped, and concurrent subscribe/unsubscribe during dispatch is now safe.
|
|
20
|
+
- [Fix] Report a freeze duration (`_fd`) of `0` for statistics keys that first appear mid-stream (e.g. a new broker or partition) instead of the time since the previous emission, which was incorrect and made the related spec flaky.
|
|
21
|
+
- [Fix] Make assigning a setting on a frozen `Configurable::Node` atomic. The ivar-backed writer evaluated `@configs_refs[name] = value` before `instance_variable_set`, so a frozen node mutated the canonical store and only then raised `FrozenError`, leaving the store and the ivar-backed reader permanently out of sync. It now raises before touching any state.
|
|
22
|
+
- [Fix] `Configurable::Node#to_h` no longer raises for a setting using the `->(default) { ... }` constructor form when its value has not been set yet (e.g. `#to_h` on an unconfigured instance or an unread lazy setting).
|
|
23
|
+
- [Fix] Honor `excluded_keys` inside `StatisticsDecorator` `only_keys` decoration. A key listed in both `only_keys` and `excluded_keys` was still decorated because the direct-access decoration loop never consulted `excluded_keys`; exclusion now wins, matching the full-decoration path.
|
|
24
|
+
- [Fix] Strip the tests/specs root directory as an anchored prefix (`sub(/\A.../)`) instead of a global `gsub` in `MinitestLocator` and `RSpecLocator`. When the root directory string recurred later in a test/spec file path, the global replace removed every occurrence and corrupted the derived subject class path; only the leading prefix is now removed.
|
|
25
|
+
|
|
3
26
|
## 2.6.1 (2026-06-15)
|
|
4
|
-
- [Enhancement] Speed up `Contract#call` by ~1.
|
|
27
|
+
- [Enhancement] Speed up `Contract#call` by ~1.25–1.4x, the per-message validation path in WaterDrop producers.
|
|
5
28
|
- [Fix] `Contract#call` with rule paths of 3+ keys no longer raises `NoMethodError` when an intermediate value is not a `Hash` and reports the path as missing instead, consistent with the 2-key path behavior.
|
|
6
29
|
- [Change] Reject reserved setting names with an `ArgumentError` in `Configurable::Node#setting` and `#register`: internal state names (`node_name`, `children`, `nestings`, `compiled`, `configs_refs`, `local_defs`) and the node public API names (`setting`, `configure`, `to_h`, `deep_dup`, `register`, `compile`). Previously such names silently shadowed the node own accessors, breaking `deep_dup` or `to_h`, and assignments like `config.children = value` corrupted the node internal state.
|
|
7
|
-
- [Enhancement]
|
|
8
|
-
- [Enhancement]
|
|
30
|
+
- [Enhancement] Speed up `Monitor#instrument` dispatch (~1.2x) on the common no-subscribers path.
|
|
31
|
+
- [Enhancement] Speed up `Configurable::Node` settings reads (~1.4x flat, ~1.6x nested) on hot paths.
|
|
9
32
|
- [Enhancement] Instantiate each `Configurable::Node` through a per-layout anonymous subclass so the ivar-backed settings do not grow object shape variations on the shared `Node` class (which would degrade ivar access and trigger Ruby performance warnings). `deep_dup` reuses the template's subclass, so duplicated configs share object shapes.
|
|
10
33
|
- [Fix] Symbolize setting names at definition time (`setting`, same as `register`) and on config store writes so `String` setting names work end to end (accessors, `#to_h`, recompilation state) and cannot corrupt node internal state when matching reserved internal names (previously string-named settings were quietly broken as accessors and the store disagreed on the key type).
|
|
11
34
|
- [Change] Config nodes are now instances of anonymous `Node` subclasses: `is_a?(Karafka::Core::Configurable::Node)` still holds, but `instance_of?(Node)` is now `false` and `node.class.name` is `nil`.
|
|
@@ -33,6 +56,7 @@
|
|
|
33
56
|
- [Enhancement] Replace `StatisticsDecorator#diff` pending-writes buffer with `keys.each` direct-write iteration, eliminating the buffer and write-back loop for ~13% faster decoration at scale (10 brokers, 20 topics, 2000 partitions).
|
|
34
57
|
- [Enhancement] Reorder `StatisticsDecorator#diff` type checks to test `Numeric` before `Hash`, matching the ~80% numeric value distribution in librdkafka statistics.
|
|
35
58
|
- [Enhancement] Support `only_keys` option in `StatisticsDecorator` to decorate only specified numeric keys (e.g. `consumer_lag`, `committed_offset`). When combined with `excluded_keys`, reduces decoration cost from ~80ms to ~8.5ms per call on large clusters (10 brokers, 20 topics, 2000 partitions) by using structure-aware navigation of the librdkafka statistics tree and direct key access instead of full-hash iteration.
|
|
59
|
+
- [Enhancement] Cache `Tags#to_a` values array and invalidate on `add`/`delete`/`clear` to avoid allocating a new Array and running `uniq` on every call, yielding ~7x faster reads at 5 tags and ~28x faster at 20 tags.
|
|
36
60
|
|
|
37
61
|
## 2.5.10 (2026-03-02)
|
|
38
62
|
- [Enhancement] Introduce `MinitestLocator` helper for minitest/spec subject class auto-discovery from test file paths.
|
data/Gemfile.lint.lock
CHANGED
|
@@ -55,11 +55,11 @@ GEM
|
|
|
55
55
|
unicode-display_width (3.2.0)
|
|
56
56
|
unicode-emoji (~> 4.1)
|
|
57
57
|
unicode-emoji (4.2.0)
|
|
58
|
-
yard (0.9.
|
|
59
|
-
yard-lint (1.
|
|
58
|
+
yard (0.9.45)
|
|
59
|
+
yard-lint (1.10.1)
|
|
60
60
|
yard (~> 0.9)
|
|
61
61
|
zeitwerk (~> 2.6)
|
|
62
|
-
zeitwerk (2.
|
|
62
|
+
zeitwerk (2.8.2)
|
|
63
63
|
|
|
64
64
|
PLATFORMS
|
|
65
65
|
ruby
|
|
@@ -75,6 +75,7 @@ DEPENDENCIES
|
|
|
75
75
|
|
|
76
76
|
CHECKSUMS
|
|
77
77
|
ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
|
|
78
|
+
bundler (4.0.17) sha256=214e21431b5665dd2f99df8a5511c6b151d7a72e8015c8b38f8b775b61cbb6c1
|
|
78
79
|
json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
|
|
79
80
|
language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
|
|
80
81
|
lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
|
|
@@ -95,9 +96,9 @@ CHECKSUMS
|
|
|
95
96
|
standard-performance (1.9.0) sha256=49483d31be448292951d80e5e67cdcb576c2502103c7b40aec6f1b6e9c88e3f2
|
|
96
97
|
unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
|
|
97
98
|
unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f
|
|
98
|
-
yard (0.9.
|
|
99
|
-
yard-lint (1.
|
|
100
|
-
zeitwerk (2.
|
|
99
|
+
yard (0.9.45) sha256=52e211493f7cb8a3ebf7e104a25a1e73937a3103092545d34cb88fafebb3dc51
|
|
100
|
+
yard-lint (1.10.1) sha256=172ea1905304580bd515c093d82831ec4889e725100c49d5bf77e62101f49129
|
|
101
|
+
zeitwerk (2.8.2) sha256=7212a61311083c604184b1ea2574b9aa05cd14f855a0841c06985cabe9181d12
|
|
101
102
|
|
|
102
103
|
BUNDLED WITH
|
|
103
|
-
4.0.
|
|
104
|
+
4.0.17
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
karafka-core (2.6.
|
|
4
|
+
karafka-core (2.6.3)
|
|
5
5
|
karafka-rdkafka (>= 0.20.0)
|
|
6
6
|
logger (>= 1.6.0)
|
|
7
7
|
|
|
@@ -10,7 +10,6 @@ GEM
|
|
|
10
10
|
specs:
|
|
11
11
|
byebug (13.0.0)
|
|
12
12
|
reline (>= 0.6.0)
|
|
13
|
-
docile (1.4.1)
|
|
14
13
|
drb (2.2.3)
|
|
15
14
|
ffi (1.17.4)
|
|
16
15
|
ffi (1.17.4-aarch64-linux-gnu)
|
|
@@ -24,38 +23,38 @@ GEM
|
|
|
24
23
|
ffi (1.17.4-x86_64-linux-gnu)
|
|
25
24
|
ffi (1.17.4-x86_64-linux-musl)
|
|
26
25
|
io-console (0.8.2)
|
|
27
|
-
json (2.
|
|
28
|
-
karafka-rdkafka (0.
|
|
26
|
+
json (2.21.2)
|
|
27
|
+
karafka-rdkafka (0.28.0)
|
|
29
28
|
ffi (~> 1.17.1)
|
|
30
29
|
json (> 2.0)
|
|
31
30
|
logger
|
|
32
31
|
mini_portile2 (~> 2.6)
|
|
33
32
|
rake (> 12)
|
|
34
|
-
karafka-rdkafka (0.
|
|
33
|
+
karafka-rdkafka (0.28.0-aarch64-linux-gnu)
|
|
35
34
|
ffi (~> 1.17.1)
|
|
36
35
|
json (> 2.0)
|
|
37
36
|
logger
|
|
38
37
|
mini_portile2 (~> 2.6)
|
|
39
38
|
rake (> 12)
|
|
40
|
-
karafka-rdkafka (0.
|
|
39
|
+
karafka-rdkafka (0.28.0-aarch64-linux-musl)
|
|
41
40
|
ffi (~> 1.17.1)
|
|
42
41
|
json (> 2.0)
|
|
43
42
|
logger
|
|
44
43
|
mini_portile2 (~> 2.6)
|
|
45
44
|
rake (> 12)
|
|
46
|
-
karafka-rdkafka (0.
|
|
45
|
+
karafka-rdkafka (0.28.0-arm64-darwin)
|
|
47
46
|
ffi (~> 1.17.1)
|
|
48
47
|
json (> 2.0)
|
|
49
48
|
logger
|
|
50
49
|
mini_portile2 (~> 2.6)
|
|
51
50
|
rake (> 12)
|
|
52
|
-
karafka-rdkafka (0.
|
|
51
|
+
karafka-rdkafka (0.28.0-x86_64-linux-gnu)
|
|
53
52
|
ffi (~> 1.17.1)
|
|
54
53
|
json (> 2.0)
|
|
55
54
|
logger
|
|
56
55
|
mini_portile2 (~> 2.6)
|
|
57
56
|
rake (> 12)
|
|
58
|
-
karafka-rdkafka (0.
|
|
57
|
+
karafka-rdkafka (0.28.0-x86_64-linux-musl)
|
|
59
58
|
ffi (~> 1.17.1)
|
|
60
59
|
json (> 2.0)
|
|
61
60
|
logger
|
|
@@ -70,12 +69,7 @@ GEM
|
|
|
70
69
|
rake (13.4.2)
|
|
71
70
|
reline (0.6.3)
|
|
72
71
|
io-console (~> 0.5)
|
|
73
|
-
simplecov (
|
|
74
|
-
docile (~> 1.1)
|
|
75
|
-
simplecov-html (~> 0.11)
|
|
76
|
-
simplecov_json_formatter (~> 0.1)
|
|
77
|
-
simplecov-html (0.13.2)
|
|
78
|
-
simplecov_json_formatter (0.1.4)
|
|
72
|
+
simplecov (1.1.1)
|
|
79
73
|
warning (1.6.0)
|
|
80
74
|
|
|
81
75
|
PLATFORMS
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Karafka
|
|
4
|
+
module Core
|
|
5
|
+
module Configurable
|
|
6
|
+
# Base class for config defaults injectors.
|
|
7
|
+
#
|
|
8
|
+
# An injector enriches a config-like hash with a set of default values, applying each
|
|
9
|
+
# default only when the corresponding key is not already present. This lets a component
|
|
10
|
+
# ship sane defaults while still letting users override any of them by pre-populating the
|
|
11
|
+
# key themselves.
|
|
12
|
+
#
|
|
13
|
+
# Injectors are meant to be layered. A base (e.g. OSS) injector defines its {defaults} and
|
|
14
|
+
# an extension (e.g. Pro) prepends a module onto the singleton class and calls `super` to
|
|
15
|
+
# contribute additional defaults on top:
|
|
16
|
+
#
|
|
17
|
+
# Base.singleton_class.prepend(Extension)
|
|
18
|
+
#
|
|
19
|
+
# The only-if-absent rule applies to the target being enriched, not to the defaults
|
|
20
|
+
# themselves: a key the user already set in the target is never touched. How two layers
|
|
21
|
+
# resolve a key they both define is up to the layers -- e.g. an extension using
|
|
22
|
+
# `super.merge(extra)` lets its own value win, while `extra.merge(super)` would keep the
|
|
23
|
+
# base value.
|
|
24
|
+
#
|
|
25
|
+
# @example Define an injector with defaults
|
|
26
|
+
# class MyInjector < Karafka::Core::Configurable::Injector
|
|
27
|
+
# DEFAULTS = { 'a' => 1, 'b' => 2 }.freeze
|
|
28
|
+
#
|
|
29
|
+
# class << self
|
|
30
|
+
# def defaults
|
|
31
|
+
# DEFAULTS
|
|
32
|
+
# end
|
|
33
|
+
# end
|
|
34
|
+
# end
|
|
35
|
+
#
|
|
36
|
+
# MyInjector.call({ 'b' => 20 }) #=> { 'b' => 20, 'a' => 1 }
|
|
37
|
+
#
|
|
38
|
+
# @example Layer extra defaults (e.g. Pro) via prepend + super
|
|
39
|
+
# module ProDefaults
|
|
40
|
+
# def defaults
|
|
41
|
+
# super.merge('c' => 3)
|
|
42
|
+
# end
|
|
43
|
+
# end
|
|
44
|
+
#
|
|
45
|
+
# MyInjector.singleton_class.prepend(ProDefaults)
|
|
46
|
+
# MyInjector.call({}) #=> { 'a' => 1, 'b' => 2, 'c' => 3 }
|
|
47
|
+
#
|
|
48
|
+
# @note The base class holds no state and defines no defaults on its own; it is not useful
|
|
49
|
+
# directly and is expected to be subclassed. Subclasses should return the same defaults
|
|
50
|
+
# object (e.g. a frozen constant) on each call so that a prepended layer doing
|
|
51
|
+
# `super.merge(...)` never mutates it.
|
|
52
|
+
#
|
|
53
|
+
# @note Default values are injected by reference, not copied. A mutable default (an array or
|
|
54
|
+
# hash) is therefore shared across every target it is injected into, and mutating it in one
|
|
55
|
+
# place is visible everywhere. When a per-target mutable value is needed, the defaults
|
|
56
|
+
# layer should hand out a copy (e.g. `dup` it in `.defaults`).
|
|
57
|
+
class Injector
|
|
58
|
+
# Empty, immutable defaults used by the base class, which defines none of its own.
|
|
59
|
+
EMPTY_DEFAULTS = {}.freeze
|
|
60
|
+
|
|
61
|
+
private_constant :EMPTY_DEFAULTS
|
|
62
|
+
|
|
63
|
+
class << self
|
|
64
|
+
# Enriches the target with the defaults, without overwriting any key that is already
|
|
65
|
+
# present in it. The target is mutated in place.
|
|
66
|
+
#
|
|
67
|
+
# @param target [Hash] config hash to enrich in place
|
|
68
|
+
# @return [Hash] the same target, enriched with the missing defaults
|
|
69
|
+
def call(target)
|
|
70
|
+
defaults.each do |key, value|
|
|
71
|
+
next if target.key?(key)
|
|
72
|
+
|
|
73
|
+
target[key] = value
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
target
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# @return [Hash] default values to inject. Override in subclasses; extensions may
|
|
80
|
+
# prepend a module onto the singleton class and call `super` to contribute additional
|
|
81
|
+
# defaults on top. The base class defines none.
|
|
82
|
+
def defaults
|
|
83
|
+
EMPTY_DEFAULTS
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -3,8 +3,7 @@
|
|
|
3
3
|
module Karafka
|
|
4
4
|
module Core
|
|
5
5
|
module Configurable
|
|
6
|
-
# Single non-leaf node
|
|
7
|
-
# This is a core component for the configurable settings
|
|
6
|
+
# Single non-leaf node. This is a core component for the configurable settings
|
|
8
7
|
#
|
|
9
8
|
# The idea here is simple: we collect settings (leafs) and children (nodes) information and
|
|
10
9
|
# we only compile/initialize the values prior to user running the `#configure` API. This API
|
|
@@ -116,7 +115,10 @@ module Karafka
|
|
|
116
115
|
result = if @configs_refs.key?(value.node_name)
|
|
117
116
|
@configs_refs[value.node_name]
|
|
118
117
|
elsif value.constructor
|
|
119
|
-
|
|
118
|
+
# Use the arity-aware helper (same as `#compile`) so a `->(default) { ... }`
|
|
119
|
+
# constructor receives its default instead of being called with no arguments,
|
|
120
|
+
# which would raise `ArgumentError: wrong number of arguments`.
|
|
121
|
+
call_constructor(value)
|
|
120
122
|
elsif value.default
|
|
121
123
|
value.default
|
|
122
124
|
end
|
|
@@ -144,6 +146,17 @@ module Karafka
|
|
|
144
146
|
# After inheritance we need to reload the state so the leafs are recompiled again
|
|
145
147
|
value = value.dup
|
|
146
148
|
value.compiled = false
|
|
149
|
+
|
|
150
|
+
# `Struct#dup` is intentionally shallow here: the leaf's `default` value is shared by
|
|
151
|
+
# reference across the class template and every config instance produced by
|
|
152
|
+
# `deep_dup`. This is the contract -- one uniform rule for all default types -- and it
|
|
153
|
+
# is what lets a shared service object passed as a default (e.g. a logger) keep its
|
|
154
|
+
# identity across all configs instead of being cloned per instance. The flip side is
|
|
155
|
+
# that an in-place mutation of a mutable container default (e.g. `config.list << :x`)
|
|
156
|
+
# is visible on every other instance and on the template. A caller that needs a
|
|
157
|
+
# per-instance mutable default should not rely on a mutable `default:` (e.g.
|
|
158
|
+
# `default: []`): assign the value inside a `configure` block, or dup it themselves,
|
|
159
|
+
# so each instance owns its own copy.
|
|
147
160
|
value
|
|
148
161
|
else
|
|
149
162
|
value.deep_dup
|
|
@@ -175,7 +188,12 @@ module Karafka
|
|
|
175
188
|
|
|
176
189
|
prevent_reserved_names!(name)
|
|
177
190
|
|
|
178
|
-
|
|
191
|
+
# Check the defined children, not just @configs_refs: a lazy setting with a constructor
|
|
192
|
+
# is not written to @configs_refs until first read, so a `@configs_refs.key?` guard
|
|
193
|
+
# alone would silently overwrite it instead of raising the documented error.
|
|
194
|
+
if @children.any? { |child| child.node_name == name }
|
|
195
|
+
raise ArgumentError, "#{name} is already registered"
|
|
196
|
+
end
|
|
179
197
|
|
|
180
198
|
leaf = Leaf.new(name, value, nil, true, false)
|
|
181
199
|
@children << leaf
|
|
@@ -190,7 +208,12 @@ module Karafka
|
|
|
190
208
|
# Do not redefine something that was already set during compilation
|
|
191
209
|
# This will allow us to reconfigure things and skip override with defaults
|
|
192
210
|
skippable = @configs_refs.key?(value.node_name) || (value.is_a?(Leaf) && value.compiled?)
|
|
193
|
-
|
|
211
|
+
# A leaf is only treated as lazy (a dynamically (re)evaluated accessor) when it
|
|
212
|
+
# actually has a constructor to evaluate. `lazy: true` without a constructor has
|
|
213
|
+
# nothing to evaluate, so it behaves like a regular setting backed by its default.
|
|
214
|
+
# Otherwise the lazy path builds a dynamic accessor that calls `constructor.arity`
|
|
215
|
+
# on a `nil` constructor and crashes on first read.
|
|
216
|
+
lazy_leaf = value.is_a?(Leaf) && value.lazy? && !value.constructor.nil?
|
|
194
217
|
|
|
195
218
|
# Do not create accessor for leafs that are lazy as they will get a custom method
|
|
196
219
|
# created instead
|
|
@@ -295,7 +318,8 @@ module Karafka
|
|
|
295
318
|
ivar_name = :"@#{reader_name}"
|
|
296
319
|
|
|
297
320
|
define_singleton_method(:"#{reader_name}=") do |new_value|
|
|
298
|
-
instance_variable_set(ivar_name,
|
|
321
|
+
instance_variable_set(ivar_name, new_value)
|
|
322
|
+
@configs_refs[reader_name] = new_value
|
|
299
323
|
end
|
|
300
324
|
else
|
|
301
325
|
define_singleton_method(:"#{reader_name}=") do |new_value|
|
|
@@ -40,8 +40,14 @@ module Karafka
|
|
|
40
40
|
def nested(path, &)
|
|
41
41
|
init_accu
|
|
42
42
|
@nested << path
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
begin
|
|
44
|
+
instance_eval(&)
|
|
45
|
+
ensure
|
|
46
|
+
# Always pop, even if the block raises. Otherwise a rescued exception during
|
|
47
|
+
# contract definition would leave `path` on @nested and prefix it onto every rule
|
|
48
|
+
# defined afterwards.
|
|
49
|
+
@nested.pop
|
|
50
|
+
end
|
|
45
51
|
end
|
|
46
52
|
|
|
47
53
|
# Defines a rule for a required field (required means, that will automatically create an
|
|
@@ -61,10 +67,19 @@ module Karafka
|
|
|
61
67
|
@rules << Rule.new(@nested + keys, :optional, block).freeze
|
|
62
68
|
end
|
|
63
69
|
|
|
64
|
-
#
|
|
70
|
+
# Defines a virtual rule that validates the whole data rather than a single key. Unlike
|
|
71
|
+
# `required`/`optional`, the block receives the full data and returns its own errors.
|
|
72
|
+
#
|
|
73
|
+
# @param block [Proc] validation rule, called with `(data, errors, contract)`. It must
|
|
74
|
+
# return either a non-Array (`true`/`nil`/`false`) for "no errors", or an `Array` of
|
|
75
|
+
# `[path, message]` error pairs (where `path` is itself an `Array` of symbols).
|
|
65
76
|
#
|
|
66
|
-
# @note
|
|
67
|
-
#
|
|
77
|
+
# @note The returned error `Array` is owned by the contract: `#call` prepends the current
|
|
78
|
+
# scope onto each pair in place and collects them, so a rule must return a freshly
|
|
79
|
+
# built `Array` on every call. Returning a memoized, shared or frozen `Array` is not
|
|
80
|
+
# supported -- in-place scoping would accumulate the prefix across validations (or
|
|
81
|
+
# raise `FrozenError`). Build the result in the block (e.g. `[[%i[id], :invalid]]`)
|
|
82
|
+
# rather than returning a constant.
|
|
68
83
|
def virtual(&block)
|
|
69
84
|
init_accu
|
|
70
85
|
@rules << Rule.new([], :virtual, block).freeze
|
|
@@ -94,19 +109,32 @@ module Karafka
|
|
|
94
109
|
def call(data, scope: EMPTY_ARRAY)
|
|
95
110
|
errors = []
|
|
96
111
|
|
|
112
|
+
# A non-Hash root has no keys to dig; resolve it once here so #dig stays on its lean
|
|
113
|
+
# fast path and is only invoked with a Hash. Non-Hash data makes every required rule
|
|
114
|
+
# report missing, consistent with the non-Hash intermediate handling inside #dig.
|
|
115
|
+
data_is_hash = data.is_a?(Hash)
|
|
116
|
+
|
|
97
117
|
self.class.rules.each do |rule|
|
|
98
118
|
if rule.type == :virtual
|
|
99
119
|
result = rule.validator.call(data, errors, self)
|
|
100
120
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
121
|
+
# A virtual rule signals "no errors" with any non-Array result (true, but also a
|
|
122
|
+
# falsy `nil`/`false` returned by e.g. `condition && [[...], :err]`). Only an Array
|
|
123
|
+
# of error pairs is iterated; previously a `false` return reached `false.each` and
|
|
124
|
+
# raised NoMethodError (a `nil` return was already tolerated by the safe navigation).
|
|
125
|
+
next unless result.is_a?(Array)
|
|
126
|
+
|
|
127
|
+
# Apply the scope prefix in place on the rule's returned pairs and collect them
|
|
128
|
+
# directly. Per the `virtual` contract the rule hands back a freshly built Array
|
|
129
|
+
# each call (see `DSL#virtual`), so mutating it here is safe and avoids allocating a
|
|
130
|
+
# new pair per error.
|
|
131
|
+
result.each do |sub_result|
|
|
104
132
|
sub_result[0] = scope + sub_result[0]
|
|
105
133
|
end
|
|
106
134
|
|
|
107
135
|
errors.push(*result)
|
|
108
136
|
else
|
|
109
|
-
for_checking = dig(data, rule.path)
|
|
137
|
+
for_checking = data_is_hash ? dig(data, rule.path) : DIG_MISS
|
|
110
138
|
|
|
111
139
|
if DIG_MISS.equal?(for_checking)
|
|
112
140
|
errors << [scope + rule.path, :missing] if rule.type == :required
|
|
@@ -24,7 +24,7 @@ module Karafka
|
|
|
24
24
|
this = self
|
|
25
25
|
# Allows "auto subject" definitions for the `describe` method, as it will figure
|
|
26
26
|
# out the proper class that we want to describe
|
|
27
|
-
#
|
|
27
|
+
# block - block with tests
|
|
28
28
|
minitest_module.define_singleton_method :describe_current do |&block|
|
|
29
29
|
describe(this.inherited, &block)
|
|
30
30
|
end
|
|
@@ -36,7 +36,7 @@ module Karafka
|
|
|
36
36
|
.first
|
|
37
37
|
.split(":")
|
|
38
38
|
.first
|
|
39
|
-
.
|
|
39
|
+
.sub(/\A#{::Regexp.escape(@tests_root_dir)}/, "")
|
|
40
40
|
.gsub("_test.rb", "")
|
|
41
41
|
.split("/")
|
|
42
42
|
.delete_if(&:empty?)
|
|
@@ -27,7 +27,7 @@ module Karafka
|
|
|
27
27
|
this = self
|
|
28
28
|
# Allows "auto subject" definitions for the `#describe` method, as it will figure
|
|
29
29
|
# out the proper class that we want to describe
|
|
30
|
-
#
|
|
30
|
+
# block - block with specs
|
|
31
31
|
rspec.define_singleton_method :describe_current do |&block|
|
|
32
32
|
describe(this.inherited, &block)
|
|
33
33
|
end
|
|
@@ -39,7 +39,7 @@ module Karafka
|
|
|
39
39
|
.first
|
|
40
40
|
.split(":")
|
|
41
41
|
.first
|
|
42
|
-
.
|
|
42
|
+
.sub(/\A#{::Regexp.escape(@specs_root_dir)}/, "")
|
|
43
43
|
.gsub("_spec.rb", "")
|
|
44
44
|
.split("/")
|
|
45
45
|
.delete_if(&:empty?)
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
module Karafka
|
|
4
4
|
module Core
|
|
5
|
-
# Karafka instrumentation related shared components
|
|
6
5
|
module Instrumentation
|
|
7
6
|
# This manager allows us to register multiple callbacks into a hook that is suppose to support
|
|
8
7
|
# a single callback
|
|
@@ -10,20 +9,21 @@ module Karafka
|
|
|
10
9
|
# @return [::Karafka::Core::Instrumentation::CallbacksManager]
|
|
11
10
|
def initialize
|
|
12
11
|
@callbacks = {}
|
|
13
|
-
@
|
|
12
|
+
@values = [].freeze
|
|
13
|
+
@mutex = Mutex.new
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
# Invokes all the callbacks registered one after another
|
|
17
17
|
#
|
|
18
18
|
# @param args [Object] any args that should go to the callbacks
|
|
19
|
-
# @note
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
19
|
+
# @note Copy-on-write: dispatch iterates an immutable snapshot that `add`/`delete`
|
|
20
|
+
# rebuild and swap in under a mutex. Because `#call` never mutates shared state, it
|
|
21
|
+
# needs neither a lock nor a per-call `#values` allocation, and a callback registered
|
|
22
|
+
# or removed from another thread is never lost; it just takes effect on the next
|
|
23
|
+
# `#call`. A cache invalidated from within `#call` could not be updated atomically
|
|
24
|
+
# against this read, so a stale write-back would permanently drop callbacks.
|
|
25
25
|
def call(*args)
|
|
26
|
-
|
|
26
|
+
@values.each { |callback| callback.call(*args) }
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
# Adds a callback to the manager
|
|
@@ -31,15 +31,19 @@ module Karafka
|
|
|
31
31
|
# @param id [String] id of the callback (used when deleting it)
|
|
32
32
|
# @param callable [#call] object that responds to a `#call` method
|
|
33
33
|
def add(id, callable)
|
|
34
|
-
@
|
|
35
|
-
|
|
34
|
+
@mutex.synchronize do
|
|
35
|
+
@callbacks[id] = callable
|
|
36
|
+
@values = @callbacks.values.freeze
|
|
37
|
+
end
|
|
36
38
|
end
|
|
37
39
|
|
|
38
40
|
# Removes the callback from the manager
|
|
39
41
|
# @param id [String] id of the callback we want to remove
|
|
40
42
|
def delete(id)
|
|
41
|
-
@
|
|
42
|
-
|
|
43
|
+
@mutex.synchronize do
|
|
44
|
+
@callbacks.delete(id)
|
|
45
|
+
@values = @callbacks.values.freeze
|
|
46
|
+
end
|
|
43
47
|
end
|
|
44
48
|
end
|
|
45
49
|
end
|
|
@@ -48,8 +48,17 @@ module Karafka
|
|
|
48
48
|
# @param event_id [String] the key of the event to clear listeners for.
|
|
49
49
|
def clear(event_id = nil)
|
|
50
50
|
@mutex.synchronize do
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
# Copy-on-write: replace the per-event arrays rather than mutating them in place, so a
|
|
52
|
+
# dispatch iterating a previously captured array is unaffected (see #notify_listeners).
|
|
53
|
+
unless event_id
|
|
54
|
+
@listeners.transform_values! { [] }
|
|
55
|
+
return
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
if @listeners.key?(event_id)
|
|
59
|
+
@listeners[event_id] = []
|
|
60
|
+
return
|
|
61
|
+
end
|
|
53
62
|
|
|
54
63
|
raise(EventNotRegistered, "#{event_id} not registered!")
|
|
55
64
|
end
|
|
@@ -76,21 +85,23 @@ module Karafka
|
|
|
76
85
|
|
|
77
86
|
raise EventNotRegistered, event_id unless @listeners.key?(event_id)
|
|
78
87
|
|
|
79
|
-
|
|
88
|
+
# Copy-on-write: append by replacing the array, never mutating the one a concurrent
|
|
89
|
+
# dispatch may be iterating (see #notify_listeners).
|
|
90
|
+
@listeners[event_id] += [block]
|
|
80
91
|
else
|
|
81
92
|
listener = event_id_or_listener
|
|
82
93
|
|
|
83
94
|
@listeners.each_key do |reg_event_id|
|
|
84
95
|
next unless listener.respond_to?(@events_methods_map[reg_event_id])
|
|
85
96
|
|
|
86
|
-
@listeners[reg_event_id]
|
|
97
|
+
@listeners[reg_event_id] += [listener]
|
|
87
98
|
end
|
|
88
99
|
end
|
|
89
100
|
end
|
|
90
101
|
end
|
|
91
102
|
|
|
92
|
-
# Allows for unsubscription from events
|
|
93
|
-
#
|
|
103
|
+
# Allows for unsubscription from events. This method will remove the listener/block from all
|
|
104
|
+
# events where it's currently subscribed.
|
|
94
105
|
#
|
|
95
106
|
# @param listener_or_block [Object] listener object or block to remove from all events
|
|
96
107
|
#
|
|
@@ -98,9 +109,9 @@ module Karafka
|
|
|
98
109
|
# unsubscribe(my_listener)
|
|
99
110
|
def unsubscribe(listener_or_block)
|
|
100
111
|
@mutex.synchronize do
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
112
|
+
# Copy-on-write: rebuild each array without the listener instead of deleting in place,
|
|
113
|
+
# so a dispatch iterating a previously captured array still sees the full set.
|
|
114
|
+
@listeners.transform_values! { |event_listeners| event_listeners - [listener_or_block] }
|
|
104
115
|
end
|
|
105
116
|
end
|
|
106
117
|
|
|
@@ -14,7 +14,16 @@ module Karafka
|
|
|
14
14
|
# - KEY_fd - freeze duration - describes how long the delta remains unchanged (zero)
|
|
15
15
|
# and can be useful for detecting values that "hang" for extended period of time
|
|
16
16
|
# and do not have any change (delta always zero). This value is in ms for the
|
|
17
|
-
# consistency with other time operators we use.
|
|
17
|
+
# consistency with other time operators we use. A newly introduced key (one
|
|
18
|
+
# that had no value in the previous emission) starts at a freeze duration of
|
|
19
|
+
# zero, since there is no prior value it could have been "frozen" against.
|
|
20
|
+
#
|
|
21
|
+
# The `_d` and `_fd` suffixes are reserved. For every numeric KEY the decorator writes
|
|
22
|
+
# `KEY_d` and `KEY_fd` into the same hash, so if the input already contains a real key
|
|
23
|
+
# literally named `KEY_d` or `KEY_fd` (for another numeric KEY present in that hash) it is
|
|
24
|
+
# overwritten by the computed delta/freeze duration. librdkafka statistics never use these
|
|
25
|
+
# suffixes, so this does not happen with real stats; it only matters if you feed custom data
|
|
26
|
+
# through the decorator.
|
|
18
27
|
class StatisticsDecorator
|
|
19
28
|
include Helpers::Time
|
|
20
29
|
|
|
@@ -28,11 +37,17 @@ module Karafka
|
|
|
28
37
|
# duration suffixes. This is useful for skipping large subtrees of the librdkafka
|
|
29
38
|
# statistics that are not consumed by the application (e.g. broker toppars, window
|
|
30
39
|
# stats like int_latency, outbuf_latency, throttle, batchsize, batchcnt, req).
|
|
31
|
-
# @param only_keys [Array<String>] when set, only these numeric keys
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
40
|
+
# @param only_keys [Array<String>] when set, only these numeric keys are decorated with
|
|
41
|
+
# delta/freeze duration suffixes, and only at the levels of the known librdkafka
|
|
42
|
+
# statistics tree: the root, each broker, each topic, each partition and cgrp. The
|
|
43
|
+
# decorator navigates that known structure directly and decorates the listed keys it
|
|
44
|
+
# finds at each of those levels. It deliberately does NOT descend into nested
|
|
45
|
+
# sub-objects within a broker, topic, partition or cgrp (e.g. broker window stats like
|
|
46
|
+
# rtt/throttle/int_latency, or the toppars map) -- skipping that descent is exactly what
|
|
47
|
+
# makes this mode cheap on large clusters. Non-librdkafka hash children found at the
|
|
48
|
+
# root are still fully recursed for correctness. If you need a key nested inside one of
|
|
49
|
+
# those sub-objects decorated, use the default full-decoration mode. When empty
|
|
50
|
+
# (default), all numeric keys at every depth are decorated.
|
|
36
51
|
def initialize(excluded_keys: [], only_keys: [])
|
|
37
52
|
@previous = EMPTY_HASH
|
|
38
53
|
# Operate on ms precision only
|
|
@@ -44,9 +59,12 @@ module Karafka
|
|
|
44
59
|
@excluded_keys = unless excluded_keys.empty?
|
|
45
60
|
excluded_keys.each_with_object({}) { |k, h| h[k] = true }.freeze
|
|
46
61
|
end
|
|
47
|
-
# Frozen array for direct-access decoration, nil when empty to use full decoration
|
|
62
|
+
# Frozen array for direct-access decoration, nil when empty to use full decoration.
|
|
63
|
+
# Exclusions are applied once here (excluded_keys wins over only_keys), so the hot
|
|
64
|
+
# decoration loop iterates an already-filtered list and never re-checks exclusions.
|
|
48
65
|
@only_keys = unless only_keys.empty?
|
|
49
|
-
only_keys.
|
|
66
|
+
effective = @excluded_keys ? only_keys.reject { |k| @excluded_keys.key?(k) } : only_keys
|
|
67
|
+
effective.freeze
|
|
50
68
|
end
|
|
51
69
|
end
|
|
52
70
|
|
|
@@ -120,18 +138,17 @@ module Karafka
|
|
|
120
138
|
if value.is_a?(Numeric)
|
|
121
139
|
prev_value = filled_previous[key]
|
|
122
140
|
|
|
141
|
+
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
142
|
+
|
|
123
143
|
if prev_value.nil?
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
result = value - prev_value
|
|
144
|
+
current[pair[0]] = 0
|
|
145
|
+
current[pair[1]] = 0
|
|
127
146
|
else
|
|
128
|
-
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
147
|
+
result = value - prev_value
|
|
132
148
|
|
|
133
|
-
|
|
134
|
-
|
|
149
|
+
current[pair[0]] = (result == 0) ? (filled_previous[pair[0]] || 0) + change_d : 0
|
|
150
|
+
current[pair[1]] = result
|
|
151
|
+
end
|
|
135
152
|
elsif value.is_a?(Hash)
|
|
136
153
|
diff_all(filled_previous[key], value, change_d)
|
|
137
154
|
end
|
|
@@ -156,8 +173,13 @@ module Karafka
|
|
|
156
173
|
# librdkafka statistics layout: root → brokers → broker, root → topics → topic →
|
|
157
174
|
# partitions → partition, root → cgrp.
|
|
158
175
|
#
|
|
159
|
-
#
|
|
160
|
-
#
|
|
176
|
+
# Broker, topic, partition and cgrp nodes are decorated as leaves: their listed keys are
|
|
177
|
+
# decorated, but their nested sub-objects (e.g. broker window stats like rtt/throttle, or
|
|
178
|
+
# the toppars map) are NOT descended into. Not descending into those large sub-objects is
|
|
179
|
+
# the whole point of this path, so they are intentionally left untouched here.
|
|
180
|
+
#
|
|
181
|
+
# For non-librdkafka hash children at the root (e.g. custom or test data), falls back to
|
|
182
|
+
# generic recursion to maintain correctness with arbitrary nested structures.
|
|
161
183
|
#
|
|
162
184
|
# @param previous [Object] previous value from the given scope
|
|
163
185
|
# @param current [Hash] current stats hash (root level)
|
|
@@ -209,8 +231,10 @@ module Karafka
|
|
|
209
231
|
end
|
|
210
232
|
|
|
211
233
|
# Consumer group (leaf-like)
|
|
212
|
-
|
|
213
|
-
|
|
234
|
+
unless excluded&.key?("cgrp")
|
|
235
|
+
cgrp = current["cgrp"]
|
|
236
|
+
decorate_keys(cgrp, filled_previous["cgrp"] || EMPTY_HASH, change_d) if cgrp.is_a?(Hash)
|
|
237
|
+
end
|
|
214
238
|
|
|
215
239
|
# Fallback: handle any non-standard hash children not in the known structure.
|
|
216
240
|
# This ensures correctness for arbitrary nested data while the known paths above
|
|
@@ -275,18 +299,17 @@ module Karafka
|
|
|
275
299
|
|
|
276
300
|
prev_value = filled_previous[key]
|
|
277
301
|
|
|
302
|
+
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
303
|
+
|
|
278
304
|
if prev_value.nil?
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
result = value - prev_value
|
|
305
|
+
current[pair[0]] = 0
|
|
306
|
+
current[pair[1]] = 0
|
|
282
307
|
else
|
|
283
|
-
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
308
|
+
result = value - prev_value
|
|
287
309
|
|
|
288
|
-
|
|
289
|
-
|
|
310
|
+
current[pair[0]] = (result == 0) ? (filled_previous[pair[0]] || 0) + change_d : 0
|
|
311
|
+
current[pair[1]] = result
|
|
312
|
+
end
|
|
290
313
|
end
|
|
291
314
|
end
|
|
292
315
|
end
|
|
@@ -27,9 +27,28 @@ module Karafka
|
|
|
27
27
|
) do |client_ptr, err_code, reason, _opaque|
|
|
28
28
|
return nil unless ::Rdkafka::Config.error_callback
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
# Guard against a null client pointer. librdkafka can invoke the error callback
|
|
31
|
+
# with a NULL `rd_kafka_t` (e.g. very early in client construction), and calling
|
|
32
|
+
# `rd_kafka_name` on it dereferences the pointer and segfaults the whole process.
|
|
33
|
+
# Mirrors the upstream `ErrorCallback`.
|
|
34
|
+
name = client_ptr.null? ? nil : ::Rdkafka::Bindings.rd_kafka_name(client_ptr)
|
|
35
|
+
|
|
36
|
+
# Resolve fatal errors to their underlying cause. `ERR__FATAL` is only a generic
|
|
37
|
+
# marker; the real error code and description must be fetched from librdkafka via
|
|
38
|
+
# `rd_kafka_fatal_error` (done by `RdkafkaError.build_fatal`). Without this the
|
|
39
|
+
# callback would report the generic fatal code instead of the actual error.
|
|
40
|
+
# Mirrors the upstream `ErrorCallback`.
|
|
41
|
+
error = if err_code == ::Rdkafka::Bindings::RD_KAFKA_RESP_ERR__FATAL
|
|
42
|
+
::Rdkafka::RdkafkaError.build_fatal(
|
|
43
|
+
client_ptr,
|
|
44
|
+
fallback_error_code: err_code,
|
|
45
|
+
fallback_message: reason,
|
|
46
|
+
instance_name: name
|
|
47
|
+
)
|
|
48
|
+
else
|
|
49
|
+
::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
|
50
|
+
end
|
|
31
51
|
|
|
32
|
-
error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
|
33
52
|
error.set_backtrace(caller)
|
|
34
53
|
|
|
35
54
|
::Rdkafka::Config.error_callback.call(name, error)
|
|
@@ -10,6 +10,7 @@ module Karafka
|
|
|
10
10
|
# Creates new tags accumulator
|
|
11
11
|
def initialize
|
|
12
12
|
@accu = {}
|
|
13
|
+
@values_cache = nil
|
|
13
14
|
end
|
|
14
15
|
|
|
15
16
|
# Adds a tag with a given name to tags
|
|
@@ -17,22 +18,25 @@ module Karafka
|
|
|
17
18
|
# @param tag [#to_s] any object that can be converted into a string via `#to_s`
|
|
18
19
|
def add(name, tag)
|
|
19
20
|
@accu[name] = tag.to_s
|
|
21
|
+
@values_cache = nil
|
|
20
22
|
end
|
|
21
23
|
|
|
22
24
|
# Removes all the tags
|
|
23
25
|
def clear
|
|
24
26
|
@accu.clear
|
|
27
|
+
@values_cache = nil
|
|
25
28
|
end
|
|
26
29
|
|
|
27
30
|
# Removes a tag with a given name
|
|
28
31
|
# @param name [Symbol] name of the tag
|
|
29
32
|
def delete(name)
|
|
30
33
|
@accu.delete(name)
|
|
34
|
+
@values_cache = nil
|
|
31
35
|
end
|
|
32
36
|
|
|
33
37
|
# @return [Array<String>] all unique tags registered
|
|
34
38
|
def to_a
|
|
35
|
-
@accu.values.
|
|
39
|
+
@values_cache ||= @accu.values.uniq
|
|
36
40
|
end
|
|
37
41
|
|
|
38
42
|
# @param _args [Object] anything that the standard `to_json` accepts
|
data/lib/karafka/core/version.rb
CHANGED
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Karafka
|
|
4
4
|
module Core
|
|
5
|
-
# Current Karafka::Core version
|
|
6
|
-
|
|
7
|
-
VERSION = "2.6.1"
|
|
5
|
+
# Current Karafka::Core version. We follow the versioning schema of given Karafka version
|
|
6
|
+
VERSION = "2.6.3"
|
|
8
7
|
end
|
|
9
8
|
end
|
data/lib/karafka-core.rb
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "logger"
|
|
4
4
|
require "yaml"
|
|
5
|
+
require "pathname"
|
|
5
6
|
require "rdkafka"
|
|
6
7
|
require "karafka/core"
|
|
7
8
|
require "karafka/core/version"
|
|
@@ -14,6 +15,7 @@ require "karafka/core/monitoring/statistics_decorator"
|
|
|
14
15
|
require "karafka/core/configurable"
|
|
15
16
|
require "karafka/core/configurable/leaf"
|
|
16
17
|
require "karafka/core/configurable/node"
|
|
18
|
+
require "karafka/core/configurable/injector"
|
|
17
19
|
require "karafka/core/contractable/contract"
|
|
18
20
|
require "karafka/core/contractable/result"
|
|
19
21
|
require "karafka/core/contractable/rule"
|
|
@@ -35,12 +37,12 @@ rd_config = Rdkafka::Config
|
|
|
35
37
|
|
|
36
38
|
# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
|
|
37
39
|
# each callback type. Callback manager allows us to register more than one callback
|
|
38
|
-
#
|
|
40
|
+
# Those managers are also used by Karafka for consumer related statistics
|
|
39
41
|
rd_config.statistics_callback = instrumentation.statistics_callbacks
|
|
40
42
|
rd_config.error_callback = instrumentation.error_callbacks
|
|
41
43
|
rd_config.oauthbearer_token_refresh_callback = instrumentation.oauthbearer_token_refresh_callbacks
|
|
42
44
|
|
|
43
45
|
# This loads librdkafka components into memory prior to initializing the client.
|
|
44
46
|
# This mitigates macos forking issues.
|
|
45
|
-
#
|
|
47
|
+
# See https://github.com/confluentinc/librdkafka/issues/4590
|
|
46
48
|
Rdkafka::Bindings.rd_kafka_global_init if Rdkafka::Bindings.respond_to?(:rd_kafka_global_init)
|
data/package-lock.json
CHANGED
|
@@ -286,9 +286,9 @@
|
|
|
286
286
|
}
|
|
287
287
|
},
|
|
288
288
|
"node_modules/smol-toml": {
|
|
289
|
-
"version": "1.
|
|
290
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
291
|
-
"integrity": "sha512-
|
|
289
|
+
"version": "1.7.1",
|
|
290
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.1.tgz",
|
|
291
|
+
"integrity": "sha512-PPlsspAZ4jbMBu5DMFhfUGDQLu/vrL4SyBROVS37x8ynnVmFIs1VPBz1Co8Xks3TvpIaZXmU85y4DrQ+UyVFoQ==",
|
|
292
292
|
"dev": true,
|
|
293
293
|
"license": "BSD-3-Clause",
|
|
294
294
|
"engines": {
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: karafka-core
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.6.
|
|
4
|
+
version: 2.6.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Maciej Mensfeld
|
|
@@ -72,6 +72,7 @@ files:
|
|
|
72
72
|
- lib/karafka-core.rb
|
|
73
73
|
- lib/karafka/core.rb
|
|
74
74
|
- lib/karafka/core/configurable.rb
|
|
75
|
+
- lib/karafka/core/configurable/injector.rb
|
|
75
76
|
- lib/karafka/core/configurable/leaf.rb
|
|
76
77
|
- lib/karafka/core/configurable/node.rb
|
|
77
78
|
- lib/karafka/core/contractable.rb
|
|
@@ -120,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
120
121
|
- !ruby/object:Gem::Version
|
|
121
122
|
version: '0'
|
|
122
123
|
requirements: []
|
|
123
|
-
rubygems_version: 4.0.
|
|
124
|
+
rubygems_version: 4.0.16
|
|
124
125
|
specification_version: 4
|
|
125
126
|
summary: Karafka ecosystem core modules
|
|
126
127
|
test_files: []
|