karafka-core 2.6.0 → 2.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +7 -7
- data/.github/workflows/push.yml +3 -3
- data/.github/workflows/verify-action-pins.yml +1 -1
- data/CHANGELOG.md +32 -0
- data/Gemfile.lock +1 -1
- data/lib/karafka/core/configurable/node.rb +146 -13
- data/lib/karafka/core/contractable/contract.rb +77 -87
- data/lib/karafka/core/helpers/minitest_locator.rb +1 -1
- data/lib/karafka/core/helpers/rspec_locator.rb +1 -1
- data/lib/karafka/core/instrumentation/callbacks_manager.rb +17 -12
- data/lib/karafka/core/monitoring/monitor.rb +9 -1
- data/lib/karafka/core/monitoring/notifications.rb +18 -7
- data/lib/karafka/core/monitoring/statistics_decorator.rb +53 -30
- data/lib/karafka/core/patches/rdkafka/bindings.rb +21 -2
- data/lib/karafka/core/taggable/tags.rb +5 -1
- data/lib/karafka/core/version.rb +1 -1
- data/lib/karafka-core.rb +1 -0
- data/renovate.json +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b5f044e02de2bc5b5e41fb033c9e4f0a4e66f45890615053d3c1fe1c33ec8058
|
|
4
|
+
data.tar.gz: 5ad2604a98478db69950c2a94064dcd20d6ceafa6d3eab03d907700b6cadf3eb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0cd334edd644f023a15d87bd35cfe535ca0edcddb39f37630c283801bed26821f3cfa6bfcb478bdd987fae0a549bbdbda56cb4258430f8f676636c368afcddcc
|
|
7
|
+
data.tar.gz: 44a3eba562af3a0315391f362b28684c54f543110f6f76dd6e99e576655c749866a737f1235cf1971992c0274ed3a1da21db3cf83b07cc83ec28e76ef8d17424
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -29,7 +29,7 @@ jobs:
|
|
|
29
29
|
- ruby: '4.0'
|
|
30
30
|
coverage: 'true'
|
|
31
31
|
steps:
|
|
32
|
-
- uses: actions/checkout@
|
|
32
|
+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
33
33
|
with:
|
|
34
34
|
fetch-depth: 0
|
|
35
35
|
|
|
@@ -37,7 +37,7 @@ jobs:
|
|
|
37
37
|
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
|
38
38
|
|
|
39
39
|
- name: Set up Ruby
|
|
40
|
-
uses: ruby/setup-ruby@
|
|
40
|
+
uses: ruby/setup-ruby@9eb537ca036ebaed86729dcb9309076e4c5c3b74 # v1.314.0
|
|
41
41
|
with:
|
|
42
42
|
ruby-version: ${{matrix.ruby}}
|
|
43
43
|
bundler: 'latest'
|
|
@@ -65,11 +65,11 @@ jobs:
|
|
|
65
65
|
env:
|
|
66
66
|
BUNDLE_GEMFILE: Gemfile.lint
|
|
67
67
|
steps:
|
|
68
|
-
- uses: actions/checkout@
|
|
68
|
+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
69
69
|
with:
|
|
70
70
|
fetch-depth: 0
|
|
71
71
|
- name: Set up Ruby
|
|
72
|
-
uses: ruby/setup-ruby@
|
|
72
|
+
uses: ruby/setup-ruby@9eb537ca036ebaed86729dcb9309076e4c5c3b74 # v1.314.0
|
|
73
73
|
with:
|
|
74
74
|
ruby-version: '4.0.5'
|
|
75
75
|
bundler-cache: true
|
|
@@ -82,11 +82,11 @@ jobs:
|
|
|
82
82
|
env:
|
|
83
83
|
BUNDLE_GEMFILE: Gemfile.lint
|
|
84
84
|
steps:
|
|
85
|
-
- uses: actions/checkout@
|
|
85
|
+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
86
86
|
with:
|
|
87
87
|
fetch-depth: 0
|
|
88
88
|
- name: Set up Ruby
|
|
89
|
-
uses: ruby/setup-ruby@
|
|
89
|
+
uses: ruby/setup-ruby@9eb537ca036ebaed86729dcb9309076e4c5c3b74 # v1.314.0
|
|
90
90
|
with:
|
|
91
91
|
ruby-version: '4.0.5'
|
|
92
92
|
bundler-cache: true
|
|
@@ -97,7 +97,7 @@ jobs:
|
|
|
97
97
|
timeout-minutes: 5
|
|
98
98
|
runs-on: ubuntu-latest
|
|
99
99
|
steps:
|
|
100
|
-
- uses: actions/checkout@
|
|
100
|
+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
101
101
|
with:
|
|
102
102
|
fetch-depth: 0
|
|
103
103
|
- name: Set up Node.js
|
data/.github/workflows/push.yml
CHANGED
|
@@ -19,12 +19,12 @@ jobs:
|
|
|
19
19
|
id-token: write
|
|
20
20
|
|
|
21
21
|
steps:
|
|
22
|
-
- uses: actions/checkout@
|
|
22
|
+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
23
23
|
with:
|
|
24
24
|
fetch-depth: 0
|
|
25
25
|
|
|
26
26
|
- name: Set up Ruby
|
|
27
|
-
uses: ruby/setup-ruby@
|
|
27
|
+
uses: ruby/setup-ruby@9eb537ca036ebaed86729dcb9309076e4c5c3b74 # v1.314.0
|
|
28
28
|
with:
|
|
29
29
|
bundler-cache: false
|
|
30
30
|
|
|
@@ -32,4 +32,4 @@ jobs:
|
|
|
32
32
|
run: |
|
|
33
33
|
bundle install --jobs 4 --retry 3
|
|
34
34
|
|
|
35
|
-
- uses: rubygems/release-gem@
|
|
35
|
+
- uses: rubygems/release-gem@052cc82692552de3ef2b81fd670e41d13cba8092 # v1.4.0
|
|
@@ -7,7 +7,7 @@ jobs:
|
|
|
7
7
|
verify_action_pins:
|
|
8
8
|
runs-on: ubuntu-latest
|
|
9
9
|
steps:
|
|
10
|
-
- uses: actions/checkout@
|
|
10
|
+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
|
11
11
|
- name: Check SHA pins
|
|
12
12
|
run: |
|
|
13
13
|
if grep -E -r "uses: .*/.*@(v[0-9]+|main|master)($|[[:space:]]|$)" --include="*.yml" --include="*.yaml" .github/workflows/ | grep -v "#"; then
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
# Karafka Core Changelog
|
|
2
2
|
|
|
3
|
+
## 2.6.2 (2026-06-29)
|
|
4
|
+
- [Enhancement] Document that a leaf's `default` value is intentionally shared by reference across the class template and every config instance produced by `Configurable::Node#deep_dup`. This uniform rule (the leaf is shallow-copied) is what lets a shared service object passed as a default (e.g. a logger) keep its identity across all configs; the flip side is that an in-place mutation of a mutable container default (`config.list << :x`) is visible on every instance. Callers that need a per-instance mutable default should assign it inside a `configure` block or dup it themselves rather than relying on a mutable `default:` (e.g. `default: []`). Adds characterization tests covering the shared-default behavior.
|
|
5
|
+
- [Fix] `require "pathname"` explicitly in `lib/karafka-core.rb` (with the other top-level requires). `Karafka::Core.gem_root` returns a `Pathname`, but the gem never required `pathname` -- it only worked because Bundler (or another gem) happened to load it first. In an environment where nothing else loads it, `gem_root` raised `NameError: uninitialized constant Pathname`.
|
|
6
|
+
- [Enhancement] Document the `virtual` rule result contract: a rule must return a freshly built `Array` of `[path, message]` error pairs on every call. `Contract#call` takes ownership of that array and prepends the current scope onto each pair in place (avoiding a per-error allocation), so returning a memoized, shared or frozen array is unsupported -- the in-place scoping would accumulate the scope prefix across validations or raise `FrozenError`. Adds characterization tests for the supported and unsupported patterns.
|
|
7
|
+
- [Fix] `Configurable::Node#register` raises the documented "already registered" `ArgumentError` for a name already used by an unread lazy-with-constructor setting. The duplicate guard only checked `@configs_refs`, but a lazy setting with a constructor is absent from it until first read, so `register` silently overwrote it; it now checks the defined children.
|
|
8
|
+
- [Fix] `Contractable::Contract.nested` now pops its path in an `ensure`. If the block raised while the contract was being defined and the caller rescued it, the path stayed on the nesting stack and was prefixed onto every rule defined afterwards.
|
|
9
|
+
- [Fix] `Contract#call` no longer raises `NoMethodError` when validating a non-Hash root with a 1-key or 2-key rule path; it reports the path as missing, consistent with the 3+-key path (and the non-Hash intermediate handling added in 2.6.1).
|
|
10
|
+
- [Fix] Honor `excluded_keys` containing `"cgrp"` in `StatisticsDecorator` `only_keys` mode. The `cgrp` branch of the structure-aware fast path lacked the exclusion guard that the `brokers` and `topics` branches have, so excluding the consumer-group subtree still decorated it (inconsistent with the full-decoration path).
|
|
11
|
+
- [Fix] Guard the patched rdkafka error callback against a null client pointer. librdkafka can invoke the error callback with a NULL `rd_kafka_t` (e.g. very early in client construction); calling `rd_kafka_name` on it dereferenced the null pointer and could segfault the process. Mirrors the upstream `ErrorCallback`.
|
|
12
|
+
- [Fix] Resolve fatal errors in the patched rdkafka error callback. `ERR__FATAL` is only a generic marker, so the callback now fetches the real underlying error code and description via `RdkafkaError.build_fatal` (`rd_kafka_fatal_error`) instead of reporting the generic fatal code. Mirrors the upstream `ErrorCallback`.
|
|
13
|
+
- [Fix] A lazy setting declared without a constructor (`setting(:x, lazy: true)`) no longer raises when its accessor is read. `lazy: true` only makes sense together with a constructor to (re)evaluate; without one there is nothing to evaluate, so such a setting now behaves like a regular setting backed by its default. Previously reading it raised `NoMethodError` — `nil.arity` through the dynamic accessor for a falsy default, or a missing accessor for a truthy default.
|
|
14
|
+
- [Fix] `Contract#call` no longer raises `NoMethodError` when a virtual rule returns `false`. A virtual rule now signals "no errors" with any non-Array result (`true`/`false`/`nil`); only an `Array` of error pairs is collected. Previously a `false` return reached `false.each` (a `nil` return was already tolerated).
|
|
15
|
+
- [Fix] Manage `CallbacksManager` callbacks copy-on-write: `add`/`delete` rebuild and atomically swap an immutable values snapshot under a mutex, and `#call` iterates that snapshot directly. The previous values cache (introduced in 2.5.11) was lazily invalidated from within `#call`, which could not be done atomically against a concurrent `add`/`delete`, so a callback racing with dispatch could be silently lost — a removed one kept firing forever, or a newly added one never fired. The copy-on-write read takes no lock and allocates nothing per call, so the race is fixed without reintroducing a per-call `#values` allocation. librdkafka statistics/error callbacks fire from a background thread while callbacks are registered/unregistered, so this was reachable in practice.
|
|
16
|
+
- [Fix] Manage `Notifications` subscriptions copy-on-write (`#subscribe`, `#unsubscribe` and `#clear` replace the per-event listener array instead of mutating it in place) so a listener that unsubscribes itself (or another) from within its own handler no longer causes the listener following it to be silently skipped, and concurrent subscribe/unsubscribe during dispatch is safe. Dispatch keeps iterating the live array directly, so there is no per-notification allocation on the hot path.
|
|
17
|
+
- [Fix] Report a freeze duration (`_fd`) of `0` for statistics keys that are newly introduced in an emission (e.g. a broker or partition that appears mid-stream) instead of the elapsed time since the previous emission. A key that did not exist in the prior emission could not have been "frozen" for any duration, so accumulating the inter-emission gap was incorrect and also made the related `StatisticsDecorator` spec flaky on slow CIs (`_fd` depended on the wall-clock gap between the two emissions).
|
|
18
|
+
- [Fix] Make assigning a setting on a frozen `Configurable::Node` atomic. The ivar-backed writer evaluated `@configs_refs[name] = value` before `instance_variable_set`, so a frozen node mutated the canonical store and only then raised `FrozenError`, leaving the store and the ivar-backed reader permanently out of sync. It now raises before touching any state.
|
|
19
|
+
- [Fix] `Configurable::Node#to_h` now evaluates a setting's constructor with its default (arity-aware, matching `#compile`) instead of calling it with no arguments. The documented `->(default) { ... }` constructor form previously raised `ArgumentError: wrong number of arguments` from `#to_h` whenever the value was not yet in the config store (e.g. `#to_h` on an unconfigured instance, or an unread lazy setting).
|
|
20
|
+
- [Fix] Honor `excluded_keys` inside `StatisticsDecorator` `only_keys` decoration. A key listed in both `only_keys` and `excluded_keys` was still decorated because the direct-access decoration loop never consulted `excluded_keys`; exclusion now wins, matching the full-decoration path.
|
|
21
|
+
- [Fix] Strip the tests/specs root directory as an anchored prefix (`sub(/\A.../)`) instead of a global `gsub` in `MinitestLocator` and `RSpecLocator`. When the root directory string recurred later in a test/spec file path, the global replace removed every occurrence and corrupted the derived subject class path; only the leading prefix is now removed.
|
|
22
|
+
|
|
23
|
+
## 2.6.1 (2026-06-15)
|
|
24
|
+
- [Enhancement] Speed up `Contract#call` by ~1.25x for minimal and ~1.4x for fully populated data: resolve rule paths with a single `Hash#fetch` per level instead of `key?` + `[]`, inline the per-rule type dispatch into the rules loop, and compare the dig sentinel via `#equal?` so `#==` is never dispatched to the validated (user-provided) values. This is the per-message validation path in WaterDrop producers.
|
|
25
|
+
- [Fix] `Contract#call` with rule paths of 3+ keys no longer raises `NoMethodError` when an intermediate value is not a `Hash` and reports the path as missing instead, consistent with the 2-key path behavior.
|
|
26
|
+
- [Change] Reject reserved setting names with an `ArgumentError` in `Configurable::Node#setting` and `#register`: internal state names (`node_name`, `children`, `nestings`, `compiled`, `configs_refs`, `local_defs`) and the node public API names (`setting`, `configure`, `to_h`, `deep_dup`, `register`, `compile`). Previously such names silently shadowed the node own accessors, breaking `deep_dup` or `to_h`, and assignments like `config.children = value` corrupted the node internal state.
|
|
27
|
+
- [Enhancement] Skip the event name mapping hash lookup in `Monitor#instrument` when no namespace is used and the event id is already a `String`, which is the case for all events in the Karafka ecosystem (~1.2x faster dispatch on the common no-subscribers path). Symbol event ids and namespaced monitors keep going through the mapping.
|
|
28
|
+
- [Enhancement] Mirror config values into instance variables and use `attr_reader` based readers in `Configurable::Node`, yielding ~1.4x faster flat and ~1.6x faster nested settings reads on hot paths. `@configs_refs` remains the canonical store; non-identifier setting names (e.g. registered names with dashes) keep the previous hash-based accessors.
|
|
29
|
+
- [Enhancement] Instantiate each `Configurable::Node` through a per-layout anonymous subclass so the ivar-backed settings do not grow object shape variations on the shared `Node` class (which would degrade ivar access and trigger Ruby performance warnings). `deep_dup` reuses the template's subclass, so duplicated configs share object shapes.
|
|
30
|
+
- [Fix] Symbolize setting names at definition time (`setting`, same as `register`) and on config store writes so `String` setting names work end to end (accessors, `#to_h`, recompilation state) and cannot corrupt node internal state when matching reserved internal names (previously string-named settings were quietly broken as accessors and the store disagreed on the key type).
|
|
31
|
+
- [Change] Config nodes are now instances of anonymous `Node` subclasses: `is_a?(Karafka::Core::Configurable::Node)` still holds, but `instance_of?(Node)` is now `false` and `node.class.name` is `nil`.
|
|
32
|
+
- [Change] Assigning a setting on a frozen config node now raises `FrozenError` (previously the write silently mutated internal storage despite the freeze).
|
|
33
|
+
|
|
3
34
|
## 2.6.0 (2026-06-10)
|
|
4
35
|
- [Enhancement] Add `Node#register` to allow runtime key-value registration on compiled nodes without going through the static `setting` DSL. Useful for dynamic registries (e.g. named clusters) where setting names are not known at class-load time.
|
|
5
36
|
- [Enhancement] Replace version-gated `Warning[:performance]` with a `Warning.categories`-based loop that enables all opt-in Ruby warning categories automatically, picking up new categories (e.g. `strict_unused_block` in Ruby 3.4+) without future patches.
|
|
@@ -22,6 +53,7 @@
|
|
|
22
53
|
- [Enhancement] Replace `StatisticsDecorator#diff` pending-writes buffer with `keys.each` direct-write iteration, eliminating the buffer and write-back loop for ~13% faster decoration at scale (10 brokers, 20 topics, 2000 partitions).
|
|
23
54
|
- [Enhancement] Reorder `StatisticsDecorator#diff` type checks to test `Numeric` before `Hash`, matching the ~80% numeric value distribution in librdkafka statistics.
|
|
24
55
|
- [Enhancement] Support `only_keys` option in `StatisticsDecorator` to decorate only specified numeric keys (e.g. `consumer_lag`, `committed_offset`). When combined with `excluded_keys`, reduces decoration cost from ~80ms to ~8.5ms per call on large clusters (10 brokers, 20 topics, 2000 partitions) by using structure-aware navigation of the librdkafka statistics tree and direct key access instead of full-hash iteration.
|
|
56
|
+
- [Enhancement] Cache `Tags#to_a` values array and invalidate on `add`/`delete`/`clear` to avoid allocating a new Array and running `uniq` on every call, yielding ~7x faster reads at 5 tags and ~28x faster at 20 tags.
|
|
25
57
|
|
|
26
58
|
## 2.5.10 (2026-03-02)
|
|
27
59
|
- [Enhancement] Introduce `MinitestLocator` helper for minitest/spec subject class auto-discovery from test file paths.
|
data/Gemfile.lock
CHANGED
|
@@ -15,6 +15,49 @@ module Karafka
|
|
|
15
15
|
# We need to be able to redefine children for deep copy
|
|
16
16
|
attr_accessor :children
|
|
17
17
|
|
|
18
|
+
# Names that cannot be used as setting names because they would collide with the node
|
|
19
|
+
# internal state or the node public API: their accessors would shadow the node own
|
|
20
|
+
# readers (breaking for example `#deep_dup` or `#to_h`) and writers like `children=`
|
|
21
|
+
# would overwrite internal ivars. `#setting` and `#register` reject them upfront and
|
|
22
|
+
# `#ivar_backed?` keeps guarding the ivar mirror as defense in depth.
|
|
23
|
+
# Private method names are deliberately not reserved: that would make internal
|
|
24
|
+
# implementation details part of the public contract
|
|
25
|
+
RESERVED_NAMES = %i[
|
|
26
|
+
node_name
|
|
27
|
+
children
|
|
28
|
+
nestings
|
|
29
|
+
compiled
|
|
30
|
+
configs_refs
|
|
31
|
+
local_defs
|
|
32
|
+
setting
|
|
33
|
+
configure
|
|
34
|
+
to_h
|
|
35
|
+
deep_dup
|
|
36
|
+
register
|
|
37
|
+
compile
|
|
38
|
+
].to_h { |name| [name, true] }.freeze
|
|
39
|
+
|
|
40
|
+
# Setting names that match this format can be backed by instance variables and use the
|
|
41
|
+
# fast `attr_reader` based readers. Others (e.g. registered names with dashes) fall back
|
|
42
|
+
# to the hash-based accessors
|
|
43
|
+
IVAR_NAMEABLE_FORMAT = /\A[A-Za-z_][A-Za-z0-9_]*\z/
|
|
44
|
+
|
|
45
|
+
private_constant :RESERVED_NAMES, :IVAR_NAMEABLE_FORMAT
|
|
46
|
+
|
|
47
|
+
class << self
|
|
48
|
+
# Builds each node through its own anonymous subclass. Since setting values are
|
|
49
|
+
# mirrored into instance variables for fast access and each node layout carries a
|
|
50
|
+
# different set of them, instantiating nodes directly from this class would grow its
|
|
51
|
+
# object shape variations past the Ruby limit, degrading ivar access for all nodes.
|
|
52
|
+
# A subclass per layout keeps shape variations per class minimal (late `setting`
|
|
53
|
+
# calls after inheritance or runtime `register` calls may add a few more, staying
|
|
54
|
+
# well under the limit). `#deep_dup` reuses the subclass of its template, so
|
|
55
|
+
# duplicated configs share shapes as well.
|
|
56
|
+
def new(...)
|
|
57
|
+
equal?(Node) ? Class.new(self).new(...) : super
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
18
61
|
# @param node_name [Symbol] node name
|
|
19
62
|
# @param nestings [Proc] block for nested settings
|
|
20
63
|
# @param evaluate [Boolean] when false, skip evaluating the nestings block. Used by
|
|
@@ -31,12 +74,20 @@ module Karafka
|
|
|
31
74
|
|
|
32
75
|
# Allows for a single leaf or nested node definition
|
|
33
76
|
#
|
|
34
|
-
# @param node_name [Symbol] setting or nested node name
|
|
77
|
+
# @param node_name [Symbol, String] setting or nested node name
|
|
35
78
|
# @param default [Object] default value
|
|
36
79
|
# @param constructor [#call, nil] callable or nil
|
|
37
80
|
# @param lazy [Boolean] is this a lazy leaf
|
|
38
81
|
# @param block [Proc] block for nested settings
|
|
82
|
+
# @raise [ArgumentError] when the name is reserved for the node internal state
|
|
39
83
|
def setting(node_name, default: nil, constructor: nil, lazy: false, &block)
|
|
84
|
+
# Symbolize at definition time (same as `#register`) so the config store, accessors,
|
|
85
|
+
# `#to_h` and the compile state checks all agree on the key type also when a String
|
|
86
|
+
# name is provided
|
|
87
|
+
node_name = node_name.to_sym
|
|
88
|
+
|
|
89
|
+
prevent_reserved_names!(node_name)
|
|
90
|
+
|
|
40
91
|
@children << if block
|
|
41
92
|
Node.new(node_name, block)
|
|
42
93
|
else
|
|
@@ -65,7 +116,10 @@ module Karafka
|
|
|
65
116
|
result = if @configs_refs.key?(value.node_name)
|
|
66
117
|
@configs_refs[value.node_name]
|
|
67
118
|
elsif value.constructor
|
|
68
|
-
|
|
119
|
+
# Use the arity-aware helper (same as `#compile`) so a `->(default) { ... }`
|
|
120
|
+
# constructor receives its default instead of being called with no arguments,
|
|
121
|
+
# which would raise `ArgumentError: wrong number of arguments`.
|
|
122
|
+
call_constructor(value)
|
|
69
123
|
elsif value.default
|
|
70
124
|
value.default
|
|
71
125
|
end
|
|
@@ -85,13 +139,25 @@ module Karafka
|
|
|
85
139
|
# and non-side-effect usage on an instance/inherited.
|
|
86
140
|
# @return [Node] duplicated node
|
|
87
141
|
def deep_dup
|
|
88
|
-
|
|
142
|
+
# Same-layout nodes reuse the class of their template so they share object shapes
|
|
143
|
+
dupped = self.class.new(node_name, nestings, evaluate: false)
|
|
89
144
|
|
|
90
145
|
children.each do |value|
|
|
91
146
|
dupped.children << if value.is_a?(Leaf)
|
|
92
147
|
# After inheritance we need to reload the state so the leafs are recompiled again
|
|
93
148
|
value = value.dup
|
|
94
149
|
value.compiled = false
|
|
150
|
+
|
|
151
|
+
# `Struct#dup` is intentionally shallow here: the leaf's `default` value is shared by
|
|
152
|
+
# reference across the class template and every config instance produced by
|
|
153
|
+
# `deep_dup`. This is the contract -- one uniform rule for all default types -- and it
|
|
154
|
+
# is what lets a shared service object passed as a default (e.g. a logger) keep its
|
|
155
|
+
# identity across all configs instead of being cloned per instance. The flip side is
|
|
156
|
+
# that an in-place mutation of a mutable container default (e.g. `config.list << :x`)
|
|
157
|
+
# is visible on every other instance and on the template. A caller that needs a
|
|
158
|
+
# per-instance mutable default should not rely on a mutable `default:` (e.g.
|
|
159
|
+
# `default: []`): assign the value inside a `configure` block, or dup it themselves,
|
|
160
|
+
# so each instance owns its own copy.
|
|
95
161
|
value
|
|
96
162
|
else
|
|
97
163
|
value.deep_dup
|
|
@@ -116,16 +182,24 @@ module Karafka
|
|
|
116
182
|
# @param name [Symbol, String] setting name
|
|
117
183
|
# @param value [Object] the setting value assigned immediately; also used as the default
|
|
118
184
|
# when the node is deep-duped and recompiled on a new instance
|
|
119
|
-
# @raise [ArgumentError] when the name is already taken
|
|
185
|
+
# @raise [ArgumentError] when the name is already taken or reserved for the node
|
|
186
|
+
# internal state
|
|
120
187
|
def register(name, value)
|
|
121
188
|
name = name.to_sym
|
|
122
189
|
|
|
123
|
-
|
|
190
|
+
prevent_reserved_names!(name)
|
|
191
|
+
|
|
192
|
+
# Check the defined children, not just @configs_refs: a lazy setting with a constructor
|
|
193
|
+
# is not written to @configs_refs until first read, so a `@configs_refs.key?` guard
|
|
194
|
+
# alone would silently overwrite it instead of raising the documented error.
|
|
195
|
+
if @children.any? { |child| child.node_name == name }
|
|
196
|
+
raise ArgumentError, "#{name} is already registered"
|
|
197
|
+
end
|
|
124
198
|
|
|
125
199
|
leaf = Leaf.new(name, value, nil, true, false)
|
|
126
200
|
@children << leaf
|
|
127
201
|
build_accessors(leaf)
|
|
128
|
-
|
|
202
|
+
config_write(name, value)
|
|
129
203
|
end
|
|
130
204
|
|
|
131
205
|
# Converts the settings definitions into end children
|
|
@@ -135,7 +209,12 @@ module Karafka
|
|
|
135
209
|
# Do not redefine something that was already set during compilation
|
|
136
210
|
# This will allow us to reconfigure things and skip override with defaults
|
|
137
211
|
skippable = @configs_refs.key?(value.node_name) || (value.is_a?(Leaf) && value.compiled?)
|
|
138
|
-
|
|
212
|
+
# A leaf is only treated as lazy (a dynamically (re)evaluated accessor) when it
|
|
213
|
+
# actually has a constructor to evaluate. `lazy: true` without a constructor has
|
|
214
|
+
# nothing to evaluate, so it behaves like a regular setting backed by its default.
|
|
215
|
+
# Otherwise the lazy path builds a dynamic accessor that calls `constructor.arity`
|
|
216
|
+
# on a `nil` constructor and crashes on first read.
|
|
217
|
+
lazy_leaf = value.is_a?(Leaf) && value.lazy? && !value.constructor.nil?
|
|
139
218
|
|
|
140
219
|
# Do not create accessor for leafs that are lazy as they will get a custom method
|
|
141
220
|
# created instead
|
|
@@ -161,7 +240,7 @@ module Karafka
|
|
|
161
240
|
if lazy_leaf && !initialized
|
|
162
241
|
build_dynamic_accessor(value)
|
|
163
242
|
else
|
|
164
|
-
|
|
243
|
+
config_write(value.node_name, initialized)
|
|
165
244
|
end
|
|
166
245
|
end
|
|
167
246
|
|
|
@@ -207,6 +286,12 @@ module Karafka
|
|
|
207
286
|
|
|
208
287
|
# Builds regular accessors for value fetching
|
|
209
288
|
#
|
|
289
|
+
# Settings with names that can form valid instance variables get `attr_reader` based
|
|
290
|
+
# readers backed by an ivar mirror of the config value. This is significantly faster
|
|
291
|
+
# than a method with a hash lookup, which matters since settings are read on hot paths
|
|
292
|
+
# across the whole ecosystem. `@configs_refs` remains the canonical store used by
|
|
293
|
+
# `#to_h`, `#compile` and `#register`, with `#config_write` keeping the mirror in sync.
|
|
294
|
+
#
|
|
210
295
|
# @param value [Leaf]
|
|
211
296
|
def build_accessors(value)
|
|
212
297
|
reader_name = value.node_name.to_sym
|
|
@@ -219,17 +304,65 @@ module Karafka
|
|
|
219
304
|
if reader_respond ? !@local_defs.key?(reader_name) : true
|
|
220
305
|
@local_defs[reader_name] = true
|
|
221
306
|
|
|
222
|
-
|
|
223
|
-
|
|
307
|
+
if ivar_backed?(reader_name)
|
|
308
|
+
singleton_class.attr_reader(reader_name)
|
|
309
|
+
else
|
|
310
|
+
define_singleton_method(reader_name) do
|
|
311
|
+
@configs_refs[reader_name]
|
|
312
|
+
end
|
|
224
313
|
end
|
|
225
314
|
end
|
|
226
315
|
|
|
227
|
-
return if respond_to?(:"#{
|
|
316
|
+
return if respond_to?(:"#{reader_name}=")
|
|
228
317
|
|
|
229
|
-
|
|
230
|
-
|
|
318
|
+
if ivar_backed?(reader_name)
|
|
319
|
+
ivar_name = :"@#{reader_name}"
|
|
320
|
+
|
|
321
|
+
define_singleton_method(:"#{reader_name}=") do |new_value|
|
|
322
|
+
instance_variable_set(ivar_name, new_value)
|
|
323
|
+
@configs_refs[reader_name] = new_value
|
|
324
|
+
end
|
|
325
|
+
else
|
|
326
|
+
define_singleton_method(:"#{reader_name}=") do |new_value|
|
|
327
|
+
@configs_refs[reader_name] = new_value
|
|
328
|
+
end
|
|
231
329
|
end
|
|
232
330
|
end
|
|
331
|
+
|
|
332
|
+
# Writes a config value to the canonical store and mirrors it into the backing instance
|
|
333
|
+
# variable when the setting uses the fast ivar-backed reader
|
|
334
|
+
#
|
|
335
|
+
# @param name [Symbol, String] setting name
|
|
336
|
+
# @param value [Object] config value assigned to the setting
|
|
337
|
+
def config_write(name, value)
|
|
338
|
+
# Accessors operate on symbolized names, so the store has to be keyed consistently.
|
|
339
|
+
# This also guarantees that a String name matching a reserved internal name is
|
|
340
|
+
# recognized by the `ivar_backed?` guard and cannot corrupt the node internal state
|
|
341
|
+
name = name.to_sym
|
|
342
|
+
|
|
343
|
+
@configs_refs[name] = value
|
|
344
|
+
instance_variable_set(:"@#{name}", value) if ivar_backed?(name)
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# @param name [Symbol] setting name
|
|
348
|
+
# @return [Boolean] true if this setting can be backed by an instance variable and use
|
|
349
|
+
# the fast `attr_reader` based reader
|
|
350
|
+
def ivar_backed?(name)
|
|
351
|
+
!RESERVED_NAMES.key?(name) && IVAR_NAMEABLE_FORMAT.match?(name)
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
# Rejects setting names that would collide with the node internal state. Without this,
|
|
355
|
+
# such names would shadow the node own accessors, breaking `#deep_dup` and silently
|
|
356
|
+
# corrupting internals on assignment (e.g. `config.children = value` hitting the node
|
|
357
|
+
# own `attr_writer`)
|
|
358
|
+
#
|
|
359
|
+
# @param name [Symbol] already symbolized setting name
|
|
360
|
+
# @raise [ArgumentError] when the name is reserved
|
|
361
|
+
def prevent_reserved_names!(name)
|
|
362
|
+
return unless RESERVED_NAMES.key?(name)
|
|
363
|
+
|
|
364
|
+
raise ArgumentError, "#{name} is a reserved name and cannot be used as a setting name"
|
|
365
|
+
end
|
|
233
366
|
end
|
|
234
367
|
end
|
|
235
368
|
end
|
|
@@ -15,7 +15,7 @@ module Karafka
|
|
|
15
15
|
DIG_MISS = Object.new
|
|
16
16
|
|
|
17
17
|
# Empty array for scope default to avoid allocating a new Array on each
|
|
18
|
-
# `#call` / `#validate!` invocation. Safe because scope is never mutated
|
|
18
|
+
# `#call` / `#validate!` invocation. Safe because scope is never mutated - it is only
|
|
19
19
|
# used in `scope + rule.path` which creates a new Array.
|
|
20
20
|
EMPTY_ARRAY = [].freeze
|
|
21
21
|
|
|
@@ -40,8 +40,14 @@ module Karafka
|
|
|
40
40
|
def nested(path, &)
|
|
41
41
|
init_accu
|
|
42
42
|
@nested << path
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
begin
|
|
44
|
+
instance_eval(&)
|
|
45
|
+
ensure
|
|
46
|
+
# Always pop, even if the block raises. Otherwise a rescued exception during
|
|
47
|
+
# contract definition would leave `path` on @nested and prefix it onto every rule
|
|
48
|
+
# defined afterwards.
|
|
49
|
+
@nested.pop
|
|
50
|
+
end
|
|
45
51
|
end
|
|
46
52
|
|
|
47
53
|
# Defines a rule for a required field (required means, that will automatically create an
|
|
@@ -61,10 +67,19 @@ module Karafka
|
|
|
61
67
|
@rules << Rule.new(@nested + keys, :optional, block).freeze
|
|
62
68
|
end
|
|
63
69
|
|
|
64
|
-
#
|
|
70
|
+
# Defines a virtual rule that validates the whole data rather than a single key. Unlike
|
|
71
|
+
# `required`/`optional`, the block receives the full data and returns its own errors.
|
|
65
72
|
#
|
|
66
|
-
# @
|
|
67
|
-
#
|
|
73
|
+
# @param block [Proc] validation rule, called with `(data, errors, contract)`. It must
|
|
74
|
+
# return either a non-Array (`true`/`nil`/`false`) for "no errors", or an `Array` of
|
|
75
|
+
# `[path, message]` error pairs (where `path` is itself an `Array` of symbols).
|
|
76
|
+
#
|
|
77
|
+
# @note The returned error `Array` is owned by the contract: `#call` prepends the current
|
|
78
|
+
# scope onto each pair in place and collects them, so a rule must return a freshly
|
|
79
|
+
# built `Array` on every call. Returning a memoized, shared or frozen `Array` is not
|
|
80
|
+
# supported -- in-place scoping would accumulate the prefix across validations (or
|
|
81
|
+
# raise `FrozenError`). Build the result in the block (e.g. `[[%i[id], :invalid]]`)
|
|
82
|
+
# rather than returning a constant.
|
|
68
83
|
def virtual(&block)
|
|
69
84
|
init_accu
|
|
70
85
|
@rules << Rule.new([], :virtual, block).freeze
|
|
@@ -81,6 +96,12 @@ module Karafka
|
|
|
81
96
|
|
|
82
97
|
# Runs the validation
|
|
83
98
|
#
|
|
99
|
+
# The per-rule handling is inlined instead of dispatching to per-type methods because
|
|
100
|
+
# this runs per rule per validation, including the per-message validations in
|
|
101
|
+
# WaterDrop. Required and optional rules share the whole flow except the missing-key
|
|
102
|
+
# handling. `DIG_MISS` is compared via `#equal?` so we never dispatch `#==` to the
|
|
103
|
+
# validated (user-provided) values.
|
|
104
|
+
#
|
|
84
105
|
# @param data [Hash] hash with data we want to validate
|
|
85
106
|
# @param scope [Array<String>] scope of this contract (if any) or empty array if no parent
|
|
86
107
|
# scope is needed if contract starts from root
|
|
@@ -88,14 +109,42 @@ module Karafka
|
|
|
88
109
|
def call(data, scope: EMPTY_ARRAY)
|
|
89
110
|
errors = []
|
|
90
111
|
|
|
112
|
+
# A non-Hash root has no keys to dig; resolve it once here so #dig stays on its lean
|
|
113
|
+
# fast path and is only invoked with a Hash. Non-Hash data makes every required rule
|
|
114
|
+
# report missing, consistent with the non-Hash intermediate handling inside #dig.
|
|
115
|
+
data_is_hash = data.is_a?(Hash)
|
|
116
|
+
|
|
91
117
|
self.class.rules.each do |rule|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
118
|
+
if rule.type == :virtual
|
|
119
|
+
result = rule.validator.call(data, errors, self)
|
|
120
|
+
|
|
121
|
+
# A virtual rule signals "no errors" with any non-Array result (true, but also a
|
|
122
|
+
# falsy `nil`/`false` returned by e.g. `condition && [[...], :err]`). Only an Array
|
|
123
|
+
# of error pairs is iterated; previously a `false` return reached `false.each` and
|
|
124
|
+
# raised NoMethodError (a `nil` return was already tolerated by the safe navigation).
|
|
125
|
+
next unless result.is_a?(Array)
|
|
126
|
+
|
|
127
|
+
# Apply the scope prefix in place on the rule's returned pairs and collect them
|
|
128
|
+
# directly. Per the `virtual` contract the rule hands back a freshly built Array
|
|
129
|
+
# each call (see `DSL#virtual`), so mutating it here is safe and avoids allocating a
|
|
130
|
+
# new pair per error.
|
|
131
|
+
result.each do |sub_result|
|
|
132
|
+
sub_result[0] = scope + sub_result[0]
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
errors.push(*result)
|
|
136
|
+
else
|
|
137
|
+
for_checking = data_is_hash ? dig(data, rule.path) : DIG_MISS
|
|
138
|
+
|
|
139
|
+
if DIG_MISS.equal?(for_checking)
|
|
140
|
+
errors << [scope + rule.path, :missing] if rule.type == :required
|
|
141
|
+
else
|
|
142
|
+
result = rule.validator.call(for_checking, data, errors, self)
|
|
143
|
+
|
|
144
|
+
next if result == true
|
|
145
|
+
|
|
146
|
+
errors << [scope + rule.path, result || :format]
|
|
147
|
+
end
|
|
99
148
|
end
|
|
100
149
|
end
|
|
101
150
|
|
|
@@ -121,98 +170,39 @@ module Karafka
|
|
|
121
170
|
|
|
122
171
|
private
|
|
123
172
|
|
|
124
|
-
# Runs validation for rules on fields that are required and adds errors (if any) to the
|
|
125
|
-
# errors array
|
|
126
|
-
#
|
|
127
|
-
# @param data [Hash] input hash
|
|
128
|
-
# @param rule [Rule] validation rule
|
|
129
|
-
# @param errors [Array] array with errors from previous rules (if any)
|
|
130
|
-
# @param scope [Array<String>]
|
|
131
|
-
def validate_required(data, rule, errors, scope)
|
|
132
|
-
for_checking = dig(data, rule.path)
|
|
133
|
-
|
|
134
|
-
# We need to compare `DIG_MISS` against stuff because of the ownership of the `#==` method
|
|
135
|
-
if for_checking == DIG_MISS
|
|
136
|
-
errors << [scope + rule.path, :missing]
|
|
137
|
-
else
|
|
138
|
-
result = rule.validator.call(for_checking, data, errors, self)
|
|
139
|
-
|
|
140
|
-
return if result == true
|
|
141
|
-
|
|
142
|
-
errors << [scope + rule.path, result || :format]
|
|
143
|
-
end
|
|
144
|
-
end
|
|
145
|
-
|
|
146
|
-
# Runs validation for rules on fields that are optional and adds errors (if any) to the
|
|
147
|
-
# errors array
|
|
148
|
-
#
|
|
149
|
-
# @param data [Hash] input hash
|
|
150
|
-
# @param rule [Rule] validation rule
|
|
151
|
-
# @param errors [Array] array with errors from previous rules (if any)
|
|
152
|
-
# @param scope [Array<String>]
|
|
153
|
-
def validate_optional(data, rule, errors, scope)
|
|
154
|
-
for_checking = dig(data, rule.path)
|
|
155
|
-
|
|
156
|
-
return if for_checking == DIG_MISS
|
|
157
|
-
|
|
158
|
-
result = rule.validator.call(for_checking, data, errors, self)
|
|
159
|
-
|
|
160
|
-
return if result == true
|
|
161
|
-
|
|
162
|
-
errors << [scope + rule.path, result || :format]
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
# Runs validation for rules on virtual fields (aggregates, etc) and adds errors (if any) to
|
|
166
|
-
# the errors array
|
|
167
|
-
#
|
|
168
|
-
# @param data [Hash] input hash
|
|
169
|
-
# @param rule [Rule] validation rule
|
|
170
|
-
# @param errors [Array] array with errors from previous rules (if any)
|
|
171
|
-
# @param scope [Array<String>]
|
|
172
|
-
def validate_virtual(data, rule, errors, scope)
|
|
173
|
-
result = rule.validator.call(data, errors, self)
|
|
174
|
-
|
|
175
|
-
return if result == true
|
|
176
|
-
|
|
177
|
-
result&.each do |sub_result|
|
|
178
|
-
sub_result[0] = scope + sub_result[0]
|
|
179
|
-
end
|
|
180
|
-
|
|
181
|
-
errors.push(*result)
|
|
182
|
-
end
|
|
183
|
-
|
|
184
173
|
# Tries to dig for a given key in a hash and returns it with indication whether or not it
|
|
185
174
|
# was possible to find it (dig returns nil and we don't know if it wasn't the digged key
|
|
186
175
|
# value)
|
|
187
176
|
#
|
|
177
|
+
# Uses `Hash#fetch` with the `DIG_MISS` sentinel as the default, which resolves presence
|
|
178
|
+
# and value in a single hash lookup instead of a `key?` check followed by `[]`. This
|
|
179
|
+
# runs per rule per validation, including the per-message validations in WaterDrop,
|
|
180
|
+
# hence the lookup count matters. `fetch` with a default ignores `default_proc`, same
|
|
181
|
+
# as the previous `key?` based logic.
|
|
182
|
+
#
|
|
188
183
|
# @param data [Hash]
|
|
189
184
|
# @param keys [Array<Symbol>]
|
|
190
185
|
# @return [DIG_MISS, Object] found element or DIGG_MISS indicating that not found
|
|
191
186
|
def dig(data, keys)
|
|
192
187
|
case keys.length
|
|
193
188
|
when 1
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
return DIG_MISS unless data.key?(key)
|
|
197
|
-
|
|
198
|
-
data[key]
|
|
189
|
+
data.fetch(keys[0], DIG_MISS)
|
|
199
190
|
when 2
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
return DIG_MISS unless data.key?(key1)
|
|
191
|
+
mid = data.fetch(keys[0], DIG_MISS)
|
|
203
192
|
|
|
204
|
-
|
|
193
|
+
return DIG_MISS if DIG_MISS.equal?(mid)
|
|
194
|
+
return DIG_MISS unless mid.is_a?(Hash)
|
|
205
195
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
mid[keys[1]]
|
|
196
|
+
mid.fetch(keys[1], DIG_MISS)
|
|
209
197
|
else
|
|
210
198
|
current = data
|
|
211
199
|
|
|
212
200
|
keys.each do |nesting|
|
|
213
|
-
return DIG_MISS unless current.
|
|
201
|
+
return DIG_MISS unless current.is_a?(Hash)
|
|
202
|
+
|
|
203
|
+
current = current.fetch(nesting, DIG_MISS)
|
|
214
204
|
|
|
215
|
-
|
|
205
|
+
return DIG_MISS if DIG_MISS.equal?(current)
|
|
216
206
|
end
|
|
217
207
|
|
|
218
208
|
current
|
|
@@ -10,20 +10,21 @@ module Karafka
|
|
|
10
10
|
# @return [::Karafka::Core::Instrumentation::CallbacksManager]
|
|
11
11
|
def initialize
|
|
12
12
|
@callbacks = {}
|
|
13
|
-
@
|
|
13
|
+
@values = [].freeze
|
|
14
|
+
@mutex = Mutex.new
|
|
14
15
|
end
|
|
15
16
|
|
|
16
17
|
# Invokes all the callbacks registered one after another
|
|
17
18
|
#
|
|
18
19
|
# @param args [Object] any args that should go to the callbacks
|
|
19
|
-
# @note
|
|
20
|
-
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
20
|
+
# @note Copy-on-write: dispatch iterates an immutable snapshot that `add`/`delete`
|
|
21
|
+
# rebuild and swap in under a mutex. Because `#call` never mutates shared state, it
|
|
22
|
+
# needs neither a lock nor a per-call `#values` allocation, and a callback registered
|
|
23
|
+
# or removed from another thread is never lost; it just takes effect on the next
|
|
24
|
+
# `#call`. A cache invalidated from within `#call` could not be updated atomically
|
|
25
|
+
# against this read, so a stale write-back would permanently drop callbacks.
|
|
25
26
|
def call(*args)
|
|
26
|
-
|
|
27
|
+
@values.each { |callback| callback.call(*args) }
|
|
27
28
|
end
|
|
28
29
|
|
|
29
30
|
# Adds a callback to the manager
|
|
@@ -31,15 +32,19 @@ module Karafka
|
|
|
31
32
|
# @param id [String] id of the callback (used when deleting it)
|
|
32
33
|
# @param callable [#call] object that responds to a `#call` method
|
|
33
34
|
def add(id, callable)
|
|
34
|
-
@
|
|
35
|
-
|
|
35
|
+
@mutex.synchronize do
|
|
36
|
+
@callbacks[id] = callable
|
|
37
|
+
@values = @callbacks.values.freeze
|
|
38
|
+
end
|
|
36
39
|
end
|
|
37
40
|
|
|
38
41
|
# Removes the callback from the manager
|
|
39
42
|
# @param id [String] id of the callback we want to remove
|
|
40
43
|
def delete(id)
|
|
41
|
-
@
|
|
42
|
-
|
|
44
|
+
@mutex.synchronize do
|
|
45
|
+
@callbacks.delete(id)
|
|
46
|
+
@values = @callbacks.values.freeze
|
|
47
|
+
end
|
|
43
48
|
end
|
|
44
49
|
end
|
|
45
50
|
end
|
|
@@ -28,7 +28,15 @@ module Karafka
|
|
|
28
28
|
# @param event_id [String, Symbol] event id
|
|
29
29
|
# @param payload [Hash]
|
|
30
30
|
def instrument(event_id, payload = EMPTY_HASH, &)
|
|
31
|
-
|
|
31
|
+
# With no namespace, string event ids already are the full event names. This is the
|
|
32
|
+
# case for all the events in the Karafka ecosystem, so we can skip the mapping hash
|
|
33
|
+
# lookup on this hot path. Symbols still go through the mapping to be converted into
|
|
34
|
+
# strings without allocating on each call.
|
|
35
|
+
full_event_name = if @namespace.nil? && event_id.is_a?(String)
|
|
36
|
+
event_id
|
|
37
|
+
else
|
|
38
|
+
@mapped_events[event_id] ||= [event_id, @namespace].compact.join(".")
|
|
39
|
+
end
|
|
32
40
|
|
|
33
41
|
@notifications_bus.instrument(full_event_name, payload, &)
|
|
34
42
|
end
|
|
@@ -48,8 +48,17 @@ module Karafka
|
|
|
48
48
|
# @param event_id [String] the key of the event to clear listeners for.
|
|
49
49
|
def clear(event_id = nil)
|
|
50
50
|
@mutex.synchronize do
|
|
51
|
-
|
|
52
|
-
|
|
51
|
+
# Copy-on-write: replace the per-event arrays rather than mutating them in place, so a
|
|
52
|
+
# dispatch iterating a previously captured array is unaffected (see #notify_listeners).
|
|
53
|
+
unless event_id
|
|
54
|
+
@listeners.transform_values! { [] }
|
|
55
|
+
return
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
if @listeners.key?(event_id)
|
|
59
|
+
@listeners[event_id] = []
|
|
60
|
+
return
|
|
61
|
+
end
|
|
53
62
|
|
|
54
63
|
raise(EventNotRegistered, "#{event_id} not registered!")
|
|
55
64
|
end
|
|
@@ -76,14 +85,16 @@ module Karafka
|
|
|
76
85
|
|
|
77
86
|
raise EventNotRegistered, event_id unless @listeners.key?(event_id)
|
|
78
87
|
|
|
79
|
-
|
|
88
|
+
# Copy-on-write: append by replacing the array, never mutating the one a concurrent
|
|
89
|
+
# dispatch may be iterating (see #notify_listeners).
|
|
90
|
+
@listeners[event_id] += [block]
|
|
80
91
|
else
|
|
81
92
|
listener = event_id_or_listener
|
|
82
93
|
|
|
83
94
|
@listeners.each_key do |reg_event_id|
|
|
84
95
|
next unless listener.respond_to?(@events_methods_map[reg_event_id])
|
|
85
96
|
|
|
86
|
-
@listeners[reg_event_id]
|
|
97
|
+
@listeners[reg_event_id] += [listener]
|
|
87
98
|
end
|
|
88
99
|
end
|
|
89
100
|
end
|
|
@@ -98,9 +109,9 @@ module Karafka
|
|
|
98
109
|
# unsubscribe(my_listener)
|
|
99
110
|
def unsubscribe(listener_or_block)
|
|
100
111
|
@mutex.synchronize do
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
112
|
+
# Copy-on-write: rebuild each array without the listener instead of deleting in place,
|
|
113
|
+
# so a dispatch iterating a previously captured array still sees the full set.
|
|
114
|
+
@listeners.transform_values! { |event_listeners| event_listeners - [listener_or_block] }
|
|
104
115
|
end
|
|
105
116
|
end
|
|
106
117
|
|
|
@@ -14,7 +14,16 @@ module Karafka
|
|
|
14
14
|
# - KEY_fd - freeze duration - describes how long the delta remains unchanged (zero)
|
|
15
15
|
# and can be useful for detecting values that "hang" for extended period of time
|
|
16
16
|
# and do not have any change (delta always zero). This value is in ms for the
|
|
17
|
-
# consistency with other time operators we use.
|
|
17
|
+
# consistency with other time operators we use. A newly introduced key (one
|
|
18
|
+
# that had no value in the previous emission) starts at a freeze duration of
|
|
19
|
+
# zero, since there is no prior value it could have been "frozen" against.
|
|
20
|
+
#
|
|
21
|
+
# The `_d` and `_fd` suffixes are reserved. For every numeric KEY the decorator writes
|
|
22
|
+
# `KEY_d` and `KEY_fd` into the same hash, so if the input already contains a real key
|
|
23
|
+
# literally named `KEY_d` or `KEY_fd` (for another numeric KEY present in that hash) it is
|
|
24
|
+
# overwritten by the computed delta/freeze duration. librdkafka statistics never use these
|
|
25
|
+
# suffixes, so this does not happen with real stats; it only matters if you feed custom data
|
|
26
|
+
# through the decorator.
|
|
18
27
|
class StatisticsDecorator
|
|
19
28
|
include Helpers::Time
|
|
20
29
|
|
|
@@ -28,11 +37,17 @@ module Karafka
|
|
|
28
37
|
# duration suffixes. This is useful for skipping large subtrees of the librdkafka
|
|
29
38
|
# statistics that are not consumed by the application (e.g. broker toppars, window
|
|
30
39
|
# stats like int_latency, outbuf_latency, throttle, batchsize, batchcnt, req).
|
|
31
|
-
# @param only_keys [Array<String>] when set, only these numeric keys
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
40
|
+
# @param only_keys [Array<String>] when set, only these numeric keys are decorated with
|
|
41
|
+
# delta/freeze duration suffixes, and only at the levels of the known librdkafka
|
|
42
|
+
# statistics tree: the root, each broker, each topic, each partition and cgrp. The
|
|
43
|
+
# decorator navigates that known structure directly and decorates the listed keys it
|
|
44
|
+
# finds at each of those levels. It deliberately does NOT descend into nested
|
|
45
|
+
# sub-objects within a broker, topic, partition or cgrp (e.g. broker window stats like
|
|
46
|
+
# rtt/throttle/int_latency, or the toppars map) -- skipping that descent is exactly what
|
|
47
|
+
# makes this mode cheap on large clusters. Non-librdkafka hash children found at the
|
|
48
|
+
# root are still fully recursed for correctness. If you need a key nested inside one of
|
|
49
|
+
# those sub-objects decorated, use the default full-decoration mode. When empty
|
|
50
|
+
# (default), all numeric keys at every depth are decorated.
|
|
36
51
|
def initialize(excluded_keys: [], only_keys: [])
|
|
37
52
|
@previous = EMPTY_HASH
|
|
38
53
|
# Operate on ms precision only
|
|
@@ -44,9 +59,12 @@ module Karafka
|
|
|
44
59
|
@excluded_keys = unless excluded_keys.empty?
|
|
45
60
|
excluded_keys.each_with_object({}) { |k, h| h[k] = true }.freeze
|
|
46
61
|
end
|
|
47
|
-
# Frozen array for direct-access decoration, nil when empty to use full decoration
|
|
62
|
+
# Frozen array for direct-access decoration, nil when empty to use full decoration.
|
|
63
|
+
# Exclusions are applied once here (excluded_keys wins over only_keys), so the hot
|
|
64
|
+
# decoration loop iterates an already-filtered list and never re-checks exclusions.
|
|
48
65
|
@only_keys = unless only_keys.empty?
|
|
49
|
-
only_keys.
|
|
66
|
+
effective = @excluded_keys ? only_keys.reject { |k| @excluded_keys.key?(k) } : only_keys
|
|
67
|
+
effective.freeze
|
|
50
68
|
end
|
|
51
69
|
end
|
|
52
70
|
|
|
@@ -120,18 +138,17 @@ module Karafka
|
|
|
120
138
|
if value.is_a?(Numeric)
|
|
121
139
|
prev_value = filled_previous[key]
|
|
122
140
|
|
|
141
|
+
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
142
|
+
|
|
123
143
|
if prev_value.nil?
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
result = value - prev_value
|
|
144
|
+
current[pair[0]] = 0
|
|
145
|
+
current[pair[1]] = 0
|
|
127
146
|
else
|
|
128
|
-
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
147
|
+
result = value - prev_value
|
|
132
148
|
|
|
133
|
-
|
|
134
|
-
|
|
149
|
+
current[pair[0]] = (result == 0) ? (filled_previous[pair[0]] || 0) + change_d : 0
|
|
150
|
+
current[pair[1]] = result
|
|
151
|
+
end
|
|
135
152
|
elsif value.is_a?(Hash)
|
|
136
153
|
diff_all(filled_previous[key], value, change_d)
|
|
137
154
|
end
|
|
@@ -156,8 +173,13 @@ module Karafka
|
|
|
156
173
|
# librdkafka statistics layout: root → brokers → broker, root → topics → topic →
|
|
157
174
|
# partitions → partition, root → cgrp.
|
|
158
175
|
#
|
|
159
|
-
#
|
|
160
|
-
#
|
|
176
|
+
# Broker, topic, partition and cgrp nodes are decorated as leaves: their listed keys are
|
|
177
|
+
# decorated, but their nested sub-objects (e.g. broker window stats like rtt/throttle, or
|
|
178
|
+
# the toppars map) are NOT descended into. Not descending into those large sub-objects is
|
|
179
|
+
# the whole point of this path, so they are intentionally left untouched here.
|
|
180
|
+
#
|
|
181
|
+
# For non-librdkafka hash children at the root (e.g. custom or test data), falls back to
|
|
182
|
+
# generic recursion to maintain correctness with arbitrary nested structures.
|
|
161
183
|
#
|
|
162
184
|
# @param previous [Object] previous value from the given scope
|
|
163
185
|
# @param current [Hash] current stats hash (root level)
|
|
@@ -209,8 +231,10 @@ module Karafka
|
|
|
209
231
|
end
|
|
210
232
|
|
|
211
233
|
# Consumer group (leaf-like)
|
|
212
|
-
|
|
213
|
-
|
|
234
|
+
unless excluded&.key?("cgrp")
|
|
235
|
+
cgrp = current["cgrp"]
|
|
236
|
+
decorate_keys(cgrp, filled_previous["cgrp"] || EMPTY_HASH, change_d) if cgrp.is_a?(Hash)
|
|
237
|
+
end
|
|
214
238
|
|
|
215
239
|
# Fallback: handle any non-standard hash children not in the known structure.
|
|
216
240
|
# This ensures correctness for arbitrary nested data while the known paths above
|
|
@@ -275,18 +299,17 @@ module Karafka
|
|
|
275
299
|
|
|
276
300
|
prev_value = filled_previous[key]
|
|
277
301
|
|
|
302
|
+
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
303
|
+
|
|
278
304
|
if prev_value.nil?
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
result = value - prev_value
|
|
305
|
+
current[pair[0]] = 0
|
|
306
|
+
current[pair[1]] = 0
|
|
282
307
|
else
|
|
283
|
-
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
pair = cache[key] || (cache[key] = ["#{key}_fd".freeze, "#{key}_d".freeze].freeze)
|
|
308
|
+
result = value - prev_value
|
|
287
309
|
|
|
288
|
-
|
|
289
|
-
|
|
310
|
+
current[pair[0]] = (result == 0) ? (filled_previous[pair[0]] || 0) + change_d : 0
|
|
311
|
+
current[pair[1]] = result
|
|
312
|
+
end
|
|
290
313
|
end
|
|
291
314
|
end
|
|
292
315
|
end
|
|
@@ -27,9 +27,28 @@ module Karafka
|
|
|
27
27
|
) do |client_ptr, err_code, reason, _opaque|
|
|
28
28
|
return nil unless ::Rdkafka::Config.error_callback
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
# Guard against a null client pointer. librdkafka can invoke the error callback
|
|
31
|
+
# with a NULL `rd_kafka_t` (e.g. very early in client construction), and calling
|
|
32
|
+
# `rd_kafka_name` on it dereferences the pointer and segfaults the whole process.
|
|
33
|
+
# Mirrors the upstream `ErrorCallback`.
|
|
34
|
+
name = client_ptr.null? ? nil : ::Rdkafka::Bindings.rd_kafka_name(client_ptr)
|
|
35
|
+
|
|
36
|
+
# Resolve fatal errors to their underlying cause. `ERR__FATAL` is only a generic
|
|
37
|
+
# marker; the real error code and description must be fetched from librdkafka via
|
|
38
|
+
# `rd_kafka_fatal_error` (done by `RdkafkaError.build_fatal`). Without this the
|
|
39
|
+
# callback would report the generic fatal code instead of the actual error.
|
|
40
|
+
# Mirrors the upstream `ErrorCallback`.
|
|
41
|
+
error = if err_code == ::Rdkafka::Bindings::RD_KAFKA_RESP_ERR__FATAL
|
|
42
|
+
::Rdkafka::RdkafkaError.build_fatal(
|
|
43
|
+
client_ptr,
|
|
44
|
+
fallback_error_code: err_code,
|
|
45
|
+
fallback_message: reason,
|
|
46
|
+
instance_name: name
|
|
47
|
+
)
|
|
48
|
+
else
|
|
49
|
+
::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
|
50
|
+
end
|
|
31
51
|
|
|
32
|
-
error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
|
33
52
|
error.set_backtrace(caller)
|
|
34
53
|
|
|
35
54
|
::Rdkafka::Config.error_callback.call(name, error)
|
|
@@ -10,6 +10,7 @@ module Karafka
|
|
|
10
10
|
# Creates new tags accumulator
|
|
11
11
|
def initialize
|
|
12
12
|
@accu = {}
|
|
13
|
+
@values_cache = nil
|
|
13
14
|
end
|
|
14
15
|
|
|
15
16
|
# Adds a tag with a given name to tags
|
|
@@ -17,22 +18,25 @@ module Karafka
|
|
|
17
18
|
# @param tag [#to_s] any object that can be converted into a string via `#to_s`
|
|
18
19
|
def add(name, tag)
|
|
19
20
|
@accu[name] = tag.to_s
|
|
21
|
+
@values_cache = nil
|
|
20
22
|
end
|
|
21
23
|
|
|
22
24
|
# Removes all the tags
|
|
23
25
|
def clear
|
|
24
26
|
@accu.clear
|
|
27
|
+
@values_cache = nil
|
|
25
28
|
end
|
|
26
29
|
|
|
27
30
|
# Removes a tag with a given name
|
|
28
31
|
# @param name [Symbol] name of the tag
|
|
29
32
|
def delete(name)
|
|
30
33
|
@accu.delete(name)
|
|
34
|
+
@values_cache = nil
|
|
31
35
|
end
|
|
32
36
|
|
|
33
37
|
# @return [Array<String>] all unique tags registered
|
|
34
38
|
def to_a
|
|
35
|
-
@accu.values.
|
|
39
|
+
@values_cache ||= @accu.values.uniq
|
|
36
40
|
end
|
|
37
41
|
|
|
38
42
|
# @param _args [Object] anything that the standard `to_json` accepts
|
data/lib/karafka/core/version.rb
CHANGED
data/lib/karafka-core.rb
CHANGED
data/renovate.json
CHANGED