karafka-core 2.0.3 → 2.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +3 -3
- data/.ruby-version +1 -1
- data/CHANGELOG.md +9 -0
- data/Gemfile.lock +21 -13
- data/karafka-core.gemspec +7 -0
- data/lib/karafka/core/instrumentation/callbacks_manager.rb +42 -0
- data/lib/karafka/core/instrumentation.rb +22 -0
- data/lib/karafka/core/monitoring/notifications.rb +9 -3
- data/lib/karafka/core/patches/rdkafka/bindings.rb +44 -0
- data/lib/karafka/core/version.rb +1 -1
- data/lib/karafka-core.rb +20 -0
- data.tar.gz.sig +0 -0
- metadata +26 -3
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e11bdb1c5a745e22d10b63dff4b2fb0a7a30dd1812aa8f7d3ad555d061dd451
|
4
|
+
data.tar.gz: 27d163cb019a0b57109bb4e6d53f3adce889a1d315f708f4100ffcee4d4885f8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d838c690a75dd37b1a15b666df8eafbe38c5f8cd95a38db9d7350a0753220b1c168d9ab153284b2dc5947d920ea26b0056871879cb2ca8bc8fba65f09920304b
|
7
|
+
data.tar.gz: ec6a1bd9ba1734975d8bafb0d57009e87101de8821caf96d30aa14e358169c0042f9744e7b1db951da959d7bcae19e4a202ad6c2f997bf5e40d6a7cceb1f75c1
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data/.github/FUNDING.yml
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
custom: ['https://karafka.io/#become-pro']
|
data/.github/workflows/ci.yml
CHANGED
@@ -23,7 +23,7 @@ jobs:
|
|
23
23
|
- ruby: '3.1'
|
24
24
|
coverage: 'true'
|
25
25
|
steps:
|
26
|
-
- uses: actions/checkout@
|
26
|
+
- uses: actions/checkout@v3
|
27
27
|
- name: Install package dependencies
|
28
28
|
run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
|
29
29
|
- name: Set up Ruby
|
@@ -48,7 +48,7 @@ jobs:
|
|
48
48
|
strategy:
|
49
49
|
fail-fast: false
|
50
50
|
steps:
|
51
|
-
- uses: actions/checkout@
|
51
|
+
- uses: actions/checkout@v3
|
52
52
|
with:
|
53
53
|
fetch-depth: 0
|
54
54
|
- name: Set up Ruby
|
@@ -67,7 +67,7 @@ jobs:
|
|
67
67
|
strategy:
|
68
68
|
fail-fast: false
|
69
69
|
steps:
|
70
|
-
- uses: actions/checkout@
|
70
|
+
- uses: actions/checkout@v3
|
71
71
|
with:
|
72
72
|
fetch-depth: 0
|
73
73
|
- name: Run Coditsu
|
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
3.1.
|
1
|
+
3.1.3
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,14 @@
|
|
1
1
|
# Karafka core changelog
|
2
2
|
|
3
|
+
## 2.0.5 (2022-12-07)
|
4
|
+
- Move `librdkafka` generic (producer and consumer) patches from WaterDrop here.
|
5
|
+
- Move dependency on `librdkafka` here from both Karafka and WaterDrop to unify management.
|
6
|
+
- Move `CallbacksManager` from WaterDrop because it's shared.
|
7
|
+
|
8
|
+
## 2.0.4 (2022-11-20)
|
9
|
+
- Disallow publishing events that were not registered.
|
10
|
+
- Fix a potential race condition when adding listeners concurrently from multiple threads.
|
11
|
+
|
3
12
|
## 2.0.3 (2022-10-13)
|
4
13
|
- Maintenance release. Cert chain update. No code changes.
|
5
14
|
|
data/Gemfile.lock
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka-core (2.0.
|
4
|
+
karafka-core (2.0.5)
|
5
5
|
concurrent-ruby (>= 1.1)
|
6
|
+
rdkafka (>= 0.12)
|
6
7
|
|
7
8
|
GEM
|
8
9
|
remote: https://rubygems.org/
|
@@ -18,22 +19,29 @@ GEM
|
|
18
19
|
docile (1.4.0)
|
19
20
|
factory_bot (6.2.1)
|
20
21
|
activesupport (>= 5.0.0)
|
22
|
+
ffi (1.15.5)
|
21
23
|
i18n (1.12.0)
|
22
24
|
concurrent-ruby (~> 1.0)
|
25
|
+
mini_portile2 (2.8.0)
|
23
26
|
minitest (5.16.3)
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
27
|
+
rake (13.0.6)
|
28
|
+
rdkafka (0.12.0)
|
29
|
+
ffi (~> 1.15)
|
30
|
+
mini_portile2 (~> 2.6)
|
31
|
+
rake (> 12)
|
32
|
+
rspec (3.12.0)
|
33
|
+
rspec-core (~> 3.12.0)
|
34
|
+
rspec-expectations (~> 3.12.0)
|
35
|
+
rspec-mocks (~> 3.12.0)
|
36
|
+
rspec-core (3.12.0)
|
37
|
+
rspec-support (~> 3.12.0)
|
38
|
+
rspec-expectations (3.12.0)
|
31
39
|
diff-lcs (>= 1.2.0, < 2.0)
|
32
|
-
rspec-support (~> 3.
|
33
|
-
rspec-mocks (3.
|
40
|
+
rspec-support (~> 3.12.0)
|
41
|
+
rspec-mocks (3.12.0)
|
34
42
|
diff-lcs (>= 1.2.0, < 2.0)
|
35
|
-
rspec-support (~> 3.
|
36
|
-
rspec-support (3.
|
43
|
+
rspec-support (~> 3.12.0)
|
44
|
+
rspec-support (3.12.0)
|
37
45
|
simplecov (0.21.2)
|
38
46
|
docile (~> 1.1)
|
39
47
|
simplecov-html (~> 0.11)
|
@@ -55,4 +63,4 @@ DEPENDENCIES
|
|
55
63
|
simplecov
|
56
64
|
|
57
65
|
BUNDLED WITH
|
58
|
-
2.3.
|
66
|
+
2.3.26
|
data/karafka-core.gemspec
CHANGED
@@ -15,7 +15,9 @@ Gem::Specification.new do |spec|
|
|
15
15
|
spec.summary = 'Karafka ecosystem core modules'
|
16
16
|
spec.description = 'A toolset of small support modules used throughout the Karafka ecosystem'
|
17
17
|
spec.licenses = %w[MIT]
|
18
|
+
|
18
19
|
spec.add_dependency 'concurrent-ruby', '>= 1.1'
|
20
|
+
spec.add_dependency 'rdkafka', '>= 0.12'
|
19
21
|
|
20
22
|
spec.required_ruby_version = '>= 2.6.0'
|
21
23
|
|
@@ -28,7 +30,12 @@ Gem::Specification.new do |spec|
|
|
28
30
|
spec.require_paths = %w[lib]
|
29
31
|
|
30
32
|
spec.metadata = {
|
33
|
+
'funding_uri' => 'https://karafka.io/#become-pro',
|
34
|
+
'homepage_uri' => 'https://karafka.io',
|
35
|
+
'changelog_uri' => 'https://github.com/karafka/karafka-core/blob/master/CHANGELOG.md',
|
36
|
+
'bug_tracker_uri' => 'https://github.com/karafka/karafka-core/issues',
|
31
37
|
'source_code_uri' => 'https://github.com/karafka/karafka-core',
|
38
|
+
'documentation_uri' => 'https://karafka.io/docs',
|
32
39
|
'rubygems_mfa_required' => 'true'
|
33
40
|
}
|
34
41
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Core
|
5
|
+
# Karafka instrumentation related shared components
|
6
|
+
module Instrumentation
|
7
|
+
# This manager allows us to register multiple callbacks into a hook that is suppose to support
|
8
|
+
# a single callback
|
9
|
+
class CallbacksManager
|
10
|
+
# @return [::Karafka::Core::Instrumentation::CallbacksManager]
|
11
|
+
def initialize
|
12
|
+
@callbacks = Concurrent::Hash.new
|
13
|
+
end
|
14
|
+
|
15
|
+
# Invokes all the callbacks registered one after another
|
16
|
+
#
|
17
|
+
# @param args [Object] any args that should go to the callbacks
|
18
|
+
# @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
|
19
|
+
# callbacks and add new at the same time. Since we don't know when and in what thread
|
20
|
+
# things are going to be added to the manager, we need to extract values into an array and
|
21
|
+
# run it. That way we can add new things the same time.
|
22
|
+
def call(*args)
|
23
|
+
@callbacks.values.each { |callback| callback.call(*args) }
|
24
|
+
end
|
25
|
+
|
26
|
+
# Adds a callback to the manager
|
27
|
+
#
|
28
|
+
# @param id [String] id of the callback (used when deleting it)
|
29
|
+
# @param callable [#call] object that responds to a `#call` method
|
30
|
+
def add(id, callable)
|
31
|
+
@callbacks[id] = callable
|
32
|
+
end
|
33
|
+
|
34
|
+
# Removes the callback from the manager
|
35
|
+
# @param id [String] id of the callback we want to remove
|
36
|
+
def delete(id)
|
37
|
+
@callbacks.delete(id)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Core
|
5
|
+
# All the instrumentation shared across Karafka ecosystem
|
6
|
+
module Instrumentation
|
7
|
+
class << self
|
8
|
+
# Builds a manager for statistics callbacks
|
9
|
+
# @return [WaterDrop::CallbacksManager]
|
10
|
+
def statistics_callbacks
|
11
|
+
@statistics_callbacks ||= CallbacksManager.new
|
12
|
+
end
|
13
|
+
|
14
|
+
# Builds a manager for error callbacks
|
15
|
+
# @return [WaterDrop::CallbacksManager]
|
16
|
+
def error_callbacks
|
17
|
+
@error_callbacks ||= CallbacksManager.new
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -20,14 +20,17 @@ module Karafka
|
|
20
20
|
private_constant :EMPTY_HASH
|
21
21
|
|
22
22
|
def initialize
|
23
|
-
@listeners = Concurrent::Map.new
|
23
|
+
@listeners = Concurrent::Map.new do |k, v|
|
24
|
+
k.compute_if_absent(v) { Concurrent::Array.new }
|
25
|
+
end
|
26
|
+
|
24
27
|
# This allows us to optimize the method calling lookups
|
25
28
|
@events_methods_map = Concurrent::Map.new
|
26
29
|
end
|
27
30
|
|
28
31
|
# Registers a new event on which we can publish
|
29
32
|
#
|
30
|
-
# @param event_id [String
|
33
|
+
# @param event_id [String] event id
|
31
34
|
def register_event(event_id)
|
32
35
|
@listeners[event_id]
|
33
36
|
@events_methods_map[event_id] = :"on_#{event_id.to_s.tr('.', '_')}"
|
@@ -73,7 +76,7 @@ module Karafka
|
|
73
76
|
# Allows for code instrumentation
|
74
77
|
# Runs the provided code and sends the instrumentation details to all registered listeners
|
75
78
|
#
|
76
|
-
# @param event_id [String
|
79
|
+
# @param event_id [String] id of the event
|
77
80
|
# @param payload [Hash] payload for the instrumentation
|
78
81
|
# @param block [Proc] instrumented code
|
79
82
|
# @return [Object] whatever the provided block (if any) returns
|
@@ -83,6 +86,9 @@ module Karafka
|
|
83
86
|
# sleep(1)
|
84
87
|
# end
|
85
88
|
def instrument(event_id, payload = EMPTY_HASH, &block)
|
89
|
+
# Allow for instrumentation of only events we registered
|
90
|
+
raise EventNotRegistered, event_id unless @listeners.key?(event_id)
|
91
|
+
|
86
92
|
result, time = measure_time_taken(&block) if block_given?
|
87
93
|
|
88
94
|
event = Event.new(
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Core
|
5
|
+
# Patches to dependencies and components
|
6
|
+
module Patches
|
7
|
+
# Patches to rdkafka
|
8
|
+
module Rdkafka
|
9
|
+
# Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
|
10
|
+
# to work with in a bit different way than rdkafka itself
|
11
|
+
module Bindings
|
12
|
+
class << self
|
13
|
+
# Add extra methods that we need
|
14
|
+
# @param mod [::Rdkafka::Bindings] rdkafka bindings module
|
15
|
+
def included(mod)
|
16
|
+
mod.attach_function :rd_kafka_name, [:pointer], :string
|
17
|
+
|
18
|
+
# Default rdkafka setup for errors doest not propagate client details, thus it always
|
19
|
+
# publishes all the stuff for all rdkafka instances. We change that by providing
|
20
|
+
# function that fetches the instance name, allowing us to have better notifications
|
21
|
+
mod.send(:remove_const, :ErrorCallback)
|
22
|
+
mod.const_set(:ErrorCallback, build_error_callback)
|
23
|
+
end
|
24
|
+
|
25
|
+
# @return [FFI::Function] overwritten callback function
|
26
|
+
def build_error_callback
|
27
|
+
FFI::Function.new(
|
28
|
+
:void, %i[pointer int string pointer]
|
29
|
+
) do |client_prr, err_code, reason, _opaque|
|
30
|
+
return nil unless ::Rdkafka::Config.error_callback
|
31
|
+
|
32
|
+
name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
|
33
|
+
|
34
|
+
error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
35
|
+
|
36
|
+
::Rdkafka::Config.error_callback.call(name, error)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/karafka/core/version.rb
CHANGED
data/lib/karafka-core.rb
CHANGED
@@ -2,24 +2,44 @@
|
|
2
2
|
|
3
3
|
%w[
|
4
4
|
yaml
|
5
|
+
rdkafka
|
6
|
+
|
5
7
|
concurrent/map
|
6
8
|
concurrent/hash
|
7
9
|
concurrent/array
|
10
|
+
|
8
11
|
karafka/core
|
9
12
|
karafka/core/version
|
13
|
+
|
10
14
|
karafka/core/monitoring
|
11
15
|
karafka/core/monitoring/event
|
12
16
|
karafka/core/monitoring/monitor
|
13
17
|
karafka/core/monitoring/notifications
|
14
18
|
karafka/core/monitoring/statistics_decorator
|
19
|
+
|
15
20
|
karafka/core/configurable
|
16
21
|
karafka/core/configurable/leaf
|
17
22
|
karafka/core/configurable/node
|
23
|
+
|
18
24
|
karafka/core/contractable/contract
|
19
25
|
karafka/core/contractable/result
|
20
26
|
karafka/core/contractable/rule
|
27
|
+
|
28
|
+
karafka/core/instrumentation
|
29
|
+
karafka/core/instrumentation/callbacks_manager
|
30
|
+
|
31
|
+
karafka/core/patches/rdkafka/bindings
|
21
32
|
].each { |dependency| require dependency }
|
22
33
|
|
23
34
|
# Karafka framework main namespace
|
24
35
|
module Karafka
|
25
36
|
end
|
37
|
+
|
38
|
+
# Patch rdkafka
|
39
|
+
::Rdkafka::Bindings.include(::Karafka::Core::Patches::Rdkafka::Bindings)
|
40
|
+
|
41
|
+
# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
|
42
|
+
# each callback type. Callback manager allows us to register more than one callback
|
43
|
+
# @note Those managers are also used by Karafka for consumer related statistics
|
44
|
+
::Rdkafka::Config.statistics_callback = ::Karafka::Core::Instrumentation.statistics_callbacks
|
45
|
+
::Rdkafka::Config.error_callback = ::Karafka::Core::Instrumentation.error_callbacks
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka-core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -35,7 +35,7 @@ cert_chain:
|
|
35
35
|
Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
|
36
36
|
MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
|
37
37
|
-----END CERTIFICATE-----
|
38
|
-
date: 2022-
|
38
|
+
date: 2022-12-07 00:00:00.000000000 Z
|
39
39
|
dependencies:
|
40
40
|
- !ruby/object:Gem::Dependency
|
41
41
|
name: concurrent-ruby
|
@@ -51,6 +51,20 @@ dependencies:
|
|
51
51
|
- - ">="
|
52
52
|
- !ruby/object:Gem::Version
|
53
53
|
version: '1.1'
|
54
|
+
- !ruby/object:Gem::Dependency
|
55
|
+
name: rdkafka
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0.12'
|
61
|
+
type: :runtime
|
62
|
+
prerelease: false
|
63
|
+
version_requirements: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0.12'
|
54
68
|
description: A toolset of small support modules used throughout the Karafka ecosystem
|
55
69
|
email:
|
56
70
|
- contact@karafka.io
|
@@ -61,6 +75,7 @@ files:
|
|
61
75
|
- ".coditsu/ci.yml"
|
62
76
|
- ".console_irbrc"
|
63
77
|
- ".diffend.yml"
|
78
|
+
- ".github/FUNDING.yml"
|
64
79
|
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
65
80
|
- ".github/ISSUE_TEMPLATE/feature_request.md"
|
66
81
|
- ".github/workflows/ci.yml"
|
@@ -86,18 +101,26 @@ files:
|
|
86
101
|
- lib/karafka/core/contractable/contract.rb
|
87
102
|
- lib/karafka/core/contractable/result.rb
|
88
103
|
- lib/karafka/core/contractable/rule.rb
|
104
|
+
- lib/karafka/core/instrumentation.rb
|
105
|
+
- lib/karafka/core/instrumentation/callbacks_manager.rb
|
89
106
|
- lib/karafka/core/monitoring.rb
|
90
107
|
- lib/karafka/core/monitoring/event.rb
|
91
108
|
- lib/karafka/core/monitoring/monitor.rb
|
92
109
|
- lib/karafka/core/monitoring/notifications.rb
|
93
110
|
- lib/karafka/core/monitoring/statistics_decorator.rb
|
111
|
+
- lib/karafka/core/patches/rdkafka/bindings.rb
|
94
112
|
- lib/karafka/core/version.rb
|
95
113
|
- log/.gitkeep
|
96
114
|
homepage: https://karafka.io
|
97
115
|
licenses:
|
98
116
|
- MIT
|
99
117
|
metadata:
|
118
|
+
funding_uri: https://karafka.io/#become-pro
|
119
|
+
homepage_uri: https://karafka.io
|
120
|
+
changelog_uri: https://github.com/karafka/karafka-core/blob/master/CHANGELOG.md
|
121
|
+
bug_tracker_uri: https://github.com/karafka/karafka-core/issues
|
100
122
|
source_code_uri: https://github.com/karafka/karafka-core
|
123
|
+
documentation_uri: https://karafka.io/docs
|
101
124
|
rubygems_mfa_required: 'true'
|
102
125
|
post_install_message:
|
103
126
|
rdoc_options: []
|
@@ -114,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
114
137
|
- !ruby/object:Gem::Version
|
115
138
|
version: '0'
|
116
139
|
requirements: []
|
117
|
-
rubygems_version: 3.3.
|
140
|
+
rubygems_version: 3.3.26
|
118
141
|
signing_key:
|
119
142
|
specification_version: 4
|
120
143
|
summary: Karafka ecosystem core modules
|
metadata.gz.sig
CHANGED
Binary file
|