karafka 1.3.0.rc1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.travis.yml +0 -2
- data/CHANGELOG.md +5 -1
- data/Gemfile +0 -3
- data/Gemfile.lock +12 -13
- data/README.md +1 -1
- data/karafka.gemspec +2 -2
- data/lib/karafka/attributes_map.rb +1 -1
- data/lib/karafka/code_reloader.rb +1 -1
- data/lib/karafka/connection/batch_delegator.rb +1 -1
- data/lib/karafka/connection/client.rb +2 -2
- data/lib/karafka/connection/message_delegator.rb +1 -1
- data/lib/karafka/instrumentation/stdout_listener.rb +2 -2
- data/lib/karafka/persistence/consumers.rb +1 -1
- data/lib/karafka/routing/builder.rb +2 -2
- data/lib/karafka/version.rb +1 -1
- metadata +8 -8
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ffe10d1ca48b0b218191231ec6969f3a767f5a1a4010dca0ef00fa44c2eaab67
|
|
4
|
+
data.tar.gz: 27d1b52ba3782b562176b7a6de73563bdd48c250bc0576a1821c55bfcec07bff
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 85bdedfe0791c7d17abbc54ed49df83e2c9cbfb048124107d403559e72b2884909bcc5974c98a738744f80184315b2ee31cc8bd1048cc5a10230e6b6f0184e3d
|
|
7
|
+
data.tar.gz: 382074bee041ed27776571816a982fbd7f0af6b6f9cfa8fa393113acf1c2d4cbfb0a7573d2bcd16f2662667d1b8175546e281deb5a7b66ce30b5d9c4c19d005d
|
checksums.yaml.gz.sig
CHANGED
|
Binary file
|
data.tar.gz.sig
CHANGED
|
Binary file
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
# Karafka framework changelog
|
|
2
2
|
|
|
3
|
-
## 1.3.0
|
|
3
|
+
## 1.3.0 (2019-09-09)
|
|
4
|
+
- Drop support for Ruby 2.4
|
|
5
|
+
- YARD docs tags cleanup
|
|
6
|
+
|
|
7
|
+
## 1.3.0.rc1 (2019-07-31)
|
|
4
8
|
- Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
|
|
5
9
|
- Update ruby-kafka to the 0.7 version
|
|
6
10
|
- Support messages headers receiving
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,28 +1,29 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
karafka (1.3.0
|
|
4
|
+
karafka (1.3.0)
|
|
5
5
|
dry-configurable (~> 0.8)
|
|
6
6
|
dry-inflector (~> 0.1)
|
|
7
7
|
dry-monitor (~> 0.3)
|
|
8
8
|
dry-validation (~> 1.2)
|
|
9
|
-
envlogic (~> 1.
|
|
9
|
+
envlogic (~> 1.1)
|
|
10
10
|
irb (~> 1.0)
|
|
11
11
|
multi_json (>= 1.12)
|
|
12
12
|
rake (>= 11.3)
|
|
13
13
|
ruby-kafka (>= 0.7.8)
|
|
14
14
|
thor (~> 0.20)
|
|
15
|
-
waterdrop (~> 1.3.0
|
|
15
|
+
waterdrop (~> 1.3.0)
|
|
16
16
|
zeitwerk (~> 2.1)
|
|
17
17
|
|
|
18
18
|
GEM
|
|
19
19
|
remote: https://rubygems.org/
|
|
20
20
|
specs:
|
|
21
|
-
activesupport (
|
|
21
|
+
activesupport (6.0.0)
|
|
22
22
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
23
23
|
i18n (>= 0.7, < 2)
|
|
24
24
|
minitest (~> 5.1)
|
|
25
25
|
tzinfo (~> 1.1)
|
|
26
|
+
zeitwerk (~> 2.1, >= 2.1.8)
|
|
26
27
|
byebug (11.0.1)
|
|
27
28
|
concurrent-ruby (1.1.5)
|
|
28
29
|
delivery_boy (0.2.8)
|
|
@@ -37,7 +38,7 @@ GEM
|
|
|
37
38
|
dry-container (0.7.2)
|
|
38
39
|
concurrent-ruby (~> 1.0)
|
|
39
40
|
dry-configurable (~> 0.1, >= 0.1.3)
|
|
40
|
-
dry-core (0.4.
|
|
41
|
+
dry-core (0.4.9)
|
|
41
42
|
concurrent-ruby (~> 1.0)
|
|
42
43
|
dry-equalizer (0.2.2)
|
|
43
44
|
dry-events (0.2.0)
|
|
@@ -46,7 +47,7 @@ GEM
|
|
|
46
47
|
dry-equalizer (~> 0.2)
|
|
47
48
|
dry-inflector (0.1.2)
|
|
48
49
|
dry-initializer (3.0.1)
|
|
49
|
-
dry-logic (1.0.
|
|
50
|
+
dry-logic (1.0.3)
|
|
50
51
|
concurrent-ruby (~> 1.0)
|
|
51
52
|
dry-core (~> 0.2)
|
|
52
53
|
dry-equalizer (~> 0.2)
|
|
@@ -55,7 +56,7 @@ GEM
|
|
|
55
56
|
dry-core (~> 0.4)
|
|
56
57
|
dry-equalizer (~> 0.2)
|
|
57
58
|
dry-events (~> 0.1)
|
|
58
|
-
dry-schema (1.3.
|
|
59
|
+
dry-schema (1.3.3)
|
|
59
60
|
concurrent-ruby (~> 1.0)
|
|
60
61
|
dry-configurable (~> 0.8, >= 0.8.3)
|
|
61
62
|
dry-core (~> 0.4)
|
|
@@ -70,7 +71,7 @@ GEM
|
|
|
70
71
|
dry-equalizer (~> 0.2, >= 0.2.2)
|
|
71
72
|
dry-inflector (~> 0.1, >= 0.1.2)
|
|
72
73
|
dry-logic (~> 1.0, >= 1.0.2)
|
|
73
|
-
dry-validation (1.
|
|
74
|
+
dry-validation (1.3.1)
|
|
74
75
|
concurrent-ruby (~> 1.0)
|
|
75
76
|
dry-container (~> 0.7, >= 0.7.1)
|
|
76
77
|
dry-core (~> 0.4)
|
|
@@ -102,7 +103,7 @@ GEM
|
|
|
102
103
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
103
104
|
rspec-support (~> 3.8.0)
|
|
104
105
|
rspec-support (3.8.2)
|
|
105
|
-
ruby-kafka (0.7.
|
|
106
|
+
ruby-kafka (0.7.10)
|
|
106
107
|
digest-crc
|
|
107
108
|
simplecov (0.17.0)
|
|
108
109
|
docile (~> 1.1)
|
|
@@ -113,26 +114,24 @@ GEM
|
|
|
113
114
|
thread_safe (0.3.6)
|
|
114
115
|
tzinfo (1.2.5)
|
|
115
116
|
thread_safe (~> 0.1)
|
|
116
|
-
waterdrop (1.3.0
|
|
117
|
+
waterdrop (1.3.0)
|
|
117
118
|
delivery_boy (~> 0.2)
|
|
118
119
|
dry-configurable (~> 0.8)
|
|
119
120
|
dry-monitor (~> 0.3)
|
|
120
121
|
dry-validation (~> 1.2)
|
|
121
122
|
ruby-kafka (>= 0.7.8)
|
|
122
123
|
zeitwerk (~> 2.1)
|
|
123
|
-
zeitwerk (2.1.
|
|
124
|
+
zeitwerk (2.1.10)
|
|
124
125
|
|
|
125
126
|
PLATFORMS
|
|
126
127
|
ruby
|
|
127
128
|
|
|
128
129
|
DEPENDENCIES
|
|
129
|
-
activesupport
|
|
130
130
|
byebug
|
|
131
131
|
factory_bot
|
|
132
132
|
karafka!
|
|
133
133
|
rspec
|
|
134
134
|
simplecov
|
|
135
|
-
waterdrop (= 1.3.0.rc1)
|
|
136
135
|
|
|
137
136
|
BUNDLED WITH
|
|
138
137
|
2.0.2
|
data/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
## New release in progress!
|
|
6
6
|
|
|
7
|
-
**Note**: Documentation presented here refers to
|
|
7
|
+
**Note**: Documentation presented here refers to Karafka `1.3.0`.
|
|
8
8
|
|
|
9
9
|
If you are looking for the documentation for Karafka `1.2.*`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
|
|
10
10
|
|
data/karafka.gemspec
CHANGED
|
@@ -21,13 +21,13 @@ Gem::Specification.new do |spec|
|
|
|
21
21
|
spec.add_dependency 'dry-inflector', '~> 0.1'
|
|
22
22
|
spec.add_dependency 'dry-monitor', '~> 0.3'
|
|
23
23
|
spec.add_dependency 'dry-validation', '~> 1.2'
|
|
24
|
-
spec.add_dependency 'envlogic', '~> 1.
|
|
24
|
+
spec.add_dependency 'envlogic', '~> 1.1'
|
|
25
25
|
spec.add_dependency 'irb', '~> 1.0'
|
|
26
26
|
spec.add_dependency 'multi_json', '>= 1.12'
|
|
27
27
|
spec.add_dependency 'rake', '>= 11.3'
|
|
28
28
|
spec.add_dependency 'ruby-kafka', '>= 0.7.8'
|
|
29
29
|
spec.add_dependency 'thor', '~> 0.20'
|
|
30
|
-
spec.add_dependency 'waterdrop', '~> 1.3.0
|
|
30
|
+
spec.add_dependency 'waterdrop', '~> 1.3.0'
|
|
31
31
|
spec.add_dependency 'zeitwerk', '~> 2.1'
|
|
32
32
|
|
|
33
33
|
spec.required_ruby_version = '>= 2.4.0'
|
|
@@ -11,9 +11,9 @@ module Karafka
|
|
|
11
11
|
module AttributesMap
|
|
12
12
|
class << self
|
|
13
13
|
# What settings should go where in ruby-kafka
|
|
14
|
+
# @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
|
|
14
15
|
# @note All other settings will be passed to Kafka.new method invocation.
|
|
15
16
|
# All elements in this hash are just edge cases
|
|
16
|
-
# @return [Hash] hash with proper sections on what to proxy where in Ruby-Kafka
|
|
17
17
|
def api_adapter
|
|
18
18
|
{
|
|
19
19
|
consumer: %i[
|
|
@@ -24,10 +24,10 @@ module Karafka
|
|
|
24
24
|
end
|
|
25
25
|
|
|
26
26
|
# Binds to the instrumentation events and triggers reload
|
|
27
|
+
# @param _event [Dry::Event] empty dry event
|
|
27
28
|
# @note Since we de-register all the user defined objects and redraw routes, it means that
|
|
28
29
|
# we won't be able to do a multi-batch buffering in the development mode as each of the
|
|
29
30
|
# batches will be buffered on a newly created "per fetch" instance.
|
|
30
|
-
# @param _event [Dry::Event] empty dry event
|
|
31
31
|
def on_connection_listener_fetch_loop(_event)
|
|
32
32
|
reload
|
|
33
33
|
end
|
|
@@ -8,9 +8,9 @@ module Karafka
|
|
|
8
8
|
class << self
|
|
9
9
|
# Delegates messages (does something with them)
|
|
10
10
|
# It will either schedule or run a proper processor action for messages
|
|
11
|
-
# @note This should be looped to obtain a constant delegating of new messages
|
|
12
11
|
# @param group_id [String] group_id of a group from which a given message came
|
|
13
12
|
# @param kafka_batch [<Kafka::FetchedBatch>] raw messages fetched batch
|
|
13
|
+
# @note This should be looped to obtain a constant delegating of new messages
|
|
14
14
|
def call(group_id, kafka_batch)
|
|
15
15
|
topic = Persistence::Topics.fetch(group_id, kafka_batch.topic)
|
|
16
16
|
consumer = Persistence::Consumers.fetch(topic, kafka_batch.partition)
|
|
@@ -68,9 +68,9 @@ module Karafka
|
|
|
68
68
|
end
|
|
69
69
|
|
|
70
70
|
# Marks given message as consumed
|
|
71
|
+
# @param [Karafka::Params::Params] params message that we want to mark as processed
|
|
71
72
|
# @note This method won't trigger automatic offsets commits, rather relying on the ruby-kafka
|
|
72
73
|
# offsets time-interval based committing
|
|
73
|
-
# @param [Karafka::Params::Params] params message that we want to mark as processed
|
|
74
74
|
def mark_as_consumed(params)
|
|
75
75
|
kafka_consumer.mark_message_as_processed(
|
|
76
76
|
*ApiAdapter.mark_message_as_processed(params)
|
|
@@ -78,9 +78,9 @@ module Karafka
|
|
|
78
78
|
end
|
|
79
79
|
|
|
80
80
|
# Marks a given message as consumed and commit the offsets in a blocking way
|
|
81
|
+
# @param [Karafka::Params::Params] params message that we want to mark as processed
|
|
81
82
|
# @note This method commits the offset for each manual marking to be sure
|
|
82
83
|
# that offset commit happen asap in case of a crash
|
|
83
|
-
# @param [Karafka::Params::Params] params message that we want to mark as processed
|
|
84
84
|
def mark_as_consumed!(params)
|
|
85
85
|
mark_as_consumed(params)
|
|
86
86
|
# Trigger an immediate, blocking offset commit in order to minimize the risk of crashing
|
|
@@ -8,9 +8,9 @@ module Karafka
|
|
|
8
8
|
class << self
|
|
9
9
|
# Delegates message (does something with it)
|
|
10
10
|
# It will either schedule or run a proper processor action for the incoming message
|
|
11
|
-
# @note This should be looped to obtain a constant delegating of new messages
|
|
12
11
|
# @param group_id [String] group_id of a group from which a given message came
|
|
13
12
|
# @param kafka_message [<Kafka::FetchedMessage>] raw message from kafka
|
|
13
|
+
# @note This should be looped to obtain a constant delegating of new messages
|
|
14
14
|
def call(group_id, kafka_message)
|
|
15
15
|
topic = Persistence::Topics.fetch(group_id, kafka_message.topic)
|
|
16
16
|
consumer = Persistence::Consumers.fetch(topic, kafka_message.partition)
|
|
@@ -63,16 +63,16 @@ module Karafka
|
|
|
63
63
|
end
|
|
64
64
|
|
|
65
65
|
# Logs errors that are related to the connection itself
|
|
66
|
-
# @note Karafka will attempt to reconnect, so an error not a fatal
|
|
67
66
|
# @param event [Dry::Events::Event] event details including payload
|
|
67
|
+
# @note Karafka will attempt to reconnect, so an error not a fatal
|
|
68
68
|
def on_connection_client_fetch_loop_error(event)
|
|
69
69
|
error "Client fetch loop error: #{event[:error]}"
|
|
70
70
|
end
|
|
71
71
|
|
|
72
72
|
# Logs info about crashed fetcher
|
|
73
|
+
# @param event [Dry::Events::Event] event details including payload
|
|
73
74
|
# @note If this happens, Karafka will shutdown as it means a critical error
|
|
74
75
|
# in one of the threads
|
|
75
|
-
# @param event [Dry::Events::Event] event details including payload
|
|
76
76
|
def on_fetcher_call_error(event)
|
|
77
77
|
fatal "Fetcher crash due to an error: #{event[:error]}"
|
|
78
78
|
end
|
|
@@ -23,9 +23,9 @@ module Karafka
|
|
|
23
23
|
|
|
24
24
|
# Used to build (if block given) and/or fetch a current consumer instance that will be
|
|
25
25
|
# used to process messages from a given topic and partition
|
|
26
|
-
# @return [Karafka::BaseConsumer] base consumer descendant
|
|
27
26
|
# @param topic [Karafka::Routing::Topic] topic instance for which we might cache
|
|
28
27
|
# @param partition [Integer] number of partition for which we want to cache
|
|
28
|
+
# @return [Karafka::BaseConsumer] base consumer descendant
|
|
29
29
|
def fetch(topic, partition)
|
|
30
30
|
current[topic][partition] ||= topic.consumer.new(topic)
|
|
31
31
|
end
|
|
@@ -20,12 +20,12 @@ module Karafka
|
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
# Used to draw routes for Karafka
|
|
23
|
-
# @note After it is done drawing it will store and validate all the routes to make sure that
|
|
24
|
-
# they are correct and that there are no topic/group duplications (this is forbidden)
|
|
25
23
|
# @param block [Proc] block we will evaluate within the builder context
|
|
26
24
|
# @yield Evaluates provided block in a builder context so we can describe routes
|
|
27
25
|
# @raise [Karafka::Errors::InvalidConfigurationError] raised when configuration
|
|
28
26
|
# doesn't match with the config contract
|
|
27
|
+
# @note After it is done drawing it will store and validate all the routes to make sure that
|
|
28
|
+
# they are correct and that there are no topic/group duplications (this is forbidden)
|
|
29
29
|
# @example
|
|
30
30
|
# draw do
|
|
31
31
|
# topic :xyz do
|
data/lib/karafka/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: karafka
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.3.0
|
|
4
|
+
version: 1.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Maciej Mensfeld
|
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
|
36
36
|
KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
|
|
37
37
|
nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
|
|
38
38
|
-----END CERTIFICATE-----
|
|
39
|
-
date: 2019-
|
|
39
|
+
date: 2019-09-09 00:00:00.000000000 Z
|
|
40
40
|
dependencies:
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
42
|
name: dry-configurable
|
|
@@ -100,14 +100,14 @@ dependencies:
|
|
|
100
100
|
requirements:
|
|
101
101
|
- - "~>"
|
|
102
102
|
- !ruby/object:Gem::Version
|
|
103
|
-
version: '1.
|
|
103
|
+
version: '1.1'
|
|
104
104
|
type: :runtime
|
|
105
105
|
prerelease: false
|
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
|
107
107
|
requirements:
|
|
108
108
|
- - "~>"
|
|
109
109
|
- !ruby/object:Gem::Version
|
|
110
|
-
version: '1.
|
|
110
|
+
version: '1.1'
|
|
111
111
|
- !ruby/object:Gem::Dependency
|
|
112
112
|
name: irb
|
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -184,14 +184,14 @@ dependencies:
|
|
|
184
184
|
requirements:
|
|
185
185
|
- - "~>"
|
|
186
186
|
- !ruby/object:Gem::Version
|
|
187
|
-
version: 1.3.0
|
|
187
|
+
version: 1.3.0
|
|
188
188
|
type: :runtime
|
|
189
189
|
prerelease: false
|
|
190
190
|
version_requirements: !ruby/object:Gem::Requirement
|
|
191
191
|
requirements:
|
|
192
192
|
- - "~>"
|
|
193
193
|
- !ruby/object:Gem::Version
|
|
194
|
-
version: 1.3.0
|
|
194
|
+
version: 1.3.0
|
|
195
195
|
- !ruby/object:Gem::Dependency
|
|
196
196
|
name: zeitwerk
|
|
197
197
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -325,9 +325,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
325
325
|
version: 2.4.0
|
|
326
326
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
327
327
|
requirements:
|
|
328
|
-
- - "
|
|
328
|
+
- - ">="
|
|
329
329
|
- !ruby/object:Gem::Version
|
|
330
|
-
version:
|
|
330
|
+
version: '0'
|
|
331
331
|
requirements: []
|
|
332
332
|
rubygems_version: 3.0.3
|
|
333
333
|
signing_key:
|
metadata.gz.sig
CHANGED
|
Binary file
|