karafka 0.6.0.rc2 → 1.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -2
- data/Gemfile.lock +4 -18
- data/karafka.gemspec +0 -1
- data/lib/karafka.rb +2 -12
- data/lib/karafka/attributes_map.rb +2 -3
- data/lib/karafka/backends/inline.rb +17 -0
- data/lib/karafka/base_controller.rb +40 -96
- data/lib/karafka/base_responder.rb +19 -19
- data/lib/karafka/cli/info.rb +2 -3
- data/lib/karafka/cli/install.rb +0 -3
- data/lib/karafka/connection/messages_processor.rb +10 -6
- data/lib/karafka/controllers/includer.rb +51 -0
- data/lib/karafka/controllers/responders.rb +19 -0
- data/lib/karafka/controllers/single_params.rb +15 -0
- data/lib/karafka/errors.rb +1 -17
- data/lib/karafka/fetcher.rb +2 -2
- data/lib/karafka/helpers/class_matcher.rb +9 -10
- data/lib/karafka/params/params.rb +2 -2
- data/lib/karafka/params/params_batch.rb +2 -7
- data/lib/karafka/persistence.rb +18 -0
- data/lib/karafka/routing/builder.rb +1 -1
- data/lib/karafka/routing/router.rb +3 -11
- data/lib/karafka/routing/topic.rb +1 -13
- data/lib/karafka/schemas/config.rb +1 -12
- data/lib/karafka/schemas/consumer_group.rb +2 -2
- data/lib/karafka/setup/config.rb +14 -19
- data/lib/karafka/templates/karafka.rb.example +1 -5
- data/lib/karafka/version.rb +1 -1
- metadata +8 -24
- data/lib/karafka/base_worker.rb +0 -26
- data/lib/karafka/cli/worker.rb +0 -28
- data/lib/karafka/params/interchanger.rb +0 -35
- data/lib/karafka/setup/configurators/sidekiq.rb +0 -36
- data/lib/karafka/templates/application_worker.rb.example +0 -8
- data/lib/karafka/templates/sidekiq.yml.example +0 -26
- data/lib/karafka/workers/builder.rb +0 -51
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dff71b9105c2f08a83acaf4938b061935ccebba7
|
4
|
+
data.tar.gz: 7cc659012bbffedeb63e7195d21dafb780118dbf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3bf047a983f77817ea41240455e4ec8a00d342ee786a1df5770a2c1d80ca8fe5c366308368e4fe6f537b75c54f5512fb5afdac074719ad141ea0495b8247c5ef
|
7
|
+
data.tar.gz: '09020aca3024ed6abee2e63466527acbf4c08d0aff3ef80d4a86169bc14b2afb373d06bcbd1035905959d732af2928abe3281fb9c10808c201bf3d0f204134db'
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
-
## 0.
|
3
|
+
## 1.0.0.rc1
|
4
4
|
|
5
5
|
### Closed issues:
|
6
6
|
|
@@ -34,6 +34,10 @@
|
|
34
34
|
- (optional) pausing upon processing failures ```pause_timeout```
|
35
35
|
- Karafka console main process no longer intercepts irb errors
|
36
36
|
- Wiki updates
|
37
|
+
- #204 - Long running controllers
|
38
|
+
- Better internal API to handle multiple usage cases using ```Karafka::Controllers::Includer```
|
39
|
+
- #207 - Rename before_enqueued to after_received
|
40
|
+
- #147 - Deattach Karafka from Sidekiq by extracting Sidekiq backend
|
37
41
|
|
38
42
|
### New features and improvements
|
39
43
|
|
@@ -44,6 +48,7 @@
|
|
44
48
|
- Introduced the ```#batch_processing``` config flag (config for #126) that can be set per each consumer_group
|
45
49
|
- Added support for partition, offset and partition key in the params hash
|
46
50
|
- ```name``` option in config renamed to ```client_id```
|
51
|
+
- Long running controllers with ```persistent``` flag on a topic config level, to make controller instances persistent between messages batches (single controller instance per topic per partition no per messages batch) - turned on by default
|
47
52
|
|
48
53
|
### Incompatibilities
|
49
54
|
|
@@ -57,7 +62,10 @@
|
|
57
62
|
- Renamed content to value to better resemble ruby-kafka internal messages naming convention
|
58
63
|
- When having a responder with ```required``` topics and not using ```#respond_with``` at all, it will raise an exception
|
59
64
|
- Renamed ```inline_mode``` to ```inline_processing``` to resemble other settings conventions
|
60
|
-
- Renamed ```inline_processing``` to ```
|
65
|
+
- Renamed ```inline_processing``` to ```backend``` to reach 1.0 future compatibility
|
66
|
+
- Single controller **needs** to be used for a single topic consumption
|
67
|
+
- Renamed ```before_enqueue``` to ```after_received``` to better resemble internal logic, since for inline backend, there is no enqueue.
|
68
|
+
- Due to the level on which topic and controller are related (class level), the dynamic worker selection is no longer available.
|
61
69
|
|
62
70
|
### Other changes
|
63
71
|
- PolishGeeksDevTools removed (in favour of Coditsu)
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (0.
|
4
|
+
karafka (1.0.0.rc1)
|
5
5
|
activesupport (>= 5.0)
|
6
6
|
celluloid
|
7
7
|
dry-configurable (~> 0.7)
|
@@ -11,7 +11,6 @@ PATH
|
|
11
11
|
rake (>= 11.3)
|
12
12
|
require_all (>= 1.4)
|
13
13
|
ruby-kafka (>= 0.4)
|
14
|
-
sidekiq (>= 4.2)
|
15
14
|
thor (~> 0.19)
|
16
15
|
waterdrop (>= 0.4)
|
17
16
|
|
@@ -49,7 +48,7 @@ GEM
|
|
49
48
|
dry-container (0.6.0)
|
50
49
|
concurrent-ruby (~> 1.0)
|
51
50
|
dry-configurable (~> 0.1, >= 0.1.3)
|
52
|
-
dry-core (0.3.
|
51
|
+
dry-core (0.3.3)
|
53
52
|
concurrent-ruby (~> 1.0)
|
54
53
|
dry-equalizer (0.2.0)
|
55
54
|
dry-logic (0.4.1)
|
@@ -71,11 +70,8 @@ GEM
|
|
71
70
|
dry-equalizer (~> 0.2)
|
72
71
|
dry-logic (~> 0.4, >= 0.4.0)
|
73
72
|
dry-types (~> 0.11.0)
|
74
|
-
envlogic (1.0.
|
73
|
+
envlogic (1.0.4)
|
75
74
|
activesupport
|
76
|
-
ffi (1.9.18)
|
77
|
-
gssapi (1.2.0)
|
78
|
-
ffi (>= 1.0.1)
|
79
75
|
hitimes (1.2.6)
|
80
76
|
i18n (0.8.6)
|
81
77
|
inflecto (0.0.2)
|
@@ -83,11 +79,7 @@ GEM
|
|
83
79
|
minitest (5.10.3)
|
84
80
|
multi_json (1.12.1)
|
85
81
|
null-logger (0.1.4)
|
86
|
-
rack (2.0.3)
|
87
|
-
rack-protection (2.0.0)
|
88
|
-
rack
|
89
82
|
rake (12.0.0)
|
90
|
-
redis (3.3.3)
|
91
83
|
require_all (1.4.0)
|
92
84
|
rspec (3.6.0)
|
93
85
|
rspec-core (~> 3.6.0)
|
@@ -102,13 +94,7 @@ GEM
|
|
102
94
|
diff-lcs (>= 1.2.0, < 2.0)
|
103
95
|
rspec-support (~> 3.6.0)
|
104
96
|
rspec-support (3.6.0)
|
105
|
-
ruby-kafka (0.4.
|
106
|
-
gssapi (>= 1.2.0)
|
107
|
-
sidekiq (5.0.4)
|
108
|
-
concurrent-ruby (~> 1.0)
|
109
|
-
connection_pool (~> 2.2, >= 2.2.0)
|
110
|
-
rack-protection (>= 1.5.0)
|
111
|
-
redis (~> 3.3, >= 3.3.3)
|
97
|
+
ruby-kafka (0.4.1)
|
112
98
|
simplecov (0.15.0)
|
113
99
|
docile (~> 1.1.0)
|
114
100
|
json (>= 1.8, < 3)
|
data/karafka.gemspec
CHANGED
@@ -17,7 +17,6 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.license = 'MIT'
|
18
18
|
|
19
19
|
spec.add_dependency 'ruby-kafka', '>= 0.4'
|
20
|
-
spec.add_dependency 'sidekiq', '>= 4.2'
|
21
20
|
spec.add_dependency 'celluloid'
|
22
21
|
spec.add_dependency 'envlogic', '~> 1.0'
|
23
22
|
spec.add_dependency 'waterdrop', '>= 0.4'
|
data/lib/karafka.rb
CHANGED
@@ -1,18 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
%w[
|
4
|
-
rake
|
5
|
-
ostruct
|
6
|
-
rubygems
|
7
|
-
bundler
|
8
4
|
English
|
5
|
+
bundler
|
9
6
|
celluloid/current
|
10
7
|
waterdrop
|
11
|
-
pathname
|
12
|
-
timeout
|
13
|
-
logger
|
14
8
|
kafka
|
15
|
-
sidekiq
|
16
9
|
envlogic
|
17
10
|
thor
|
18
11
|
fileutils
|
@@ -26,9 +19,6 @@
|
|
26
19
|
active_support/descendants_tracker
|
27
20
|
active_support/inflector
|
28
21
|
karafka/loader
|
29
|
-
karafka/setup/config
|
30
|
-
karafka/status
|
31
|
-
karafka/routing/router
|
32
22
|
].each(&method(:require))
|
33
23
|
|
34
24
|
# Karafka library
|
@@ -64,7 +54,7 @@ module Karafka
|
|
64
54
|
# @return [String] path to a default file that contains booting procedure etc
|
65
55
|
# @note By default it is a file called 'karafka.rb' but it can be specified as you wish if you
|
66
56
|
# have Karafka that is merged into a Sinatra/Rails app and karafka.rb is taken.
|
67
|
-
# It will be used for console/
|
57
|
+
# It will be used for console/controllers/etc
|
68
58
|
# @example Standard only-Karafka case
|
69
59
|
# Karafka.boot_file #=> '/home/app_path/karafka.rb'
|
70
60
|
# @example Non standard case
|
@@ -33,13 +33,12 @@ module Karafka
|
|
33
33
|
# @return [Array<Symbol>] properties that can be set on a per topic level
|
34
34
|
def topic
|
35
35
|
(config_adapter[:subscription] + %i[
|
36
|
-
|
36
|
+
backend
|
37
37
|
name
|
38
|
-
worker
|
39
38
|
parser
|
40
|
-
interchanger
|
41
39
|
responder
|
42
40
|
batch_processing
|
41
|
+
persistent
|
43
42
|
]).uniq
|
44
43
|
end
|
45
44
|
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
# Namespace for all different backends Karafka supports
|
5
|
+
module Backends
|
6
|
+
# Backend that just runs stuff asap without any scheduling
|
7
|
+
module Inline
|
8
|
+
private
|
9
|
+
|
10
|
+
# Executes perform code immediately (without enqueuing)
|
11
|
+
def process
|
12
|
+
Karafka.monitor.notice(self.class, params_batch)
|
13
|
+
perform
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -3,11 +3,11 @@
|
|
3
3
|
# Karafka module namespace
|
4
4
|
module Karafka
|
5
5
|
# Base controller from which all Karafka controllers should inherit
|
6
|
-
# Similar to Rails controllers we can define
|
6
|
+
# Similar to Rails controllers we can define after_received callbacks
|
7
7
|
# that will be executed
|
8
8
|
#
|
9
|
-
# Note that if
|
10
|
-
# the perform method won't be executed
|
9
|
+
# Note that if after_received return false, the chain will be stopped and
|
10
|
+
# the perform method won't be executed
|
11
11
|
#
|
12
12
|
# @example Create simple controller
|
13
13
|
# class ExamplesController < Karafka::BaseController
|
@@ -16,9 +16,9 @@ module Karafka
|
|
16
16
|
# end
|
17
17
|
# end
|
18
18
|
#
|
19
|
-
# @example Create a controller with a block
|
19
|
+
# @example Create a controller with a block after_received
|
20
20
|
# class ExampleController < Karafka::BaseController
|
21
|
-
#
|
21
|
+
# after_received do
|
22
22
|
# # Here we should have some checking logic
|
23
23
|
# # If false is returned, won't schedule a perform action
|
24
24
|
# end
|
@@ -28,9 +28,9 @@ module Karafka
|
|
28
28
|
# end
|
29
29
|
# end
|
30
30
|
#
|
31
|
-
# @example Create a controller with a method
|
31
|
+
# @example Create a controller with a method after_received
|
32
32
|
# class ExampleController < Karafka::BaseController
|
33
|
-
#
|
33
|
+
# after_received :after_received_method
|
34
34
|
#
|
35
35
|
# def perform
|
36
36
|
# # some logic here
|
@@ -38,90 +38,72 @@ module Karafka
|
|
38
38
|
#
|
39
39
|
# private
|
40
40
|
#
|
41
|
-
# def
|
41
|
+
# def after_received_method
|
42
42
|
# # Here we should have some checking logic
|
43
43
|
# # If false is returned, won't schedule a perform action
|
44
44
|
# end
|
45
45
|
# end
|
46
|
-
#
|
47
|
-
# @example Create a controller with an after_failure action
|
48
|
-
# class ExampleController < Karafka::BaseController
|
49
|
-
# def perform
|
50
|
-
# # some logic here
|
51
|
-
# end
|
52
|
-
#
|
53
|
-
# def after_failure
|
54
|
-
# # action taken in case perform fails
|
55
|
-
# end
|
56
|
-
# end
|
57
46
|
class BaseController
|
58
47
|
extend ActiveSupport::DescendantsTracker
|
59
48
|
include ActiveSupport::Callbacks
|
60
49
|
|
61
|
-
# The
|
50
|
+
# The call method is wrapped with a set of callbacks
|
62
51
|
# We won't run perform at the backend if any of the callbacks
|
63
52
|
# returns false
|
64
53
|
# @see http://api.rubyonrails.org/classes/ActiveSupport/Callbacks/ClassMethods.html#method-i-get_callbacks
|
65
|
-
define_callbacks :
|
54
|
+
define_callbacks :after_received
|
66
55
|
|
67
|
-
# Each controller instance is always bind to a single topic. We don't place it on a class
|
68
|
-
# level because some programmers use same controller for multiple topics
|
69
|
-
attr_accessor :topic
|
70
56
|
attr_accessor :params_batch
|
71
57
|
|
72
58
|
class << self
|
73
|
-
|
59
|
+
attr_reader :topic
|
60
|
+
|
61
|
+
# Assigns a topic to a controller and build up proper controller functionalities, so it can
|
62
|
+
# cooperate with the topic settings
|
63
|
+
# @param topic [Karafka::Routing::Topic]
|
64
|
+
# @return [Karafka::Routing::Topic] assigned topic
|
65
|
+
def topic=(topic)
|
66
|
+
@topic = topic
|
67
|
+
Controllers::Includer.call(self)
|
68
|
+
@topic
|
69
|
+
end
|
70
|
+
|
71
|
+
# Creates a callback that will be executed after receiving message but before executing the
|
72
|
+
# backend for processing
|
74
73
|
# @param method_name [Symbol, String] method name or nil if we plan to provide a block
|
75
74
|
# @yield A block with a code that should be executed before scheduling
|
76
|
-
# @
|
77
|
-
#
|
78
|
-
# before_enqueue do
|
75
|
+
# @example Define a block after_received callback
|
76
|
+
# after_received do
|
79
77
|
# # logic here
|
80
78
|
# end
|
81
79
|
#
|
82
|
-
# @example Define a class name
|
83
|
-
#
|
84
|
-
def
|
85
|
-
set_callback :
|
80
|
+
# @example Define a class name after_received callback
|
81
|
+
# after_received :method_name
|
82
|
+
def after_received(method_name = nil, &block)
|
83
|
+
set_callback :after_received, :before, method_name ? method_name : block
|
86
84
|
end
|
87
85
|
end
|
88
86
|
|
87
|
+
# @return [Karafka::Routing::Topic] topic to which a given controller is subscribed
|
88
|
+
def topic
|
89
|
+
self.class.topic
|
90
|
+
end
|
91
|
+
|
89
92
|
# Creates lazy loaded params batch object
|
90
93
|
# @note Until first params usage, it won't parse data at all
|
91
94
|
# @param messages [Array<Kafka::FetchedMessage>, Array<Hash>] messages with raw
|
92
|
-
# content (from Kafka) or messages inside a hash (from
|
95
|
+
# content (from Kafka) or messages inside a hash (from backend, etc)
|
93
96
|
# @return [Karafka::Params::ParamsBatch] lazy loaded params batch
|
94
97
|
def params_batch=(messages)
|
95
98
|
@params_batch = Karafka::Params::ParamsBatch.new(messages, topic.parser)
|
96
99
|
end
|
97
100
|
|
98
|
-
# @return [Karafka::Params::Params] params instance for non batch processed controllers
|
99
|
-
# @raise [Karafka::Errors::ParamsMethodUnavailable] raised when we try to use params
|
100
|
-
# method in a batch_processed controller
|
101
|
-
def params
|
102
|
-
raise Karafka::Errors::ParamsMethodUnavailable if topic.batch_processing
|
103
|
-
params_batch.first
|
104
|
-
end
|
105
|
-
|
106
101
|
# Executes the default controller flow, runs callbacks and if not halted
|
107
|
-
# will
|
108
|
-
def schedule
|
109
|
-
run_callbacks :schedule do
|
110
|
-
case topic.processing_backend
|
111
|
-
when :inline then
|
112
|
-
call_inline
|
113
|
-
when :sidekiq then
|
114
|
-
call_async
|
115
|
-
else
|
116
|
-
raise Errors::InvalidProcessingBackend, topic.processing_backend
|
117
|
-
end
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
# @note We want to leave the #perform method as a public API, but just in case we will do some
|
122
|
-
# pre or post processing we use call method instead of directly executing #perform
|
102
|
+
# will call process method of a proper backend
|
123
103
|
def call
|
124
|
-
|
104
|
+
run_callbacks :after_received do
|
105
|
+
process
|
106
|
+
end
|
125
107
|
end
|
126
108
|
|
127
109
|
private
|
@@ -132,43 +114,5 @@ module Karafka
|
|
132
114
|
def perform
|
133
115
|
raise NotImplementedError, 'Implement this in a subclass'
|
134
116
|
end
|
135
|
-
|
136
|
-
# @return [Karafka::BaseResponder] responder instance if defined
|
137
|
-
# @return [nil] nil if no responder for this controller
|
138
|
-
def responder
|
139
|
-
@responder ||= topic.responder&.new(topic.parser)
|
140
|
-
end
|
141
|
-
|
142
|
-
# Responds with given data using given responder. This allows us to have a similar way of
|
143
|
-
# defining flows like synchronous protocols
|
144
|
-
# @param data Anything we want to pass to responder based on which we want to trigger further
|
145
|
-
# Kafka responding
|
146
|
-
# @raise [Karafka::Errors::ResponderMissing] raised when we don't have a responder defined,
|
147
|
-
# but we still try to use this method
|
148
|
-
def respond_with(*data)
|
149
|
-
raise(Errors::ResponderMissing, self.class) unless responder
|
150
|
-
Karafka.monitor.notice(self.class, data: data)
|
151
|
-
responder.call(*data)
|
152
|
-
end
|
153
|
-
|
154
|
-
# Executes perform code immediately (without enqueuing)
|
155
|
-
# @note Despite the fact, that workers won't be used, we still initialize all the
|
156
|
-
# classes and other framework elements
|
157
|
-
def call_inline
|
158
|
-
Karafka.monitor.notice(self.class, params_batch)
|
159
|
-
call
|
160
|
-
end
|
161
|
-
|
162
|
-
# Enqueues the execution of perform method into a worker.
|
163
|
-
# @note Each worker needs to have a class #perform_async method that will allow us to pass
|
164
|
-
# parameters into it. We always pass topic as a first argument and this request params_batch
|
165
|
-
# as a second one (we pass topic to be able to build back the controller in the worker)
|
166
|
-
def call_async
|
167
|
-
Karafka.monitor.notice(self.class, params_batch)
|
168
|
-
topic.worker.perform_async(
|
169
|
-
topic.id,
|
170
|
-
topic.interchanger.load(params_batch.to_a)
|
171
|
-
)
|
172
|
-
end
|
173
117
|
end
|
174
118
|
end
|
@@ -138,6 +138,25 @@ module Karafka
|
|
138
138
|
raise Karafka::Errors::InvalidResponderUsage, result.errors
|
139
139
|
end
|
140
140
|
|
141
|
+
# Takes all the messages from the buffer and delivers them one by one
|
142
|
+
# @note This method is executed after the validation, so we're sure that
|
143
|
+
# what we send is legit and it will go to a proper topics
|
144
|
+
def deliver!
|
145
|
+
messages_buffer.each do |topic, data_elements|
|
146
|
+
# We map this topic name, so it will match namespaced/etc topic in Kafka
|
147
|
+
# @note By default will not change topic (if default mapper used)
|
148
|
+
mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
|
149
|
+
|
150
|
+
data_elements.each do |(data, options)|
|
151
|
+
::WaterDrop::Message.new(
|
152
|
+
mapped_topic,
|
153
|
+
data,
|
154
|
+
options
|
155
|
+
).send!
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
141
160
|
# Method that needs to be implemented in a subclass. It should handle responding
|
142
161
|
# on registered topics
|
143
162
|
# @raise [NotImplementedError] This method needs to be implemented in a subclass
|
@@ -157,24 +176,5 @@ module Karafka
|
|
157
176
|
messages_buffer[topic.to_s] ||= []
|
158
177
|
messages_buffer[topic.to_s] << [@parser_class.generate(data), options]
|
159
178
|
end
|
160
|
-
|
161
|
-
# Takes all the messages from the buffer and delivers them one by one
|
162
|
-
# @note This method is executed after the validation, so we're sure that
|
163
|
-
# what we send is legit and it will go to a proper topics
|
164
|
-
def deliver!
|
165
|
-
messages_buffer.each do |topic, data_elements|
|
166
|
-
# We map this topic name, so it will match namespaced/etc topic in Kafka
|
167
|
-
# @note By default will not change topic (if default mapper used)
|
168
|
-
mapped_topic = Karafka::App.config.topic_mapper.outgoing(topic)
|
169
|
-
|
170
|
-
data_elements.each do |(data, options)|
|
171
|
-
::WaterDrop::Message.new(
|
172
|
-
mapped_topic,
|
173
|
-
data,
|
174
|
-
options
|
175
|
-
).send!
|
176
|
-
end
|
177
|
-
end
|
178
|
-
end
|
179
179
|
end
|
180
180
|
end
|