karafka 1.3.6 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.diffend.yml +3 -0
- data/.github/workflows/ci.yml +52 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +27 -1
- data/CODE_OF_CONDUCT.md +1 -1
- data/Gemfile +2 -0
- data/Gemfile.lock +42 -41
- data/README.md +3 -5
- data/certs/mensfeld.pem +21 -21
- data/config/errors.yml +2 -0
- data/docker-compose.yml +17 -0
- data/karafka.gemspec +3 -4
- data/lib/karafka.rb +1 -1
- data/lib/karafka/assignment_strategies/round_robin.rb +13 -0
- data/lib/karafka/attributes_map.rb +1 -0
- data/lib/karafka/cli.rb +8 -0
- data/lib/karafka/cli/base.rb +4 -4
- data/lib/karafka/cli/missingno.rb +19 -0
- data/lib/karafka/connection/api_adapter.rb +5 -3
- data/lib/karafka/connection/batch_delegator.rb +5 -1
- data/lib/karafka/consumers/batch_metadata.rb +10 -0
- data/lib/karafka/consumers/includer.rb +5 -4
- data/lib/karafka/contracts.rb +1 -1
- data/lib/karafka/contracts/consumer_group.rb +8 -3
- data/lib/karafka/helpers/class_matcher.rb +1 -1
- data/lib/karafka/instrumentation/logger.rb +2 -2
- data/lib/karafka/instrumentation/stdout_listener.rb +4 -2
- data/lib/karafka/params/batch_metadata.rb +26 -0
- data/lib/karafka/params/builders/batch_metadata.rb +30 -0
- data/lib/karafka/params/builders/params.rb +17 -15
- data/lib/karafka/params/builders/params_batch.rb +2 -2
- data/lib/karafka/params/metadata.rb +14 -29
- data/lib/karafka/params/params.rb +27 -41
- data/lib/karafka/params/params_batch.rb +15 -16
- data/lib/karafka/routing/builder.rb +1 -0
- data/lib/karafka/routing/consumer_group.rb +5 -3
- data/lib/karafka/serialization/json/deserializer.rb +2 -2
- data/lib/karafka/setup/config.rb +5 -0
- data/lib/karafka/version.rb +1 -1
- metadata +37 -46
- metadata.gz.sig +0 -0
- data/.travis.yml +0 -36
- data/lib/karafka/consumers/metadata.rb +0 -10
- data/lib/karafka/params/builders/metadata.rb +0 -33
@@ -15,47 +15,46 @@ module Karafka
|
|
15
15
|
@params_array = params_array
|
16
16
|
end
|
17
17
|
|
18
|
-
# @yieldparam [Karafka::Params::Params] each
|
19
|
-
# @note Invocation of this method will cause loading and deserializing each param after
|
20
|
-
# another.
|
21
|
-
# directly
|
18
|
+
# @yieldparam [Karafka::Params::Params] each params instance
|
19
|
+
# @note Invocation of this method will not cause loading and deserializing each param after
|
20
|
+
# another.
|
22
21
|
def each
|
23
|
-
@params_array.each { |param| yield(param
|
22
|
+
@params_array.each { |param| yield(param) }
|
24
23
|
end
|
25
24
|
|
26
25
|
# @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
|
27
26
|
# can be used for batch insert, etc. Without invoking all, up until first use, they won't
|
28
27
|
# be deserialized
|
29
28
|
def deserialize!
|
30
|
-
each(&:
|
29
|
+
each(&:payload)
|
31
30
|
end
|
32
31
|
|
33
32
|
# @return [Array<Object>] array with deserialized payloads. This method can be useful when
|
34
33
|
# we don't care about metadata and just want to extract all the data payloads from the
|
35
34
|
# batch
|
36
35
|
def payloads
|
37
|
-
|
36
|
+
map(&:payload)
|
38
37
|
end
|
39
38
|
|
40
|
-
# @return [Karafka::Params::Params] first element
|
39
|
+
# @return [Karafka::Params::Params] first element
|
41
40
|
def first
|
42
|
-
@params_array.first
|
41
|
+
@params_array.first
|
43
42
|
end
|
44
43
|
|
45
|
-
# @return [Karafka::Params::Params] last element
|
44
|
+
# @return [Karafka::Params::Params] last element
|
46
45
|
def last
|
47
|
-
@params_array.last
|
48
|
-
end
|
49
|
-
|
50
|
-
# @return [Array<Karafka::Params::Params>] pure array with params (not deserialized)
|
51
|
-
def to_a
|
52
|
-
@params_array
|
46
|
+
@params_array.last
|
53
47
|
end
|
54
48
|
|
55
49
|
# @return [Integer] number of messages in the batch
|
56
50
|
def size
|
57
51
|
@params_array.size
|
58
52
|
end
|
53
|
+
|
54
|
+
# @return [Array<Karafka::Params::Params>] pure array with params
|
55
|
+
def to_a
|
56
|
+
@params_array
|
57
|
+
end
|
59
58
|
end
|
60
59
|
end
|
61
60
|
end
|
@@ -8,9 +8,11 @@ module Karafka
|
|
8
8
|
class ConsumerGroup
|
9
9
|
extend Helpers::ConfigRetriever
|
10
10
|
|
11
|
-
attr_reader
|
12
|
-
|
13
|
-
|
11
|
+
attr_reader(
|
12
|
+
:topics,
|
13
|
+
:id,
|
14
|
+
:name
|
15
|
+
)
|
14
16
|
|
15
17
|
# @param name [String, Symbol] raw name of this consumer group. Raw means, that it does not
|
16
18
|
# yet have an application client_id namespace, this will be added here by default.
|
@@ -17,8 +17,8 @@ module Karafka
|
|
17
17
|
# }
|
18
18
|
# Deserializer.call(params) #=> { 'a' => 1 }
|
19
19
|
def call(params)
|
20
|
-
::
|
21
|
-
rescue ::
|
20
|
+
params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
|
21
|
+
rescue ::JSON::ParserError => e
|
22
22
|
raise ::Karafka::Errors::DeserializationError, e
|
23
23
|
end
|
24
24
|
end
|
data/lib/karafka/setup/config.rb
CHANGED
@@ -89,6 +89,9 @@ module Karafka
|
|
89
89
|
# are stored for further processing. Note, that each item in the queue represents a
|
90
90
|
# response from a single broker
|
91
91
|
setting :fetcher_max_queue_size, 10
|
92
|
+
# option assignment_strategy [Object] a strategy determining the assignment of
|
93
|
+
# partitions to the consumers.
|
94
|
+
setting :assignment_strategy, Karafka::AssignmentStrategies::RoundRobin.new
|
92
95
|
# option max_bytes_per_partition [Integer] the maximum amount of data fetched
|
93
96
|
# from a single partition at a time.
|
94
97
|
setting :max_bytes_per_partition, 1_048_576
|
@@ -124,6 +127,8 @@ module Karafka
|
|
124
127
|
# will be killed. Note that some Kafka operations are by definition long-running, such as
|
125
128
|
# waiting for new messages to arrive in a partition, so don't set this value too low
|
126
129
|
setting :socket_timeout, 30
|
130
|
+
# option partitioner [Object, nil] the partitioner that should be used by the client
|
131
|
+
setting :partitioner, nil
|
127
132
|
|
128
133
|
# SSL authentication related settings
|
129
134
|
# option ca_cert [String, nil] SSL CA certificate
|
data/lib/karafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -13,30 +13,30 @@ cert_chain:
|
|
13
13
|
- |
|
14
14
|
-----BEGIN CERTIFICATE-----
|
15
15
|
MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
16
|
+
ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
|
17
|
+
NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
|
18
|
+
CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
|
19
|
+
2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
|
20
|
+
N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
|
21
|
+
hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
|
22
|
+
sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
|
23
|
+
VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
|
24
|
+
tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
|
25
|
+
Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
|
26
|
+
LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
|
27
|
+
BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
|
28
28
|
EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
29
|
+
c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
|
30
|
+
gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
|
31
|
+
2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
|
32
|
+
fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
|
33
|
+
zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
|
34
|
+
EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
|
35
|
+
65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
|
36
|
+
2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
|
37
|
+
nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2020-04
|
39
|
+
date: 2020-12-04 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: dry-configurable
|
@@ -122,20 +122,6 @@ dependencies:
|
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '1.0'
|
125
|
-
- !ruby/object:Gem::Dependency
|
126
|
-
name: multi_json
|
127
|
-
requirement: !ruby/object:Gem::Requirement
|
128
|
-
requirements:
|
129
|
-
- - ">="
|
130
|
-
- !ruby/object:Gem::Version
|
131
|
-
version: '1.12'
|
132
|
-
type: :runtime
|
133
|
-
prerelease: false
|
134
|
-
version_requirements: !ruby/object:Gem::Requirement
|
135
|
-
requirements:
|
136
|
-
- - ">="
|
137
|
-
- !ruby/object:Gem::Version
|
138
|
-
version: '1.12'
|
139
125
|
- !ruby/object:Gem::Dependency
|
140
126
|
name: rake
|
141
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -156,14 +142,14 @@ dependencies:
|
|
156
142
|
requirements:
|
157
143
|
- - ">="
|
158
144
|
- !ruby/object:Gem::Version
|
159
|
-
version: 0.
|
145
|
+
version: 1.0.0
|
160
146
|
type: :runtime
|
161
147
|
prerelease: false
|
162
148
|
version_requirements: !ruby/object:Gem::Requirement
|
163
149
|
requirements:
|
164
150
|
- - ">="
|
165
151
|
- !ruby/object:Gem::Version
|
166
|
-
version: 0.
|
152
|
+
version: 1.0.0
|
167
153
|
- !ruby/object:Gem::Dependency
|
168
154
|
name: thor
|
169
155
|
requirement: !ruby/object:Gem::Requirement
|
@@ -184,14 +170,14 @@ dependencies:
|
|
184
170
|
requirements:
|
185
171
|
- - "~>"
|
186
172
|
- !ruby/object:Gem::Version
|
187
|
-
version: 1.
|
173
|
+
version: 1.4.0
|
188
174
|
type: :runtime
|
189
175
|
prerelease: false
|
190
176
|
version_requirements: !ruby/object:Gem::Requirement
|
191
177
|
requirements:
|
192
178
|
- - "~>"
|
193
179
|
- !ruby/object:Gem::Version
|
194
|
-
version: 1.
|
180
|
+
version: 1.4.0
|
195
181
|
- !ruby/object:Gem::Dependency
|
196
182
|
name: zeitwerk
|
197
183
|
requirement: !ruby/object:Gem::Requirement
|
@@ -208,7 +194,7 @@ dependencies:
|
|
208
194
|
version: '2.1'
|
209
195
|
description: Framework used to simplify Apache Kafka based Ruby applications development
|
210
196
|
email:
|
211
|
-
- maciej@
|
197
|
+
- maciej@mensfeld.pl
|
212
198
|
- pavlo.vavruk@gmail.com
|
213
199
|
- adam99g@gmail.com
|
214
200
|
executables:
|
@@ -218,14 +204,15 @@ extra_rdoc_files: []
|
|
218
204
|
files:
|
219
205
|
- ".coditsu/ci.yml"
|
220
206
|
- ".console_irbrc"
|
207
|
+
- ".diffend.yml"
|
221
208
|
- ".github/FUNDING.yml"
|
222
209
|
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
223
210
|
- ".github/ISSUE_TEMPLATE/feature_request.md"
|
211
|
+
- ".github/workflows/ci.yml"
|
224
212
|
- ".gitignore"
|
225
213
|
- ".rspec"
|
226
214
|
- ".ruby-gemset"
|
227
215
|
- ".ruby-version"
|
228
|
-
- ".travis.yml"
|
229
216
|
- CHANGELOG.md
|
230
217
|
- CODE_OF_CONDUCT.md
|
231
218
|
- CONTRIBUTING.md
|
@@ -236,9 +223,11 @@ files:
|
|
236
223
|
- bin/karafka
|
237
224
|
- certs/mensfeld.pem
|
238
225
|
- config/errors.yml
|
226
|
+
- docker-compose.yml
|
239
227
|
- karafka.gemspec
|
240
228
|
- lib/karafka.rb
|
241
229
|
- lib/karafka/app.rb
|
230
|
+
- lib/karafka/assignment_strategies/round_robin.rb
|
242
231
|
- lib/karafka/attributes_map.rb
|
243
232
|
- lib/karafka/backends/inline.rb
|
244
233
|
- lib/karafka/base_consumer.rb
|
@@ -249,6 +238,7 @@ files:
|
|
249
238
|
- lib/karafka/cli/flow.rb
|
250
239
|
- lib/karafka/cli/info.rb
|
251
240
|
- lib/karafka/cli/install.rb
|
241
|
+
- lib/karafka/cli/missingno.rb
|
252
242
|
- lib/karafka/cli/server.rb
|
253
243
|
- lib/karafka/code_reloader.rb
|
254
244
|
- lib/karafka/connection/api_adapter.rb
|
@@ -257,9 +247,9 @@ files:
|
|
257
247
|
- lib/karafka/connection/client.rb
|
258
248
|
- lib/karafka/connection/listener.rb
|
259
249
|
- lib/karafka/connection/message_delegator.rb
|
250
|
+
- lib/karafka/consumers/batch_metadata.rb
|
260
251
|
- lib/karafka/consumers/callbacks.rb
|
261
252
|
- lib/karafka/consumers/includer.rb
|
262
|
-
- lib/karafka/consumers/metadata.rb
|
263
253
|
- lib/karafka/consumers/responders.rb
|
264
254
|
- lib/karafka/consumers/single_params.rb
|
265
255
|
- lib/karafka/contracts.rb
|
@@ -278,7 +268,8 @@ files:
|
|
278
268
|
- lib/karafka/instrumentation/monitor.rb
|
279
269
|
- lib/karafka/instrumentation/proctitle_listener.rb
|
280
270
|
- lib/karafka/instrumentation/stdout_listener.rb
|
281
|
-
- lib/karafka/params/
|
271
|
+
- lib/karafka/params/batch_metadata.rb
|
272
|
+
- lib/karafka/params/builders/batch_metadata.rb
|
282
273
|
- lib/karafka/params/builders/params.rb
|
283
274
|
- lib/karafka/params/builders/params_batch.rb
|
284
275
|
- lib/karafka/params/metadata.rb
|
@@ -329,7 +320,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
329
320
|
- !ruby/object:Gem::Version
|
330
321
|
version: '0'
|
331
322
|
requirements: []
|
332
|
-
rubygems_version: 3.1.
|
323
|
+
rubygems_version: 3.1.4
|
333
324
|
signing_key:
|
334
325
|
specification_version: 4
|
335
326
|
summary: Ruby based framework for working with Apache Kafka
|
metadata.gz.sig
CHANGED
Binary file
|
data/.travis.yml
DELETED
@@ -1,36 +0,0 @@
|
|
1
|
-
services:
|
2
|
-
- docker
|
3
|
-
|
4
|
-
dist: trusty
|
5
|
-
sudo: false
|
6
|
-
cache: bundler
|
7
|
-
|
8
|
-
git:
|
9
|
-
depth: false
|
10
|
-
|
11
|
-
test: &test
|
12
|
-
stage: Test
|
13
|
-
language: ruby
|
14
|
-
before_install:
|
15
|
-
- yes | gem update --system
|
16
|
-
script: bundle exec rspec
|
17
|
-
|
18
|
-
jobs:
|
19
|
-
include:
|
20
|
-
- <<: *test
|
21
|
-
rvm: 2.7.1
|
22
|
-
- <<: *test
|
23
|
-
rvm: 2.6.6
|
24
|
-
- <<: *test
|
25
|
-
rvm: 2.5.8
|
26
|
-
|
27
|
-
- stage: coditsu
|
28
|
-
language: ruby
|
29
|
-
rvm: 2.7.1
|
30
|
-
before_install:
|
31
|
-
- yes | gem update --system
|
32
|
-
script: \curl -sSL https://api.coditsu.io/run/ci | bash
|
33
|
-
|
34
|
-
stages:
|
35
|
-
- test
|
36
|
-
- coditsu
|
@@ -1,33 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Params
|
5
|
-
module Builders
|
6
|
-
# Builder for creating metadata object based on the message or batch informations
|
7
|
-
# @note We have 2 ways of creating metadata based on the way ruby-kafka operates
|
8
|
-
module Metadata
|
9
|
-
class << self
|
10
|
-
# Creates metadata based on the kafka batch data
|
11
|
-
# @param kafka_batch [Kafka::FetchedBatch] kafka batch details
|
12
|
-
# @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
|
13
|
-
# @return [Karafka::Params::Metadata] metadata object
|
14
|
-
def from_kafka_batch(kafka_batch, topic)
|
15
|
-
Karafka::Params::Metadata
|
16
|
-
.new
|
17
|
-
.merge!(
|
18
|
-
'batch_size' => kafka_batch.messages.count,
|
19
|
-
'first_offset' => kafka_batch.first_offset,
|
20
|
-
'highwater_mark_offset' => kafka_batch.highwater_mark_offset,
|
21
|
-
'last_offset' => kafka_batch.last_offset,
|
22
|
-
'offset_lag' => kafka_batch.offset_lag,
|
23
|
-
'deserializer' => topic.deserializer,
|
24
|
-
'partition' => kafka_batch.partition,
|
25
|
-
'topic' => kafka_batch.topic,
|
26
|
-
'unknown_last_offset' => kafka_batch.unknown_last_offset?
|
27
|
-
)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|