karafka 1.3.7 → 1.4.0.pre.rc1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of karafka might be problematic. Click here for more details.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/CHANGELOG.md +18 -0
- data/Gemfile.lock +7 -9
- data/README.md +2 -4
- data/karafka.gemspec +1 -2
- data/lib/karafka.rb +1 -1
- data/lib/karafka/connection/api_adapter.rb +5 -3
- data/lib/karafka/connection/batch_delegator.rb +5 -1
- data/lib/karafka/consumers/batch_metadata.rb +10 -0
- data/lib/karafka/consumers/includer.rb +5 -4
- data/lib/karafka/consumers/single_params.rb +1 -1
- data/lib/karafka/instrumentation/stdout_listener.rb +4 -2
- data/lib/karafka/params/batch_metadata.rb +26 -0
- data/lib/karafka/params/builders/batch_metadata.rb +30 -0
- data/lib/karafka/params/builders/params.rb +17 -15
- data/lib/karafka/params/builders/params_batch.rb +2 -2
- data/lib/karafka/params/metadata.rb +14 -29
- data/lib/karafka/params/params.rb +24 -52
- data/lib/karafka/params/params_batch.rb +15 -16
- data/lib/karafka/serialization/json/deserializer.rb +2 -2
- data/lib/karafka/version.rb +1 -1
- metadata +10 -23
- metadata.gz.sig +0 -0
- data/lib/karafka/consumers/metadata.rb +0 -10
- data/lib/karafka/params/builders/metadata.rb +0 -33
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 708eebd9feb79db2f96e93ce69c3f1270cb0964ecb08b1b63e6834ff144f8618
|
4
|
+
data.tar.gz: a6f510dc87e213b1eb8b9bd2efba1e7a65d061056c14275acb26a6895ac26a9e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1b5667ceb4801c494b752295c18a2145c43c0aa13ca274dc55b17faec0da2996c5d50984dc3a6986885e2abb20a02aa70f8c43f16b067ac70521f51214939726
|
7
|
+
data.tar.gz: cc1a9dcd6e5ec3973ec490d449b5cfa1b017c0f228e76da0f08682021a8ce29860611188e4ec83e0a619ef27d2390205014733a781db792896ed62e4ecce7dc7
|
checksums.yaml.gz.sig
CHANGED
Binary file
|
data.tar.gz.sig
CHANGED
Binary file
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,23 @@
|
|
1
1
|
# Karafka framework changelog
|
2
2
|
|
3
|
+
## 1.4.0-rc1 (unreleased)
|
4
|
+
- Rename `Karafka::Params::Metadata` to `Karafka::Params::BatchMetadata`
|
5
|
+
` Rename consumer `#metadata` to `#batch_metadata`
|
6
|
+
- Separate metadata (including Karafka native metadata) from the root of params
|
7
|
+
- Remove metadata hash dependency
|
8
|
+
- Remove params dependency on a hash in favour of PORO
|
9
|
+
- Remove batch metadata dependency on a hash
|
10
|
+
- Remove MultiJson in favour of JSON in the default deserializer
|
11
|
+
- allow accessing all the metadata without accessing the payload
|
12
|
+
- freeze params and underlying elements except for the mutable payload
|
13
|
+
- provide access to raw payload after serialization
|
14
|
+
- fixes a bug where a non-deserializable (error) params would be marked as deserialized after first unsuccessful deserialization attempt
|
15
|
+
- fixes bug where karafka would mutate internal ruby-kafka state
|
16
|
+
- fixes bug where topic name in metadata would not be mapped using topic mappers
|
17
|
+
- simplifies the params and params batch API, before `#payload` usage, it won't be deserialized
|
18
|
+
- removes the `#[]` API from params to prevent from accessing raw data in a different way than #raw_payload
|
19
|
+
- makes the params batch operations consistent as params payload is deserialized only when accessed explicitly
|
20
|
+
|
3
21
|
## 1.3.7 (2020-08-11)
|
4
22
|
- #599 - Allow metadata access without deserialization attempt (rabotyaga)
|
5
23
|
- Sync with ruby-kafka `1.2.0` api
|
data/Gemfile.lock
CHANGED
@@ -1,18 +1,17 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
karafka (1.
|
4
|
+
karafka (1.4.0.pre.rc1)
|
5
5
|
dry-configurable (~> 0.8)
|
6
6
|
dry-inflector (~> 0.1)
|
7
7
|
dry-monitor (~> 0.3)
|
8
8
|
dry-validation (~> 1.2)
|
9
9
|
envlogic (~> 1.1)
|
10
10
|
irb (~> 1.0)
|
11
|
-
multi_json (>= 1.12)
|
12
11
|
rake (>= 11.3)
|
13
12
|
ruby-kafka (>= 1.0.0)
|
14
13
|
thor (>= 0.20)
|
15
|
-
waterdrop (~> 1.
|
14
|
+
waterdrop (~> 1.4.0)
|
16
15
|
zeitwerk (~> 2.1)
|
17
16
|
|
18
17
|
GEM
|
@@ -49,7 +48,7 @@ GEM
|
|
49
48
|
dry-equalizer (~> 0.2)
|
50
49
|
dry-inflector (0.2.0)
|
51
50
|
dry-initializer (3.0.3)
|
52
|
-
dry-logic (1.0.
|
51
|
+
dry-logic (1.0.7)
|
53
52
|
concurrent-ruby (~> 1.0)
|
54
53
|
dry-core (~> 0.2)
|
55
54
|
dry-equalizer (~> 0.2)
|
@@ -58,7 +57,7 @@ GEM
|
|
58
57
|
dry-core (~> 0.4)
|
59
58
|
dry-equalizer (~> 0.2)
|
60
59
|
dry-events (~> 0.2)
|
61
|
-
dry-schema (1.5.
|
60
|
+
dry-schema (1.5.3)
|
62
61
|
concurrent-ruby (~> 1.0)
|
63
62
|
dry-configurable (~> 0.8, >= 0.8.3)
|
64
63
|
dry-core (~> 0.4)
|
@@ -73,7 +72,7 @@ GEM
|
|
73
72
|
dry-equalizer (~> 0.3)
|
74
73
|
dry-inflector (~> 0.1, >= 0.1.2)
|
75
74
|
dry-logic (~> 1.0, >= 1.0.2)
|
76
|
-
dry-validation (1.5.
|
75
|
+
dry-validation (1.5.4)
|
77
76
|
concurrent-ruby (~> 1.0)
|
78
77
|
dry-container (~> 0.7, >= 0.7.1)
|
79
78
|
dry-core (~> 0.4)
|
@@ -91,7 +90,6 @@ GEM
|
|
91
90
|
reline (>= 0.0.1)
|
92
91
|
king_konf (0.3.7)
|
93
92
|
minitest (5.14.1)
|
94
|
-
multi_json (1.15.0)
|
95
93
|
rake (13.0.1)
|
96
94
|
reline (0.1.4)
|
97
95
|
io-console (~> 0.5)
|
@@ -110,7 +108,7 @@ GEM
|
|
110
108
|
rspec-support (3.9.3)
|
111
109
|
ruby-kafka (1.2.0)
|
112
110
|
digest-crc
|
113
|
-
simplecov (0.
|
111
|
+
simplecov (0.19.0)
|
114
112
|
docile (~> 1.1)
|
115
113
|
simplecov-html (~> 0.11)
|
116
114
|
simplecov-html (0.12.2)
|
@@ -118,7 +116,7 @@ GEM
|
|
118
116
|
thread_safe (0.3.6)
|
119
117
|
tzinfo (1.2.7)
|
120
118
|
thread_safe (~> 0.1)
|
121
|
-
waterdrop (1.
|
119
|
+
waterdrop (1.4.0)
|
122
120
|
delivery_boy (>= 0.2, < 2.x)
|
123
121
|
dry-configurable (~> 0.8)
|
124
122
|
dry-monitor (~> 0.3)
|
data/README.md
CHANGED
@@ -2,11 +2,9 @@
|
|
2
2
|
|
3
3
|
[![Build Status](https://travis-ci.org/karafka/karafka.svg?branch=master)](https://travis-ci.org/karafka/karafka)
|
4
4
|
|
5
|
-
**Note**: Documentation presented here refers to Karafka `1.
|
5
|
+
**Note**: Documentation presented here refers to **not yet** released Karafka `1.4.x`.
|
6
6
|
|
7
|
-
If you
|
8
|
-
|
9
|
-
If you are looking for the documentation for Karafka `1.2.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.2).
|
7
|
+
If you are looking for the documentation for Karafka `1.3.x`, it can be found [here](https://github.com/karafka/wiki/tree/1.3).
|
10
8
|
|
11
9
|
## About Karafka
|
12
10
|
|
data/karafka.gemspec
CHANGED
@@ -23,11 +23,10 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.add_dependency 'dry-validation', '~> 1.2'
|
24
24
|
spec.add_dependency 'envlogic', '~> 1.1'
|
25
25
|
spec.add_dependency 'irb', '~> 1.0'
|
26
|
-
spec.add_dependency 'multi_json', '>= 1.12'
|
27
26
|
spec.add_dependency 'rake', '>= 11.3'
|
28
27
|
spec.add_dependency 'ruby-kafka', '>= 1.0.0'
|
29
28
|
spec.add_dependency 'thor', '>= 0.20'
|
30
|
-
spec.add_dependency 'waterdrop', '~> 1.
|
29
|
+
spec.add_dependency 'waterdrop', '~> 1.4.0'
|
31
30
|
spec.add_dependency 'zeitwerk', '~> 2.1'
|
32
31
|
|
33
32
|
spec.required_ruby_version = '>= 2.5.0'
|
data/lib/karafka.rb
CHANGED
@@ -109,11 +109,13 @@ module Karafka
|
|
109
109
|
# Majority of users don't use custom topic mappers. No need to change anything when it
|
110
110
|
# is a default mapper that does not change anything. Only some cloud providers require
|
111
111
|
# topics to be remapped
|
112
|
-
return [params] if Karafka::App.config.topic_mapper.is_a?(
|
112
|
+
return [params.metadata] if Karafka::App.config.topic_mapper.is_a?(
|
113
|
+
Karafka::Routing::TopicMapper
|
114
|
+
)
|
113
115
|
|
114
116
|
# @note We don't use tap as it is around 13% slower than non-dup version
|
115
|
-
dupped = params.dup
|
116
|
-
dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.topic)
|
117
|
+
dupped = params.metadata.dup
|
118
|
+
dupped['topic'] = Karafka::App.config.topic_mapper.outgoing(params.metadata.topic)
|
117
119
|
[dupped]
|
118
120
|
end
|
119
121
|
|
@@ -23,7 +23,11 @@ module Karafka
|
|
23
23
|
) do
|
24
24
|
# Due to how ruby-kafka is built, we have the metadata that is stored on the batch
|
25
25
|
# level only available for batch consuming
|
26
|
-
consumer.
|
26
|
+
consumer.batch_metadata = Params::Builders::BatchMetadata.from_kafka_batch(
|
27
|
+
kafka_batch,
|
28
|
+
topic
|
29
|
+
)
|
30
|
+
|
27
31
|
kafka_messages = kafka_batch.messages
|
28
32
|
|
29
33
|
# Depending on a case (persisted or not) we might use new consumer instance per
|
@@ -16,7 +16,7 @@ module Karafka
|
|
16
16
|
|
17
17
|
bind_backend(consumer, topic)
|
18
18
|
bind_params(consumer, topic)
|
19
|
-
|
19
|
+
bind_batch_metadata(consumer, topic)
|
20
20
|
bind_responders(consumer, topic)
|
21
21
|
end
|
22
22
|
|
@@ -40,13 +40,14 @@ module Karafka
|
|
40
40
|
consumer.extend(SingleParams)
|
41
41
|
end
|
42
42
|
|
43
|
-
# Adds an option to work with metadata for consumer instances that have
|
43
|
+
# Adds an option to work with batch metadata for consumer instances that have
|
44
|
+
# batch fetching enabled
|
44
45
|
# @param consumer [Karafka::BaseConsumer] consumer instance
|
45
46
|
# @param topic [Karafka::Routing::Topic] topic of a consumer class
|
46
|
-
def
|
47
|
+
def bind_batch_metadata(consumer, topic)
|
47
48
|
return unless topic.batch_fetching
|
48
49
|
|
49
|
-
consumer.extend(
|
50
|
+
consumer.extend(BatchMetadata)
|
50
51
|
end
|
51
52
|
|
52
53
|
# Adds responders support for topics and consumers with responders defined for them
|
@@ -43,7 +43,7 @@ module Karafka
|
|
43
43
|
# so it returns a topic as a string, not a routing topic
|
44
44
|
debug(
|
45
45
|
<<~MSG.chomp.tr("\n", ' ')
|
46
|
-
Params deserialization for #{event[:caller].topic} topic
|
46
|
+
Params deserialization for #{event[:caller].metadata.topic} topic
|
47
47
|
successful in #{event[:time]} ms
|
48
48
|
MSG
|
49
49
|
)
|
@@ -52,7 +52,9 @@ module Karafka
|
|
52
52
|
# Logs unsuccessful deserialization attempts of incoming data
|
53
53
|
# @param event [Dry::Events::Event] event details including payload
|
54
54
|
def on_params_params_deserialize_error(event)
|
55
|
-
|
55
|
+
topic = event[:caller].metadata.topic
|
56
|
+
error = event[:error]
|
57
|
+
error "Params deserialization error for #{topic} topic: #{error}"
|
56
58
|
end
|
57
59
|
|
58
60
|
# Logs errors that occurred in a listener fetch loop
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Params
|
5
|
+
# Simple batch metadata object that stores all non-message information received from Kafka
|
6
|
+
# cluster while fetching the data
|
7
|
+
# @note This metadata object refers to per batch metadata, not `#params.metadata`
|
8
|
+
BatchMetadata = Struct.new(
|
9
|
+
:batch_size,
|
10
|
+
:first_offset,
|
11
|
+
:highwater_mark_offset,
|
12
|
+
:unknown_last_offset,
|
13
|
+
:last_offset,
|
14
|
+
:offset_lag,
|
15
|
+
:deserializer,
|
16
|
+
:partition,
|
17
|
+
:topic,
|
18
|
+
keyword_init: true
|
19
|
+
) do
|
20
|
+
# @return [Boolean] is the last offset known or unknown
|
21
|
+
def unknown_last_offset?
|
22
|
+
unknown_last_offset
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Karafka
|
4
|
+
module Params
|
5
|
+
module Builders
|
6
|
+
# Builder for creating batch metadata object based on the batch informations
|
7
|
+
module BatchMetadata
|
8
|
+
class << self
|
9
|
+
# Creates metadata based on the kafka batch data
|
10
|
+
# @param kafka_batch [Kafka::FetchedBatch] kafka batch details
|
11
|
+
# @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
|
12
|
+
# @return [Karafka::Params::BatchMetadata] batch metadata object
|
13
|
+
def from_kafka_batch(kafka_batch, topic)
|
14
|
+
Karafka::Params::BatchMetadata.new(
|
15
|
+
batch_size: kafka_batch.messages.count,
|
16
|
+
first_offset: kafka_batch.first_offset,
|
17
|
+
highwater_mark_offset: kafka_batch.highwater_mark_offset,
|
18
|
+
unknown_last_offset: kafka_batch.unknown_last_offset?,
|
19
|
+
last_offset: kafka_batch.last_offset,
|
20
|
+
offset_lag: kafka_batch.offset_lag,
|
21
|
+
deserializer: topic.deserializer,
|
22
|
+
partition: kafka_batch.partition,
|
23
|
+
topic: topic.name
|
24
|
+
).freeze
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -12,22 +12,24 @@ module Karafka
|
|
12
12
|
class << self
|
13
13
|
# @param kafka_message [Kafka::FetchedMessage] message fetched from Kafka
|
14
14
|
# @param topic [Karafka::Routing::Topic] topic for which this message was fetched
|
15
|
-
# @return [Karafka::Params::Params] params object
|
15
|
+
# @return [Karafka::Params::Params] params object with payload and message metadata
|
16
16
|
def from_kafka_message(kafka_message, topic)
|
17
|
-
Karafka::Params::
|
18
|
-
.
|
19
|
-
.
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
17
|
+
metadata = Karafka::Params::Metadata.new(
|
18
|
+
create_time: kafka_message.create_time,
|
19
|
+
headers: kafka_message.headers || {},
|
20
|
+
is_control_record: kafka_message.is_control_record,
|
21
|
+
key: kafka_message.key,
|
22
|
+
offset: kafka_message.offset,
|
23
|
+
deserializer: topic.deserializer,
|
24
|
+
partition: kafka_message.partition,
|
25
|
+
receive_time: Time.now,
|
26
|
+
topic: topic.name
|
27
|
+
).freeze
|
28
|
+
|
29
|
+
Karafka::Params::Params.new(
|
30
|
+
kafka_message.value,
|
31
|
+
metadata
|
32
|
+
)
|
31
33
|
end
|
32
34
|
end
|
33
35
|
end
|
@@ -12,11 +12,11 @@ module Karafka
|
|
12
12
|
# @param topic [Karafka::Routing::Topic] topic for which we're received messages
|
13
13
|
# @return [Karafka::Params::ParamsBatch<Karafka::Params::Params>] batch with params
|
14
14
|
def from_kafka_messages(kafka_messages, topic)
|
15
|
-
params_array = kafka_messages.map
|
15
|
+
params_array = kafka_messages.map do |message|
|
16
16
|
Karafka::Params::Builders::Params.from_kafka_message(message, topic)
|
17
17
|
end
|
18
18
|
|
19
|
-
Karafka::Params::ParamsBatch.new(params_array)
|
19
|
+
Karafka::Params::ParamsBatch.new(params_array).freeze
|
20
20
|
end
|
21
21
|
end
|
22
22
|
end
|
@@ -2,34 +2,19 @@
|
|
2
2
|
|
3
3
|
module Karafka
|
4
4
|
module Params
|
5
|
-
#
|
6
|
-
#
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
private_constant :METHOD_ATTRIBUTES
|
21
|
-
|
22
|
-
METHOD_ATTRIBUTES.each do |attr|
|
23
|
-
# Defines a method call accessor to a particular hash field.
|
24
|
-
define_method(attr) do
|
25
|
-
self[attr]
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
# @return [Boolean] is the last offset known or unknown
|
30
|
-
def unknown_last_offset?
|
31
|
-
self['unknown_last_offset']
|
32
|
-
end
|
33
|
-
end
|
5
|
+
# Single message / params metadata details that can be accessed without the need for the
|
6
|
+
# payload deserialization
|
7
|
+
Metadata = Struct.new(
|
8
|
+
:create_time,
|
9
|
+
:headers,
|
10
|
+
:is_control_record,
|
11
|
+
:key,
|
12
|
+
:offset,
|
13
|
+
:deserializer,
|
14
|
+
:partition,
|
15
|
+
:receive_time,
|
16
|
+
:topic,
|
17
|
+
keyword_init: true
|
18
|
+
)
|
34
19
|
end
|
35
20
|
end
|
@@ -6,68 +6,40 @@ module Karafka
|
|
6
6
|
# It provides lazy loading not only until the first usage, but also allows us to skip
|
7
7
|
# using deserializer until we execute our logic. That way we can operate with
|
8
8
|
# heavy-deserialization data without slowing down the whole application.
|
9
|
-
class Params
|
10
|
-
|
11
|
-
|
12
|
-
#
|
13
|
-
#
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
key
|
20
|
-
offset
|
21
|
-
deserializer
|
22
|
-
deserialized
|
23
|
-
partition
|
24
|
-
receive_time
|
25
|
-
topic
|
26
|
-
payload
|
27
|
-
].freeze
|
28
|
-
|
29
|
-
private_constant :METHOD_ATTRIBUTES
|
30
|
-
|
31
|
-
METHOD_ATTRIBUTES.each do |attr|
|
32
|
-
# Defines a method call accessor to a particular hash field.
|
33
|
-
# @note Won't work for complex key names that contain spaces, etc
|
34
|
-
# @param key [Symbol] name of a field that we want to retrieve with a method call
|
35
|
-
# @example
|
36
|
-
# key_attr_reader :example
|
37
|
-
# params.example #=> 'my example payload'
|
38
|
-
define_method(attr) do
|
39
|
-
self[attr]
|
40
|
-
end
|
9
|
+
class Params
|
10
|
+
attr_reader :raw_payload, :metadata
|
11
|
+
|
12
|
+
# @param raw_payload [Object] incoming payload before deserialization
|
13
|
+
# @param metadata [Karafka::Params::Metadata] message metadata object
|
14
|
+
def initialize(raw_payload, metadata)
|
15
|
+
@raw_payload = raw_payload
|
16
|
+
@metadata = metadata
|
17
|
+
@deserialized = false
|
18
|
+
@payload = nil
|
41
19
|
end
|
42
20
|
|
43
|
-
#
|
44
|
-
|
45
|
-
|
46
|
-
# @return [Object] content of a given params key
|
47
|
-
def [](key)
|
48
|
-
# Payload will be deserialized only when we request for it.
|
49
|
-
deserialize! if key == 'payload'
|
50
|
-
super
|
51
|
-
end
|
21
|
+
# @return [Object] lazy-deserialized data (deserialized upon first request)
|
22
|
+
def payload
|
23
|
+
return @payload if deserialized?
|
52
24
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
25
|
+
@payload = deserialize
|
26
|
+
# We mark deserialization as successful after deserialization, as in case of an error
|
27
|
+
# this won't be falsely set to true
|
28
|
+
@deserialized = true
|
29
|
+
@payload
|
30
|
+
end
|
59
31
|
|
60
|
-
|
61
|
-
|
62
|
-
|
32
|
+
# @return [Boolean] did given params payload were deserialized already
|
33
|
+
def deserialized?
|
34
|
+
@deserialized
|
63
35
|
end
|
64
36
|
|
65
37
|
private
|
66
38
|
|
67
|
-
# @return [Object]
|
39
|
+
# @return [Object] tries de-serializes data
|
68
40
|
def deserialize
|
69
41
|
Karafka.monitor.instrument('params.params.deserialize', caller: self) do
|
70
|
-
|
42
|
+
metadata.deserializer.call(self)
|
71
43
|
end
|
72
44
|
rescue ::StandardError => e
|
73
45
|
Karafka.monitor.instrument('params.params.deserialize.error', caller: self, error: e)
|
@@ -15,47 +15,46 @@ module Karafka
|
|
15
15
|
@params_array = params_array
|
16
16
|
end
|
17
17
|
|
18
|
-
# @yieldparam [Karafka::Params::Params] each
|
19
|
-
# @note Invocation of this method will cause loading and deserializing each param after
|
20
|
-
# another.
|
21
|
-
# directly
|
18
|
+
# @yieldparam [Karafka::Params::Params] each params instance
|
19
|
+
# @note Invocation of this method will not cause loading and deserializing each param after
|
20
|
+
# another.
|
22
21
|
def each
|
23
|
-
@params_array.each { |param| yield(param
|
22
|
+
@params_array.each { |param| yield(param) }
|
24
23
|
end
|
25
24
|
|
26
25
|
# @return [Array<Karafka::Params::Params>] returns all the params in a loaded state, so they
|
27
26
|
# can be used for batch insert, etc. Without invoking all, up until first use, they won't
|
28
27
|
# be deserialized
|
29
28
|
def deserialize!
|
30
|
-
each(&:
|
29
|
+
each(&:payload)
|
31
30
|
end
|
32
31
|
|
33
32
|
# @return [Array<Object>] array with deserialized payloads. This method can be useful when
|
34
33
|
# we don't care about metadata and just want to extract all the data payloads from the
|
35
34
|
# batch
|
36
35
|
def payloads
|
37
|
-
|
36
|
+
map(&:payload)
|
38
37
|
end
|
39
38
|
|
40
|
-
# @return [Karafka::Params::Params] first element
|
39
|
+
# @return [Karafka::Params::Params] first element
|
41
40
|
def first
|
42
|
-
@params_array.first
|
41
|
+
@params_array.first
|
43
42
|
end
|
44
43
|
|
45
|
-
# @return [Karafka::Params::Params] last element
|
44
|
+
# @return [Karafka::Params::Params] last element
|
46
45
|
def last
|
47
|
-
@params_array.last
|
48
|
-
end
|
49
|
-
|
50
|
-
# @return [Array<Karafka::Params::Params>] pure array with params (not deserialized)
|
51
|
-
def to_a
|
52
|
-
@params_array
|
46
|
+
@params_array.last
|
53
47
|
end
|
54
48
|
|
55
49
|
# @return [Integer] number of messages in the batch
|
56
50
|
def size
|
57
51
|
@params_array.size
|
58
52
|
end
|
53
|
+
|
54
|
+
# @return [Array<Karafka::Params::Params>] pure array with params
|
55
|
+
def to_a
|
56
|
+
@params_array
|
57
|
+
end
|
59
58
|
end
|
60
59
|
end
|
61
60
|
end
|
@@ -17,8 +17,8 @@ module Karafka
|
|
17
17
|
# }
|
18
18
|
# Deserializer.call(params) #=> { 'a' => 1 }
|
19
19
|
def call(params)
|
20
|
-
::
|
21
|
-
rescue ::
|
20
|
+
params.raw_payload.nil? ? nil : ::JSON.parse(params.raw_payload)
|
21
|
+
rescue ::JSON::ParserError => e
|
22
22
|
raise ::Karafka::Errors::DeserializationError, e
|
23
23
|
end
|
24
24
|
end
|
data/lib/karafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: karafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0.pre.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -36,7 +36,7 @@ cert_chain:
|
|
36
36
|
2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
|
37
37
|
nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
|
38
38
|
-----END CERTIFICATE-----
|
39
|
-
date: 2020-08-
|
39
|
+
date: 2020-08-25 00:00:00.000000000 Z
|
40
40
|
dependencies:
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: dry-configurable
|
@@ -122,20 +122,6 @@ dependencies:
|
|
122
122
|
- - "~>"
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '1.0'
|
125
|
-
- !ruby/object:Gem::Dependency
|
126
|
-
name: multi_json
|
127
|
-
requirement: !ruby/object:Gem::Requirement
|
128
|
-
requirements:
|
129
|
-
- - ">="
|
130
|
-
- !ruby/object:Gem::Version
|
131
|
-
version: '1.12'
|
132
|
-
type: :runtime
|
133
|
-
prerelease: false
|
134
|
-
version_requirements: !ruby/object:Gem::Requirement
|
135
|
-
requirements:
|
136
|
-
- - ">="
|
137
|
-
- !ruby/object:Gem::Version
|
138
|
-
version: '1.12'
|
139
125
|
- !ruby/object:Gem::Dependency
|
140
126
|
name: rake
|
141
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -184,14 +170,14 @@ dependencies:
|
|
184
170
|
requirements:
|
185
171
|
- - "~>"
|
186
172
|
- !ruby/object:Gem::Version
|
187
|
-
version: 1.
|
173
|
+
version: 1.4.0
|
188
174
|
type: :runtime
|
189
175
|
prerelease: false
|
190
176
|
version_requirements: !ruby/object:Gem::Requirement
|
191
177
|
requirements:
|
192
178
|
- - "~>"
|
193
179
|
- !ruby/object:Gem::Version
|
194
|
-
version: 1.
|
180
|
+
version: 1.4.0
|
195
181
|
- !ruby/object:Gem::Dependency
|
196
182
|
name: zeitwerk
|
197
183
|
requirement: !ruby/object:Gem::Requirement
|
@@ -259,9 +245,9 @@ files:
|
|
259
245
|
- lib/karafka/connection/client.rb
|
260
246
|
- lib/karafka/connection/listener.rb
|
261
247
|
- lib/karafka/connection/message_delegator.rb
|
248
|
+
- lib/karafka/consumers/batch_metadata.rb
|
262
249
|
- lib/karafka/consumers/callbacks.rb
|
263
250
|
- lib/karafka/consumers/includer.rb
|
264
|
-
- lib/karafka/consumers/metadata.rb
|
265
251
|
- lib/karafka/consumers/responders.rb
|
266
252
|
- lib/karafka/consumers/single_params.rb
|
267
253
|
- lib/karafka/contracts.rb
|
@@ -280,7 +266,8 @@ files:
|
|
280
266
|
- lib/karafka/instrumentation/monitor.rb
|
281
267
|
- lib/karafka/instrumentation/proctitle_listener.rb
|
282
268
|
- lib/karafka/instrumentation/stdout_listener.rb
|
283
|
-
- lib/karafka/params/
|
269
|
+
- lib/karafka/params/batch_metadata.rb
|
270
|
+
- lib/karafka/params/builders/batch_metadata.rb
|
284
271
|
- lib/karafka/params/builders/params.rb
|
285
272
|
- lib/karafka/params/builders/params_batch.rb
|
286
273
|
- lib/karafka/params/metadata.rb
|
@@ -327,11 +314,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
327
314
|
version: 2.5.0
|
328
315
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
329
316
|
requirements:
|
330
|
-
- - "
|
317
|
+
- - ">"
|
331
318
|
- !ruby/object:Gem::Version
|
332
|
-
version:
|
319
|
+
version: 1.3.1
|
333
320
|
requirements: []
|
334
|
-
rubygems_version: 3.1.
|
321
|
+
rubygems_version: 3.1.4
|
335
322
|
signing_key:
|
336
323
|
specification_version: 4
|
337
324
|
summary: Ruby based framework for working with Apache Kafka
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,33 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Karafka
|
4
|
-
module Params
|
5
|
-
module Builders
|
6
|
-
# Builder for creating metadata object based on the message or batch informations
|
7
|
-
# @note We have 2 ways of creating metadata based on the way ruby-kafka operates
|
8
|
-
module Metadata
|
9
|
-
class << self
|
10
|
-
# Creates metadata based on the kafka batch data
|
11
|
-
# @param kafka_batch [Kafka::FetchedBatch] kafka batch details
|
12
|
-
# @param topic [Karafka::Routing::Topic] topic for which we've fetched the batch
|
13
|
-
# @return [Karafka::Params::Metadata] metadata object
|
14
|
-
def from_kafka_batch(kafka_batch, topic)
|
15
|
-
Karafka::Params::Metadata
|
16
|
-
.new
|
17
|
-
.merge!(
|
18
|
-
'batch_size' => kafka_batch.messages.count,
|
19
|
-
'first_offset' => kafka_batch.first_offset,
|
20
|
-
'highwater_mark_offset' => kafka_batch.highwater_mark_offset,
|
21
|
-
'last_offset' => kafka_batch.last_offset,
|
22
|
-
'offset_lag' => kafka_batch.offset_lag,
|
23
|
-
'deserializer' => topic.deserializer,
|
24
|
-
'partition' => kafka_batch.partition,
|
25
|
-
'topic' => kafka_batch.topic,
|
26
|
-
'unknown_last_offset' => kafka_batch.unknown_last_offset?
|
27
|
-
)
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|