waterdrop 2.0.7 → 2.6.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +39 -13
- data/.ruby-version +1 -1
- data/CHANGELOG.md +212 -0
- data/Gemfile +0 -2
- data/Gemfile.lock +45 -75
- data/README.md +22 -275
- data/certs/cert_chain.pem +26 -0
- data/config/locales/errors.yml +39 -0
- data/docker-compose.yml +21 -12
- data/lib/waterdrop/clients/buffered.rb +95 -0
- data/lib/waterdrop/clients/dummy.rb +69 -0
- data/lib/waterdrop/clients/rdkafka.rb +34 -0
- data/lib/{water_drop → waterdrop}/config.rb +39 -16
- data/lib/waterdrop/contracts/config.rb +43 -0
- data/lib/waterdrop/contracts/message.rb +64 -0
- data/lib/waterdrop/contracts/transactional_offset.rb +21 -0
- data/lib/{water_drop → waterdrop}/errors.rb +23 -7
- data/lib/waterdrop/helpers/counter.rb +27 -0
- data/lib/waterdrop/instrumentation/callbacks/delivery.rb +106 -0
- data/lib/{water_drop → waterdrop}/instrumentation/callbacks/error.rb +6 -2
- data/lib/{water_drop → waterdrop}/instrumentation/callbacks/statistics.rb +1 -1
- data/lib/{water_drop/instrumentation/stdout_listener.rb → waterdrop/instrumentation/logger_listener.rb} +91 -21
- data/lib/waterdrop/instrumentation/monitor.rb +20 -0
- data/lib/{water_drop/instrumentation/monitor.rb → waterdrop/instrumentation/notifications.rb} +15 -14
- data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb +210 -0
- data/lib/waterdrop/middleware.rb +50 -0
- data/lib/{water_drop → waterdrop}/producer/async.rb +40 -4
- data/lib/{water_drop → waterdrop}/producer/buffer.rb +13 -31
- data/lib/{water_drop → waterdrop}/producer/builder.rb +6 -11
- data/lib/{water_drop → waterdrop}/producer/sync.rb +44 -15
- data/lib/waterdrop/producer/transactions.rb +219 -0
- data/lib/waterdrop/producer.rb +324 -0
- data/lib/{water_drop → waterdrop}/version.rb +1 -1
- data/lib/waterdrop.rb +27 -2
- data/renovate.json +6 -0
- data/waterdrop.gemspec +14 -11
- data.tar.gz.sig +0 -0
- metadata +73 -111
- metadata.gz.sig +0 -0
- data/certs/mensfeld.pem +0 -25
- data/config/errors.yml +0 -6
- data/lib/water_drop/contracts/config.rb +0 -26
- data/lib/water_drop/contracts/message.rb +0 -42
- data/lib/water_drop/instrumentation/callbacks/delivery.rb +0 -30
- data/lib/water_drop/instrumentation/callbacks/statistics_decorator.rb +0 -77
- data/lib/water_drop/instrumentation/callbacks_manager.rb +0 -39
- data/lib/water_drop/instrumentation.rb +0 -20
- data/lib/water_drop/patches/rdkafka/bindings.rb +0 -42
- data/lib/water_drop/patches/rdkafka/producer.rb +0 -20
- data/lib/water_drop/producer/dummy_client.rb +0 -32
- data/lib/water_drop/producer.rb +0 -162
- data/lib/water_drop.rb +0 -36
- /data/lib/{water_drop → waterdrop}/contracts.rb +0 -0
- /data/lib/{water_drop → waterdrop}/producer/status.rb +0 -0
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: waterdrop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -10,102 +10,53 @@ bindir: bin
|
|
10
10
|
cert_chain:
|
11
11
|
- |
|
12
12
|
-----BEGIN CERTIFICATE-----
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
/
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
13
|
+
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
14
|
+
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
15
|
+
MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
|
16
|
+
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
17
|
+
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
|
18
|
+
8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
|
19
|
+
sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
|
20
|
+
wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
|
21
|
+
b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
|
22
|
+
ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
|
23
|
+
5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
|
24
|
+
YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
|
25
|
+
kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
|
26
|
+
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
|
27
|
+
1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
28
|
+
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
|
29
|
+
bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
|
30
|
+
3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
|
31
|
+
kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
|
32
|
+
IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
|
33
|
+
fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
|
34
|
+
qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
|
35
|
+
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
|
+
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
36
37
|
-----END CERTIFICATE-----
|
37
|
-
date:
|
38
|
+
date: 2024-02-06 00:00:00.000000000 Z
|
38
39
|
dependencies:
|
39
40
|
- !ruby/object:Gem::Dependency
|
40
|
-
name:
|
41
|
+
name: karafka-core
|
41
42
|
requirement: !ruby/object:Gem::Requirement
|
42
43
|
requirements:
|
43
44
|
- - ">="
|
44
45
|
- !ruby/object:Gem::Version
|
45
|
-
version:
|
46
|
-
|
47
|
-
prerelease: false
|
48
|
-
version_requirements: !ruby/object:Gem::Requirement
|
49
|
-
requirements:
|
50
|
-
- - ">="
|
51
|
-
- !ruby/object:Gem::Version
|
52
|
-
version: '1.1'
|
53
|
-
- !ruby/object:Gem::Dependency
|
54
|
-
name: dry-configurable
|
55
|
-
requirement: !ruby/object:Gem::Requirement
|
56
|
-
requirements:
|
57
|
-
- - "~>"
|
58
|
-
- !ruby/object:Gem::Version
|
59
|
-
version: '0.13'
|
60
|
-
type: :runtime
|
61
|
-
prerelease: false
|
62
|
-
version_requirements: !ruby/object:Gem::Requirement
|
63
|
-
requirements:
|
64
|
-
- - "~>"
|
46
|
+
version: 2.2.3
|
47
|
+
- - "<"
|
65
48
|
- !ruby/object:Gem::Version
|
66
|
-
version:
|
67
|
-
- !ruby/object:Gem::Dependency
|
68
|
-
name: dry-monitor
|
69
|
-
requirement: !ruby/object:Gem::Requirement
|
70
|
-
requirements:
|
71
|
-
- - "~>"
|
72
|
-
- !ruby/object:Gem::Version
|
73
|
-
version: '0.5'
|
49
|
+
version: 3.0.0
|
74
50
|
type: :runtime
|
75
51
|
prerelease: false
|
76
52
|
version_requirements: !ruby/object:Gem::Requirement
|
77
|
-
requirements:
|
78
|
-
- - "~>"
|
79
|
-
- !ruby/object:Gem::Version
|
80
|
-
version: '0.5'
|
81
|
-
- !ruby/object:Gem::Dependency
|
82
|
-
name: dry-validation
|
83
|
-
requirement: !ruby/object:Gem::Requirement
|
84
|
-
requirements:
|
85
|
-
- - "~>"
|
86
|
-
- !ruby/object:Gem::Version
|
87
|
-
version: '1.7'
|
88
|
-
type: :runtime
|
89
|
-
prerelease: false
|
90
|
-
version_requirements: !ruby/object:Gem::Requirement
|
91
|
-
requirements:
|
92
|
-
- - "~>"
|
93
|
-
- !ruby/object:Gem::Version
|
94
|
-
version: '1.7'
|
95
|
-
- !ruby/object:Gem::Dependency
|
96
|
-
name: rdkafka
|
97
|
-
requirement: !ruby/object:Gem::Requirement
|
98
53
|
requirements:
|
99
54
|
- - ">="
|
100
55
|
- !ruby/object:Gem::Version
|
101
|
-
version:
|
102
|
-
|
103
|
-
prerelease: false
|
104
|
-
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
requirements:
|
106
|
-
- - ">="
|
56
|
+
version: 2.2.3
|
57
|
+
- - "<"
|
107
58
|
- !ruby/object:Gem::Version
|
108
|
-
version:
|
59
|
+
version: 3.0.0
|
109
60
|
- !ruby/object:Gem::Dependency
|
110
61
|
name: zeitwerk
|
111
62
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,13 +73,14 @@ dependencies:
|
|
122
73
|
version: '2.3'
|
123
74
|
description: Kafka messaging made easy!
|
124
75
|
email:
|
125
|
-
-
|
76
|
+
- contact@karafka.io
|
126
77
|
executables: []
|
127
78
|
extensions: []
|
128
79
|
extra_rdoc_files: []
|
129
80
|
files:
|
130
81
|
- ".coditsu/ci.yml"
|
131
82
|
- ".diffend.yml"
|
83
|
+
- ".github/FUNDING.yml"
|
132
84
|
- ".github/workflows/ci.yml"
|
133
85
|
- ".gitignore"
|
134
86
|
- ".rspec"
|
@@ -139,41 +91,51 @@ files:
|
|
139
91
|
- Gemfile.lock
|
140
92
|
- MIT-LICENSE
|
141
93
|
- README.md
|
142
|
-
- certs/
|
143
|
-
- config/errors.yml
|
94
|
+
- certs/cert_chain.pem
|
95
|
+
- config/locales/errors.yml
|
144
96
|
- docker-compose.yml
|
145
|
-
- lib/water_drop.rb
|
146
|
-
- lib/water_drop/config.rb
|
147
|
-
- lib/water_drop/contracts.rb
|
148
|
-
- lib/water_drop/contracts/config.rb
|
149
|
-
- lib/water_drop/contracts/message.rb
|
150
|
-
- lib/water_drop/errors.rb
|
151
|
-
- lib/water_drop/instrumentation.rb
|
152
|
-
- lib/water_drop/instrumentation/callbacks/delivery.rb
|
153
|
-
- lib/water_drop/instrumentation/callbacks/error.rb
|
154
|
-
- lib/water_drop/instrumentation/callbacks/statistics.rb
|
155
|
-
- lib/water_drop/instrumentation/callbacks/statistics_decorator.rb
|
156
|
-
- lib/water_drop/instrumentation/callbacks_manager.rb
|
157
|
-
- lib/water_drop/instrumentation/monitor.rb
|
158
|
-
- lib/water_drop/instrumentation/stdout_listener.rb
|
159
|
-
- lib/water_drop/patches/rdkafka/bindings.rb
|
160
|
-
- lib/water_drop/patches/rdkafka/producer.rb
|
161
|
-
- lib/water_drop/producer.rb
|
162
|
-
- lib/water_drop/producer/async.rb
|
163
|
-
- lib/water_drop/producer/buffer.rb
|
164
|
-
- lib/water_drop/producer/builder.rb
|
165
|
-
- lib/water_drop/producer/dummy_client.rb
|
166
|
-
- lib/water_drop/producer/status.rb
|
167
|
-
- lib/water_drop/producer/sync.rb
|
168
|
-
- lib/water_drop/version.rb
|
169
97
|
- lib/waterdrop.rb
|
98
|
+
- lib/waterdrop/clients/buffered.rb
|
99
|
+
- lib/waterdrop/clients/dummy.rb
|
100
|
+
- lib/waterdrop/clients/rdkafka.rb
|
101
|
+
- lib/waterdrop/config.rb
|
102
|
+
- lib/waterdrop/contracts.rb
|
103
|
+
- lib/waterdrop/contracts/config.rb
|
104
|
+
- lib/waterdrop/contracts/message.rb
|
105
|
+
- lib/waterdrop/contracts/transactional_offset.rb
|
106
|
+
- lib/waterdrop/errors.rb
|
107
|
+
- lib/waterdrop/helpers/counter.rb
|
108
|
+
- lib/waterdrop/instrumentation/callbacks/delivery.rb
|
109
|
+
- lib/waterdrop/instrumentation/callbacks/error.rb
|
110
|
+
- lib/waterdrop/instrumentation/callbacks/statistics.rb
|
111
|
+
- lib/waterdrop/instrumentation/logger_listener.rb
|
112
|
+
- lib/waterdrop/instrumentation/monitor.rb
|
113
|
+
- lib/waterdrop/instrumentation/notifications.rb
|
114
|
+
- lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
|
115
|
+
- lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb
|
116
|
+
- lib/waterdrop/middleware.rb
|
117
|
+
- lib/waterdrop/producer.rb
|
118
|
+
- lib/waterdrop/producer/async.rb
|
119
|
+
- lib/waterdrop/producer/buffer.rb
|
120
|
+
- lib/waterdrop/producer/builder.rb
|
121
|
+
- lib/waterdrop/producer/status.rb
|
122
|
+
- lib/waterdrop/producer/sync.rb
|
123
|
+
- lib/waterdrop/producer/transactions.rb
|
124
|
+
- lib/waterdrop/version.rb
|
170
125
|
- log/.gitkeep
|
126
|
+
- renovate.json
|
171
127
|
- waterdrop.gemspec
|
172
128
|
homepage: https://karafka.io
|
173
129
|
licenses:
|
174
130
|
- MIT
|
175
131
|
metadata:
|
132
|
+
funding_uri: https://karafka.io/#become-pro
|
133
|
+
homepage_uri: https://karafka.io
|
134
|
+
changelog_uri: https://karafka.io/docs/Changelog-WaterDrop
|
135
|
+
bug_tracker_uri: https://github.com/karafka/waterdrop/issues
|
176
136
|
source_code_uri: https://github.com/karafka/waterdrop
|
137
|
+
documentation_uri: https://karafka.io/docs/#waterdrop
|
138
|
+
rubygems_mfa_required: 'true'
|
177
139
|
post_install_message:
|
178
140
|
rdoc_options: []
|
179
141
|
require_paths:
|
@@ -182,14 +144,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
182
144
|
requirements:
|
183
145
|
- - ">="
|
184
146
|
- !ruby/object:Gem::Version
|
185
|
-
version:
|
147
|
+
version: '0'
|
186
148
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
187
149
|
requirements:
|
188
150
|
- - ">="
|
189
151
|
- !ruby/object:Gem::Version
|
190
152
|
version: '0'
|
191
153
|
requirements: []
|
192
|
-
rubygems_version: 3.
|
154
|
+
rubygems_version: 3.5.3
|
193
155
|
signing_key:
|
194
156
|
specification_version: 4
|
195
157
|
summary: Kafka messaging made easy!
|
metadata.gz.sig
CHANGED
Binary file
|
data/certs/mensfeld.pem
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
-----BEGIN CERTIFICATE-----
|
2
|
-
MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
|
3
|
-
ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
|
4
|
-
NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
|
5
|
-
CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
|
6
|
-
ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
|
7
|
-
TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
|
8
|
-
1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
|
9
|
-
vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
|
10
|
-
qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
|
11
|
-
tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
|
12
|
-
g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
|
13
|
-
LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
|
14
|
-
BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
|
15
|
-
EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
|
16
|
-
c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
|
17
|
-
W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
|
18
|
-
yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
|
19
|
-
8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
|
20
|
-
/rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
|
21
|
-
fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
|
22
|
-
2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
|
23
|
-
R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
|
24
|
-
pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
|
25
|
-
-----END CERTIFICATE-----
|
data/config/errors.yml
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Contracts
|
5
|
-
# Contract with validation rules for WaterDrop configuration details
|
6
|
-
class Config < Dry::Validation::Contract
|
7
|
-
# Ensure valid format of each seed broker so that rdkafka doesn't fail silently
|
8
|
-
SEED_BROKER_FORMAT_REGEXP = %r{\A([^:/,]+:[0-9]+)(,[^:/,]+:[0-9]+)*\z}.freeze
|
9
|
-
|
10
|
-
private_constant :SEED_BROKER_FORMAT_REGEXP
|
11
|
-
|
12
|
-
params do
|
13
|
-
required(:id).filled(:str?)
|
14
|
-
required(:logger).filled
|
15
|
-
required(:deliver).filled(:bool?)
|
16
|
-
required(:max_payload_size).filled(:int?, gteq?: 1)
|
17
|
-
required(:max_wait_timeout).filled(:number?, gteq?: 0)
|
18
|
-
required(:wait_timeout).filled(:number?, gt?: 0)
|
19
|
-
|
20
|
-
required(:kafka).schema do
|
21
|
-
required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Contracts
|
5
|
-
# Contract with validation rules for validating that all the message options that
|
6
|
-
# we provide to producer ale valid and usable
|
7
|
-
class Message < Dry::Validation::Contract
|
8
|
-
# Regex to check that topic has a valid format
|
9
|
-
TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
|
10
|
-
|
11
|
-
# Checks, that the given value is a string
|
12
|
-
STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
|
13
|
-
|
14
|
-
private_constant :TOPIC_REGEXP, :STRING_ASSERTION
|
15
|
-
|
16
|
-
config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
|
17
|
-
|
18
|
-
option :max_payload_size
|
19
|
-
|
20
|
-
params do
|
21
|
-
required(:topic).filled(:str?, format?: TOPIC_REGEXP)
|
22
|
-
required(:payload).filled(:str?)
|
23
|
-
optional(:key).maybe(:str?, :filled?)
|
24
|
-
optional(:partition).filled(:int?, gteq?: -1)
|
25
|
-
optional(:partition_key).maybe(:str?, :filled?)
|
26
|
-
optional(:timestamp).maybe { time? | int? }
|
27
|
-
optional(:headers).maybe(:hash?)
|
28
|
-
end
|
29
|
-
|
30
|
-
rule(:headers) do
|
31
|
-
next unless value.is_a?(Hash)
|
32
|
-
|
33
|
-
key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
|
34
|
-
key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
|
35
|
-
end
|
36
|
-
|
37
|
-
rule(:payload) do
|
38
|
-
key.failure(:max_payload_size) if value.bytesize > max_payload_size
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Instrumentation
|
5
|
-
module Callbacks
|
6
|
-
# Creates a callable that we want to run upon each message delivery or failure
|
7
|
-
#
|
8
|
-
# @note We don't have to provide client_name here as this callback is per client instance
|
9
|
-
class Delivery
|
10
|
-
# @param producer_id [String] id of the current producer
|
11
|
-
# @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
|
12
|
-
def initialize(producer_id, monitor)
|
13
|
-
@producer_id = producer_id
|
14
|
-
@monitor = monitor
|
15
|
-
end
|
16
|
-
|
17
|
-
# Emits delivery details to the monitor
|
18
|
-
# @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
|
19
|
-
def call(delivery_report)
|
20
|
-
@monitor.instrument(
|
21
|
-
'message.acknowledged',
|
22
|
-
producer_id: @producer_id,
|
23
|
-
offset: delivery_report.offset,
|
24
|
-
partition: delivery_report.partition
|
25
|
-
)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
@@ -1,77 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Instrumentation
|
5
|
-
module Callbacks
|
6
|
-
# Many of the librdkafka statistics are absolute values instead of a gauge.
|
7
|
-
# This means, that for example number of messages sent is an absolute growing value
|
8
|
-
# instead of being a value of messages sent from the last statistics report.
|
9
|
-
# This decorator calculates the diff against previously emited stats, so we get also
|
10
|
-
# the diff together with the original values
|
11
|
-
class StatisticsDecorator
|
12
|
-
def initialize
|
13
|
-
@previous = {}.freeze
|
14
|
-
end
|
15
|
-
|
16
|
-
# @param emited_stats [Hash] original emited statistics
|
17
|
-
# @return [Hash] emited statistics extended with the diff data
|
18
|
-
# @note We modify the emited statistics, instead of creating new. Since we don't expose
|
19
|
-
# any API to get raw data, users can just assume that the result of this decoration is
|
20
|
-
# the proper raw stats that they can use
|
21
|
-
def call(emited_stats)
|
22
|
-
diff(
|
23
|
-
@previous,
|
24
|
-
emited_stats
|
25
|
-
)
|
26
|
-
|
27
|
-
@previous = emited_stats
|
28
|
-
|
29
|
-
emited_stats.freeze
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
# Calculates the diff of the provided values and modifies in place the emited statistics
|
35
|
-
#
|
36
|
-
# @param previous [Object] previous value from the given scope in which
|
37
|
-
# we are
|
38
|
-
# @param current [Object] current scope from emitted statistics
|
39
|
-
# @return [Object] the diff if the values were numerics or the current scope
|
40
|
-
def diff(previous, current)
|
41
|
-
if current.is_a?(Hash)
|
42
|
-
# @note We cannot use #each_key as we modify the content of the current scope
|
43
|
-
# in place (in case it's a hash)
|
44
|
-
current.keys.each do |key|
|
45
|
-
append(
|
46
|
-
current,
|
47
|
-
key,
|
48
|
-
diff((previous || {})[key], (current || {})[key])
|
49
|
-
)
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
# Diff can be computed only for numerics
|
54
|
-
return current unless current.is_a?(Numeric)
|
55
|
-
# If there was no previous value, delta is always zero
|
56
|
-
return 0 unless previous
|
57
|
-
# Should never happen but just in case, a type changed in between stats
|
58
|
-
return current unless previous.is_a?(Numeric)
|
59
|
-
|
60
|
-
current - previous
|
61
|
-
end
|
62
|
-
|
63
|
-
# Appends the result of the diff to a given key as long as the result is numeric
|
64
|
-
#
|
65
|
-
# @param current [Hash] current scope
|
66
|
-
# @param key [Symbol] key based on which we were diffing
|
67
|
-
# @param result [Object] diff result
|
68
|
-
def append(current, key, result)
|
69
|
-
return unless result.is_a?(Numeric)
|
70
|
-
return if current.frozen?
|
71
|
-
|
72
|
-
current["#{key}_d"] = result
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Instrumentation
|
5
|
-
# This manager allows us to register multiple callbacks into a hook that is suppose to support
|
6
|
-
# a single callback
|
7
|
-
class CallbacksManager
|
8
|
-
# @return [::WaterDrop::Instrumentation::CallbacksManager]
|
9
|
-
def initialize
|
10
|
-
@callbacks = Concurrent::Hash.new
|
11
|
-
end
|
12
|
-
|
13
|
-
# Invokes all the callbacks registered one after another
|
14
|
-
#
|
15
|
-
# @param args [Object] any args that should go to the callbacks
|
16
|
-
# @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
|
17
|
-
# callbacks and add new at the same time. Since we don't know when and in what thread
|
18
|
-
# things are going to be added to the manager, we need to extract values into an array and
|
19
|
-
# run it. That way we can add new things the same time.
|
20
|
-
def call(*args)
|
21
|
-
@callbacks.values.each { |callback| callback.call(*args) }
|
22
|
-
end
|
23
|
-
|
24
|
-
# Adds a callback to the manager
|
25
|
-
#
|
26
|
-
# @param id [String] id of the callback (used when deleting it)
|
27
|
-
# @param callable [#call] object that responds to a `#call` method
|
28
|
-
def add(id, callable)
|
29
|
-
@callbacks[id] = callable
|
30
|
-
end
|
31
|
-
|
32
|
-
# Removes the callback from the manager
|
33
|
-
# @param id [String] id of the callback we want to remove
|
34
|
-
def delete(id)
|
35
|
-
@callbacks.delete(id)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
@@ -1,20 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Namespace for all the things related with WaterDrop instrumentation process
|
5
|
-
module Instrumentation
|
6
|
-
class << self
|
7
|
-
# Builds a manager for statistics callbacks
|
8
|
-
# @return [WaterDrop::CallbacksManager]
|
9
|
-
def statistics_callbacks
|
10
|
-
@statistics_callbacks ||= CallbacksManager.new
|
11
|
-
end
|
12
|
-
|
13
|
-
# Builds a manager for error callbacks
|
14
|
-
# @return [WaterDrop::CallbacksManager]
|
15
|
-
def error_callbacks
|
16
|
-
@error_callbacks ||= CallbacksManager.new
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Patches
|
5
|
-
module Rdkafka
|
6
|
-
# Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
|
7
|
-
# to work with in a bit different way than rdkafka itself
|
8
|
-
module Bindings
|
9
|
-
class << self
|
10
|
-
# Add extra methods that we need
|
11
|
-
# @param mod [::Rdkafka::Bindings] rdkafka bindings module
|
12
|
-
def included(mod)
|
13
|
-
mod.attach_function :rd_kafka_name, [:pointer], :string
|
14
|
-
|
15
|
-
# Default rdkafka setup for errors doest not propagate client details, thus it always
|
16
|
-
# publishes all the stuff for all rdkafka instances. We change that by providing
|
17
|
-
# function that fetches the instance name, allowing us to have better notifications
|
18
|
-
mod.send(:remove_const, :ErrorCallback)
|
19
|
-
mod.const_set(:ErrorCallback, build_error_callback)
|
20
|
-
end
|
21
|
-
|
22
|
-
# @return [FFI::Function] overwritten callback function
|
23
|
-
def build_error_callback
|
24
|
-
FFI::Function.new(
|
25
|
-
:void, %i[pointer int string pointer]
|
26
|
-
) do |client_prr, err_code, reason, _opaque|
|
27
|
-
return nil unless ::Rdkafka::Config.error_callback
|
28
|
-
|
29
|
-
name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
|
30
|
-
|
31
|
-
error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
32
|
-
|
33
|
-
::Rdkafka::Config.error_callback.call(name, error)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
::Rdkafka::Bindings.include(::WaterDrop::Patches::Rdkafka::Bindings)
|
@@ -1,20 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Patches to external components
|
5
|
-
module Patches
|
6
|
-
# Rdkafka related patches
|
7
|
-
module Rdkafka
|
8
|
-
# Rdkafka::Producer patches
|
9
|
-
module Producer
|
10
|
-
# Adds a method that allows us to get the native kafka producer name
|
11
|
-
# @return [String] producer instance name
|
12
|
-
def name
|
13
|
-
::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer
|
@@ -1,32 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
class Producer
|
5
|
-
# A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
|
6
|
-
# want to dispatch anything to Kafka
|
7
|
-
class DummyClient
|
8
|
-
# @return [DummyClient] dummy instance
|
9
|
-
def initialize
|
10
|
-
@counter = -1
|
11
|
-
end
|
12
|
-
|
13
|
-
# Dummy method for returning the delivery report
|
14
|
-
# @param _args [Object] anything that the delivery handle accepts
|
15
|
-
# @return [::Rdkafka::Producer::DeliveryReport]
|
16
|
-
def wait(*_args)
|
17
|
-
::Rdkafka::Producer::DeliveryReport.new(0, @counter += 1)
|
18
|
-
end
|
19
|
-
|
20
|
-
# @param _args [Object] anything really, this dummy is suppose to support anything
|
21
|
-
def respond_to_missing?(*_args)
|
22
|
-
true
|
23
|
-
end
|
24
|
-
|
25
|
-
# @param _args [Object] anything really, this dummy is suppose to support anything
|
26
|
-
# @return [self] returns self for chaining cases
|
27
|
-
def method_missing(*_args)
|
28
|
-
self || super
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|