waterdrop 2.0.7 → 2.6.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/.github/FUNDING.yml +1 -0
- data/.github/workflows/ci.yml +22 -11
- data/.ruby-version +1 -1
- data/CHANGELOG.md +200 -0
- data/Gemfile +0 -2
- data/Gemfile.lock +32 -75
- data/README.md +22 -275
- data/certs/cert_chain.pem +26 -0
- data/config/locales/errors.yml +33 -0
- data/docker-compose.yml +19 -12
- data/lib/waterdrop/clients/buffered.rb +90 -0
- data/lib/waterdrop/clients/dummy.rb +69 -0
- data/lib/waterdrop/clients/rdkafka.rb +34 -0
- data/lib/{water_drop → waterdrop}/config.rb +39 -16
- data/lib/waterdrop/contracts/config.rb +43 -0
- data/lib/waterdrop/contracts/message.rb +64 -0
- data/lib/{water_drop → waterdrop}/errors.rb +14 -7
- data/lib/waterdrop/instrumentation/callbacks/delivery.rb +102 -0
- data/lib/{water_drop → waterdrop}/instrumentation/callbacks/error.rb +6 -2
- data/lib/{water_drop → waterdrop}/instrumentation/callbacks/statistics.rb +1 -1
- data/lib/{water_drop/instrumentation/stdout_listener.rb → waterdrop/instrumentation/logger_listener.rb} +66 -21
- data/lib/waterdrop/instrumentation/monitor.rb +20 -0
- data/lib/{water_drop/instrumentation/monitor.rb → waterdrop/instrumentation/notifications.rb} +12 -14
- data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json +1 -0
- data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb +210 -0
- data/lib/waterdrop/middleware.rb +50 -0
- data/lib/{water_drop → waterdrop}/producer/async.rb +40 -4
- data/lib/{water_drop → waterdrop}/producer/buffer.rb +12 -30
- data/lib/{water_drop → waterdrop}/producer/builder.rb +6 -11
- data/lib/{water_drop → waterdrop}/producer/sync.rb +44 -15
- data/lib/waterdrop/producer/transactions.rb +170 -0
- data/lib/waterdrop/producer.rb +308 -0
- data/lib/{water_drop → waterdrop}/version.rb +1 -1
- data/lib/waterdrop.rb +28 -2
- data/renovate.json +6 -0
- data/waterdrop.gemspec +14 -11
- data.tar.gz.sig +0 -0
- metadata +71 -111
- metadata.gz.sig +0 -0
- data/certs/mensfeld.pem +0 -25
- data/config/errors.yml +0 -6
- data/lib/water_drop/contracts/config.rb +0 -26
- data/lib/water_drop/contracts/message.rb +0 -42
- data/lib/water_drop/instrumentation/callbacks/delivery.rb +0 -30
- data/lib/water_drop/instrumentation/callbacks/statistics_decorator.rb +0 -77
- data/lib/water_drop/instrumentation/callbacks_manager.rb +0 -39
- data/lib/water_drop/instrumentation.rb +0 -20
- data/lib/water_drop/patches/rdkafka/bindings.rb +0 -42
- data/lib/water_drop/patches/rdkafka/producer.rb +0 -20
- data/lib/water_drop/producer/dummy_client.rb +0 -32
- data/lib/water_drop/producer.rb +0 -162
- data/lib/water_drop.rb +0 -36
- /data/lib/{water_drop → waterdrop}/contracts.rb +0 -0
- /data/lib/{water_drop → waterdrop}/producer/status.rb +0 -0
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: waterdrop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.6.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -10,102 +10,53 @@ bindir: bin
|
|
10
10
|
cert_chain:
|
11
11
|
- |
|
12
12
|
-----BEGIN CERTIFICATE-----
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
/
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
13
|
+
MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
|
14
|
+
YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
|
15
|
+
MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
|
16
|
+
dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
|
17
|
+
bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
|
18
|
+
8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
|
19
|
+
sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
|
20
|
+
wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
|
21
|
+
b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
|
22
|
+
ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
|
23
|
+
5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
|
24
|
+
YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
|
25
|
+
kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
|
26
|
+
HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
|
27
|
+
1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
|
28
|
+
bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
|
29
|
+
bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
|
30
|
+
3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
|
31
|
+
kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
|
32
|
+
IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
|
33
|
+
fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
|
34
|
+
qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
|
35
|
+
AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
|
36
|
+
msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
|
36
37
|
-----END CERTIFICATE-----
|
37
|
-
date:
|
38
|
+
date: 2023-10-25 00:00:00.000000000 Z
|
38
39
|
dependencies:
|
39
40
|
- !ruby/object:Gem::Dependency
|
40
|
-
name:
|
41
|
+
name: karafka-core
|
41
42
|
requirement: !ruby/object:Gem::Requirement
|
42
43
|
requirements:
|
43
44
|
- - ">="
|
44
45
|
- !ruby/object:Gem::Version
|
45
|
-
version:
|
46
|
-
|
47
|
-
prerelease: false
|
48
|
-
version_requirements: !ruby/object:Gem::Requirement
|
49
|
-
requirements:
|
50
|
-
- - ">="
|
51
|
-
- !ruby/object:Gem::Version
|
52
|
-
version: '1.1'
|
53
|
-
- !ruby/object:Gem::Dependency
|
54
|
-
name: dry-configurable
|
55
|
-
requirement: !ruby/object:Gem::Requirement
|
56
|
-
requirements:
|
57
|
-
- - "~>"
|
58
|
-
- !ruby/object:Gem::Version
|
59
|
-
version: '0.13'
|
60
|
-
type: :runtime
|
61
|
-
prerelease: false
|
62
|
-
version_requirements: !ruby/object:Gem::Requirement
|
63
|
-
requirements:
|
64
|
-
- - "~>"
|
46
|
+
version: 2.2.3
|
47
|
+
- - "<"
|
65
48
|
- !ruby/object:Gem::Version
|
66
|
-
version:
|
67
|
-
- !ruby/object:Gem::Dependency
|
68
|
-
name: dry-monitor
|
69
|
-
requirement: !ruby/object:Gem::Requirement
|
70
|
-
requirements:
|
71
|
-
- - "~>"
|
72
|
-
- !ruby/object:Gem::Version
|
73
|
-
version: '0.5'
|
49
|
+
version: 3.0.0
|
74
50
|
type: :runtime
|
75
51
|
prerelease: false
|
76
52
|
version_requirements: !ruby/object:Gem::Requirement
|
77
|
-
requirements:
|
78
|
-
- - "~>"
|
79
|
-
- !ruby/object:Gem::Version
|
80
|
-
version: '0.5'
|
81
|
-
- !ruby/object:Gem::Dependency
|
82
|
-
name: dry-validation
|
83
|
-
requirement: !ruby/object:Gem::Requirement
|
84
|
-
requirements:
|
85
|
-
- - "~>"
|
86
|
-
- !ruby/object:Gem::Version
|
87
|
-
version: '1.7'
|
88
|
-
type: :runtime
|
89
|
-
prerelease: false
|
90
|
-
version_requirements: !ruby/object:Gem::Requirement
|
91
|
-
requirements:
|
92
|
-
- - "~>"
|
93
|
-
- !ruby/object:Gem::Version
|
94
|
-
version: '1.7'
|
95
|
-
- !ruby/object:Gem::Dependency
|
96
|
-
name: rdkafka
|
97
|
-
requirement: !ruby/object:Gem::Requirement
|
98
53
|
requirements:
|
99
54
|
- - ">="
|
100
55
|
- !ruby/object:Gem::Version
|
101
|
-
version:
|
102
|
-
|
103
|
-
prerelease: false
|
104
|
-
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
requirements:
|
106
|
-
- - ">="
|
56
|
+
version: 2.2.3
|
57
|
+
- - "<"
|
107
58
|
- !ruby/object:Gem::Version
|
108
|
-
version:
|
59
|
+
version: 3.0.0
|
109
60
|
- !ruby/object:Gem::Dependency
|
110
61
|
name: zeitwerk
|
111
62
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,13 +73,14 @@ dependencies:
|
|
122
73
|
version: '2.3'
|
123
74
|
description: Kafka messaging made easy!
|
124
75
|
email:
|
125
|
-
-
|
76
|
+
- contact@karafka.io
|
126
77
|
executables: []
|
127
78
|
extensions: []
|
128
79
|
extra_rdoc_files: []
|
129
80
|
files:
|
130
81
|
- ".coditsu/ci.yml"
|
131
82
|
- ".diffend.yml"
|
83
|
+
- ".github/FUNDING.yml"
|
132
84
|
- ".github/workflows/ci.yml"
|
133
85
|
- ".gitignore"
|
134
86
|
- ".rspec"
|
@@ -139,41 +91,49 @@ files:
|
|
139
91
|
- Gemfile.lock
|
140
92
|
- MIT-LICENSE
|
141
93
|
- README.md
|
142
|
-
- certs/
|
143
|
-
- config/errors.yml
|
94
|
+
- certs/cert_chain.pem
|
95
|
+
- config/locales/errors.yml
|
144
96
|
- docker-compose.yml
|
145
|
-
- lib/water_drop.rb
|
146
|
-
- lib/water_drop/config.rb
|
147
|
-
- lib/water_drop/contracts.rb
|
148
|
-
- lib/water_drop/contracts/config.rb
|
149
|
-
- lib/water_drop/contracts/message.rb
|
150
|
-
- lib/water_drop/errors.rb
|
151
|
-
- lib/water_drop/instrumentation.rb
|
152
|
-
- lib/water_drop/instrumentation/callbacks/delivery.rb
|
153
|
-
- lib/water_drop/instrumentation/callbacks/error.rb
|
154
|
-
- lib/water_drop/instrumentation/callbacks/statistics.rb
|
155
|
-
- lib/water_drop/instrumentation/callbacks/statistics_decorator.rb
|
156
|
-
- lib/water_drop/instrumentation/callbacks_manager.rb
|
157
|
-
- lib/water_drop/instrumentation/monitor.rb
|
158
|
-
- lib/water_drop/instrumentation/stdout_listener.rb
|
159
|
-
- lib/water_drop/patches/rdkafka/bindings.rb
|
160
|
-
- lib/water_drop/patches/rdkafka/producer.rb
|
161
|
-
- lib/water_drop/producer.rb
|
162
|
-
- lib/water_drop/producer/async.rb
|
163
|
-
- lib/water_drop/producer/buffer.rb
|
164
|
-
- lib/water_drop/producer/builder.rb
|
165
|
-
- lib/water_drop/producer/dummy_client.rb
|
166
|
-
- lib/water_drop/producer/status.rb
|
167
|
-
- lib/water_drop/producer/sync.rb
|
168
|
-
- lib/water_drop/version.rb
|
169
97
|
- lib/waterdrop.rb
|
98
|
+
- lib/waterdrop/clients/buffered.rb
|
99
|
+
- lib/waterdrop/clients/dummy.rb
|
100
|
+
- lib/waterdrop/clients/rdkafka.rb
|
101
|
+
- lib/waterdrop/config.rb
|
102
|
+
- lib/waterdrop/contracts.rb
|
103
|
+
- lib/waterdrop/contracts/config.rb
|
104
|
+
- lib/waterdrop/contracts/message.rb
|
105
|
+
- lib/waterdrop/errors.rb
|
106
|
+
- lib/waterdrop/instrumentation/callbacks/delivery.rb
|
107
|
+
- lib/waterdrop/instrumentation/callbacks/error.rb
|
108
|
+
- lib/waterdrop/instrumentation/callbacks/statistics.rb
|
109
|
+
- lib/waterdrop/instrumentation/logger_listener.rb
|
110
|
+
- lib/waterdrop/instrumentation/monitor.rb
|
111
|
+
- lib/waterdrop/instrumentation/notifications.rb
|
112
|
+
- lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
|
113
|
+
- lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb
|
114
|
+
- lib/waterdrop/middleware.rb
|
115
|
+
- lib/waterdrop/producer.rb
|
116
|
+
- lib/waterdrop/producer/async.rb
|
117
|
+
- lib/waterdrop/producer/buffer.rb
|
118
|
+
- lib/waterdrop/producer/builder.rb
|
119
|
+
- lib/waterdrop/producer/status.rb
|
120
|
+
- lib/waterdrop/producer/sync.rb
|
121
|
+
- lib/waterdrop/producer/transactions.rb
|
122
|
+
- lib/waterdrop/version.rb
|
170
123
|
- log/.gitkeep
|
124
|
+
- renovate.json
|
171
125
|
- waterdrop.gemspec
|
172
126
|
homepage: https://karafka.io
|
173
127
|
licenses:
|
174
128
|
- MIT
|
175
129
|
metadata:
|
130
|
+
funding_uri: https://karafka.io/#become-pro
|
131
|
+
homepage_uri: https://karafka.io
|
132
|
+
changelog_uri: https://karafka.io/docs/Changelog-WaterDrop
|
133
|
+
bug_tracker_uri: https://github.com/karafka/waterdrop/issues
|
176
134
|
source_code_uri: https://github.com/karafka/waterdrop
|
135
|
+
documentation_uri: https://karafka.io/docs/#waterdrop
|
136
|
+
rubygems_mfa_required: 'true'
|
177
137
|
post_install_message:
|
178
138
|
rdoc_options: []
|
179
139
|
require_paths:
|
@@ -182,14 +142,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
182
142
|
requirements:
|
183
143
|
- - ">="
|
184
144
|
- !ruby/object:Gem::Version
|
185
|
-
version:
|
145
|
+
version: '0'
|
186
146
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
187
147
|
requirements:
|
188
148
|
- - ">="
|
189
149
|
- !ruby/object:Gem::Version
|
190
150
|
version: '0'
|
191
151
|
requirements: []
|
192
|
-
rubygems_version: 3.
|
152
|
+
rubygems_version: 3.4.19
|
193
153
|
signing_key:
|
194
154
|
specification_version: 4
|
195
155
|
summary: Kafka messaging made easy!
|
metadata.gz.sig
CHANGED
Binary file
|
data/certs/mensfeld.pem
DELETED
@@ -1,25 +0,0 @@
|
|
1
|
-
-----BEGIN CERTIFICATE-----
|
2
|
-
MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
|
3
|
-
ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
|
4
|
-
NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
|
5
|
-
CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
|
6
|
-
ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
|
7
|
-
TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
|
8
|
-
1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
|
9
|
-
vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
|
10
|
-
qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
|
11
|
-
tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
|
12
|
-
g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
|
13
|
-
LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
|
14
|
-
BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
|
15
|
-
EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
|
16
|
-
c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
|
17
|
-
W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
|
18
|
-
yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
|
19
|
-
8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
|
20
|
-
/rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
|
21
|
-
fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
|
22
|
-
2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
|
23
|
-
R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
|
24
|
-
pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
|
25
|
-
-----END CERTIFICATE-----
|
data/config/errors.yml
DELETED
@@ -1,26 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Contracts
|
5
|
-
# Contract with validation rules for WaterDrop configuration details
|
6
|
-
class Config < Dry::Validation::Contract
|
7
|
-
# Ensure valid format of each seed broker so that rdkafka doesn't fail silently
|
8
|
-
SEED_BROKER_FORMAT_REGEXP = %r{\A([^:/,]+:[0-9]+)(,[^:/,]+:[0-9]+)*\z}.freeze
|
9
|
-
|
10
|
-
private_constant :SEED_BROKER_FORMAT_REGEXP
|
11
|
-
|
12
|
-
params do
|
13
|
-
required(:id).filled(:str?)
|
14
|
-
required(:logger).filled
|
15
|
-
required(:deliver).filled(:bool?)
|
16
|
-
required(:max_payload_size).filled(:int?, gteq?: 1)
|
17
|
-
required(:max_wait_timeout).filled(:number?, gteq?: 0)
|
18
|
-
required(:wait_timeout).filled(:number?, gt?: 0)
|
19
|
-
|
20
|
-
required(:kafka).schema do
|
21
|
-
required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Contracts
|
5
|
-
# Contract with validation rules for validating that all the message options that
|
6
|
-
# we provide to producer ale valid and usable
|
7
|
-
class Message < Dry::Validation::Contract
|
8
|
-
# Regex to check that topic has a valid format
|
9
|
-
TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
|
10
|
-
|
11
|
-
# Checks, that the given value is a string
|
12
|
-
STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
|
13
|
-
|
14
|
-
private_constant :TOPIC_REGEXP, :STRING_ASSERTION
|
15
|
-
|
16
|
-
config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
|
17
|
-
|
18
|
-
option :max_payload_size
|
19
|
-
|
20
|
-
params do
|
21
|
-
required(:topic).filled(:str?, format?: TOPIC_REGEXP)
|
22
|
-
required(:payload).filled(:str?)
|
23
|
-
optional(:key).maybe(:str?, :filled?)
|
24
|
-
optional(:partition).filled(:int?, gteq?: -1)
|
25
|
-
optional(:partition_key).maybe(:str?, :filled?)
|
26
|
-
optional(:timestamp).maybe { time? | int? }
|
27
|
-
optional(:headers).maybe(:hash?)
|
28
|
-
end
|
29
|
-
|
30
|
-
rule(:headers) do
|
31
|
-
next unless value.is_a?(Hash)
|
32
|
-
|
33
|
-
key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
|
34
|
-
key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
|
35
|
-
end
|
36
|
-
|
37
|
-
rule(:payload) do
|
38
|
-
key.failure(:max_payload_size) if value.bytesize > max_payload_size
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
@@ -1,30 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Instrumentation
|
5
|
-
module Callbacks
|
6
|
-
# Creates a callable that we want to run upon each message delivery or failure
|
7
|
-
#
|
8
|
-
# @note We don't have to provide client_name here as this callback is per client instance
|
9
|
-
class Delivery
|
10
|
-
# @param producer_id [String] id of the current producer
|
11
|
-
# @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
|
12
|
-
def initialize(producer_id, monitor)
|
13
|
-
@producer_id = producer_id
|
14
|
-
@monitor = monitor
|
15
|
-
end
|
16
|
-
|
17
|
-
# Emits delivery details to the monitor
|
18
|
-
# @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
|
19
|
-
def call(delivery_report)
|
20
|
-
@monitor.instrument(
|
21
|
-
'message.acknowledged',
|
22
|
-
producer_id: @producer_id,
|
23
|
-
offset: delivery_report.offset,
|
24
|
-
partition: delivery_report.partition
|
25
|
-
)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
@@ -1,77 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Instrumentation
|
5
|
-
module Callbacks
|
6
|
-
# Many of the librdkafka statistics are absolute values instead of a gauge.
|
7
|
-
# This means, that for example number of messages sent is an absolute growing value
|
8
|
-
# instead of being a value of messages sent from the last statistics report.
|
9
|
-
# This decorator calculates the diff against previously emited stats, so we get also
|
10
|
-
# the diff together with the original values
|
11
|
-
class StatisticsDecorator
|
12
|
-
def initialize
|
13
|
-
@previous = {}.freeze
|
14
|
-
end
|
15
|
-
|
16
|
-
# @param emited_stats [Hash] original emited statistics
|
17
|
-
# @return [Hash] emited statistics extended with the diff data
|
18
|
-
# @note We modify the emited statistics, instead of creating new. Since we don't expose
|
19
|
-
# any API to get raw data, users can just assume that the result of this decoration is
|
20
|
-
# the proper raw stats that they can use
|
21
|
-
def call(emited_stats)
|
22
|
-
diff(
|
23
|
-
@previous,
|
24
|
-
emited_stats
|
25
|
-
)
|
26
|
-
|
27
|
-
@previous = emited_stats
|
28
|
-
|
29
|
-
emited_stats.freeze
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
# Calculates the diff of the provided values and modifies in place the emited statistics
|
35
|
-
#
|
36
|
-
# @param previous [Object] previous value from the given scope in which
|
37
|
-
# we are
|
38
|
-
# @param current [Object] current scope from emitted statistics
|
39
|
-
# @return [Object] the diff if the values were numerics or the current scope
|
40
|
-
def diff(previous, current)
|
41
|
-
if current.is_a?(Hash)
|
42
|
-
# @note We cannot use #each_key as we modify the content of the current scope
|
43
|
-
# in place (in case it's a hash)
|
44
|
-
current.keys.each do |key|
|
45
|
-
append(
|
46
|
-
current,
|
47
|
-
key,
|
48
|
-
diff((previous || {})[key], (current || {})[key])
|
49
|
-
)
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
# Diff can be computed only for numerics
|
54
|
-
return current unless current.is_a?(Numeric)
|
55
|
-
# If there was no previous value, delta is always zero
|
56
|
-
return 0 unless previous
|
57
|
-
# Should never happen but just in case, a type changed in between stats
|
58
|
-
return current unless previous.is_a?(Numeric)
|
59
|
-
|
60
|
-
current - previous
|
61
|
-
end
|
62
|
-
|
63
|
-
# Appends the result of the diff to a given key as long as the result is numeric
|
64
|
-
#
|
65
|
-
# @param current [Hash] current scope
|
66
|
-
# @param key [Symbol] key based on which we were diffing
|
67
|
-
# @param result [Object] diff result
|
68
|
-
def append(current, key, result)
|
69
|
-
return unless result.is_a?(Numeric)
|
70
|
-
return if current.frozen?
|
71
|
-
|
72
|
-
current["#{key}_d"] = result
|
73
|
-
end
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
end
|
@@ -1,39 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Instrumentation
|
5
|
-
# This manager allows us to register multiple callbacks into a hook that is suppose to support
|
6
|
-
# a single callback
|
7
|
-
class CallbacksManager
|
8
|
-
# @return [::WaterDrop::Instrumentation::CallbacksManager]
|
9
|
-
def initialize
|
10
|
-
@callbacks = Concurrent::Hash.new
|
11
|
-
end
|
12
|
-
|
13
|
-
# Invokes all the callbacks registered one after another
|
14
|
-
#
|
15
|
-
# @param args [Object] any args that should go to the callbacks
|
16
|
-
# @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
|
17
|
-
# callbacks and add new at the same time. Since we don't know when and in what thread
|
18
|
-
# things are going to be added to the manager, we need to extract values into an array and
|
19
|
-
# run it. That way we can add new things the same time.
|
20
|
-
def call(*args)
|
21
|
-
@callbacks.values.each { |callback| callback.call(*args) }
|
22
|
-
end
|
23
|
-
|
24
|
-
# Adds a callback to the manager
|
25
|
-
#
|
26
|
-
# @param id [String] id of the callback (used when deleting it)
|
27
|
-
# @param callable [#call] object that responds to a `#call` method
|
28
|
-
def add(id, callable)
|
29
|
-
@callbacks[id] = callable
|
30
|
-
end
|
31
|
-
|
32
|
-
# Removes the callback from the manager
|
33
|
-
# @param id [String] id of the callback we want to remove
|
34
|
-
def delete(id)
|
35
|
-
@callbacks.delete(id)
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
@@ -1,20 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Namespace for all the things related with WaterDrop instrumentation process
|
5
|
-
module Instrumentation
|
6
|
-
class << self
|
7
|
-
# Builds a manager for statistics callbacks
|
8
|
-
# @return [WaterDrop::CallbacksManager]
|
9
|
-
def statistics_callbacks
|
10
|
-
@statistics_callbacks ||= CallbacksManager.new
|
11
|
-
end
|
12
|
-
|
13
|
-
# Builds a manager for error callbacks
|
14
|
-
# @return [WaterDrop::CallbacksManager]
|
15
|
-
def error_callbacks
|
16
|
-
@error_callbacks ||= CallbacksManager.new
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Patches
|
5
|
-
module Rdkafka
|
6
|
-
# Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
|
7
|
-
# to work with in a bit different way than rdkafka itself
|
8
|
-
module Bindings
|
9
|
-
class << self
|
10
|
-
# Add extra methods that we need
|
11
|
-
# @param mod [::Rdkafka::Bindings] rdkafka bindings module
|
12
|
-
def included(mod)
|
13
|
-
mod.attach_function :rd_kafka_name, [:pointer], :string
|
14
|
-
|
15
|
-
# Default rdkafka setup for errors doest not propagate client details, thus it always
|
16
|
-
# publishes all the stuff for all rdkafka instances. We change that by providing
|
17
|
-
# function that fetches the instance name, allowing us to have better notifications
|
18
|
-
mod.send(:remove_const, :ErrorCallback)
|
19
|
-
mod.const_set(:ErrorCallback, build_error_callback)
|
20
|
-
end
|
21
|
-
|
22
|
-
# @return [FFI::Function] overwritten callback function
|
23
|
-
def build_error_callback
|
24
|
-
FFI::Function.new(
|
25
|
-
:void, %i[pointer int string pointer]
|
26
|
-
) do |client_prr, err_code, reason, _opaque|
|
27
|
-
return nil unless ::Rdkafka::Config.error_callback
|
28
|
-
|
29
|
-
name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
|
30
|
-
|
31
|
-
error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
|
32
|
-
|
33
|
-
::Rdkafka::Config.error_callback.call(name, error)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
::Rdkafka::Bindings.include(::WaterDrop::Patches::Rdkafka::Bindings)
|
@@ -1,20 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Patches to external components
|
5
|
-
module Patches
|
6
|
-
# Rdkafka related patches
|
7
|
-
module Rdkafka
|
8
|
-
# Rdkafka::Producer patches
|
9
|
-
module Producer
|
10
|
-
# Adds a method that allows us to get the native kafka producer name
|
11
|
-
# @return [String] producer instance name
|
12
|
-
def name
|
13
|
-
::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer
|
@@ -1,32 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
class Producer
|
5
|
-
# A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
|
6
|
-
# want to dispatch anything to Kafka
|
7
|
-
class DummyClient
|
8
|
-
# @return [DummyClient] dummy instance
|
9
|
-
def initialize
|
10
|
-
@counter = -1
|
11
|
-
end
|
12
|
-
|
13
|
-
# Dummy method for returning the delivery report
|
14
|
-
# @param _args [Object] anything that the delivery handle accepts
|
15
|
-
# @return [::Rdkafka::Producer::DeliveryReport]
|
16
|
-
def wait(*_args)
|
17
|
-
::Rdkafka::Producer::DeliveryReport.new(0, @counter += 1)
|
18
|
-
end
|
19
|
-
|
20
|
-
# @param _args [Object] anything really, this dummy is suppose to support anything
|
21
|
-
def respond_to_missing?(*_args)
|
22
|
-
true
|
23
|
-
end
|
24
|
-
|
25
|
-
# @param _args [Object] anything really, this dummy is suppose to support anything
|
26
|
-
# @return [self] returns self for chaining cases
|
27
|
-
def method_missing(*_args)
|
28
|
-
self || super
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|