waterdrop 2.0.7 → 2.6.11

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +22 -11
  5. data/.ruby-version +1 -1
  6. data/CHANGELOG.md +200 -0
  7. data/Gemfile +0 -2
  8. data/Gemfile.lock +32 -75
  9. data/README.md +22 -275
  10. data/certs/cert_chain.pem +26 -0
  11. data/config/locales/errors.yml +33 -0
  12. data/docker-compose.yml +19 -12
  13. data/lib/waterdrop/clients/buffered.rb +90 -0
  14. data/lib/waterdrop/clients/dummy.rb +69 -0
  15. data/lib/waterdrop/clients/rdkafka.rb +34 -0
  16. data/lib/{water_drop → waterdrop}/config.rb +39 -16
  17. data/lib/waterdrop/contracts/config.rb +43 -0
  18. data/lib/waterdrop/contracts/message.rb +64 -0
  19. data/lib/{water_drop → waterdrop}/errors.rb +14 -7
  20. data/lib/waterdrop/instrumentation/callbacks/delivery.rb +102 -0
  21. data/lib/{water_drop → waterdrop}/instrumentation/callbacks/error.rb +6 -2
  22. data/lib/{water_drop → waterdrop}/instrumentation/callbacks/statistics.rb +1 -1
  23. data/lib/{water_drop/instrumentation/stdout_listener.rb → waterdrop/instrumentation/logger_listener.rb} +66 -21
  24. data/lib/waterdrop/instrumentation/monitor.rb +20 -0
  25. data/lib/{water_drop/instrumentation/monitor.rb → waterdrop/instrumentation/notifications.rb} +12 -14
  26. data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json +1 -0
  27. data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb +210 -0
  28. data/lib/waterdrop/middleware.rb +50 -0
  29. data/lib/{water_drop → waterdrop}/producer/async.rb +40 -4
  30. data/lib/{water_drop → waterdrop}/producer/buffer.rb +12 -30
  31. data/lib/{water_drop → waterdrop}/producer/builder.rb +6 -11
  32. data/lib/{water_drop → waterdrop}/producer/sync.rb +44 -15
  33. data/lib/waterdrop/producer/transactions.rb +170 -0
  34. data/lib/waterdrop/producer.rb +308 -0
  35. data/lib/{water_drop → waterdrop}/version.rb +1 -1
  36. data/lib/waterdrop.rb +28 -2
  37. data/renovate.json +6 -0
  38. data/waterdrop.gemspec +14 -11
  39. data.tar.gz.sig +0 -0
  40. metadata +71 -111
  41. metadata.gz.sig +0 -0
  42. data/certs/mensfeld.pem +0 -25
  43. data/config/errors.yml +0 -6
  44. data/lib/water_drop/contracts/config.rb +0 -26
  45. data/lib/water_drop/contracts/message.rb +0 -42
  46. data/lib/water_drop/instrumentation/callbacks/delivery.rb +0 -30
  47. data/lib/water_drop/instrumentation/callbacks/statistics_decorator.rb +0 -77
  48. data/lib/water_drop/instrumentation/callbacks_manager.rb +0 -39
  49. data/lib/water_drop/instrumentation.rb +0 -20
  50. data/lib/water_drop/patches/rdkafka/bindings.rb +0 -42
  51. data/lib/water_drop/patches/rdkafka/producer.rb +0 -20
  52. data/lib/water_drop/producer/dummy_client.rb +0 -32
  53. data/lib/water_drop/producer.rb +0 -162
  54. data/lib/water_drop.rb +0 -36
  55. /data/lib/{water_drop → waterdrop}/contracts.rb +0 -0
  56. /data/lib/{water_drop → waterdrop}/producer/status.rb +0 -0
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: waterdrop
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.7
4
+ version: 2.6.11
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -10,102 +10,53 @@ bindir: bin
10
10
  cert_chain:
11
11
  - |
12
12
  -----BEGIN CERTIFICATE-----
13
- MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
14
- ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
15
- NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
16
- CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
17
- ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
18
- TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
19
- 1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
20
- vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
21
- qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
22
- tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
23
- g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
24
- LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
25
- BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
26
- EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
27
- c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
28
- W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
29
- yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
30
- 8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
31
- /rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
32
- fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
33
- 2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
34
- R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
35
- pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
13
+ MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
14
+ YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
15
+ MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
16
+ dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
17
+ bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
18
+ 8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
19
+ sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
20
+ wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
21
+ b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
22
+ ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
23
+ 5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
24
+ YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
25
+ kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
26
+ HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
27
+ 1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
28
+ bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
29
+ bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
30
+ 3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
31
+ kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
32
+ IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
33
+ fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
34
+ qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
35
+ AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
+ msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
36
37
  -----END CERTIFICATE-----
37
- date: 2021-12-03 00:00:00.000000000 Z
38
+ date: 2023-10-25 00:00:00.000000000 Z
38
39
  dependencies:
39
40
  - !ruby/object:Gem::Dependency
40
- name: concurrent-ruby
41
+ name: karafka-core
41
42
  requirement: !ruby/object:Gem::Requirement
42
43
  requirements:
43
44
  - - ">="
44
45
  - !ruby/object:Gem::Version
45
- version: '1.1'
46
- type: :runtime
47
- prerelease: false
48
- version_requirements: !ruby/object:Gem::Requirement
49
- requirements:
50
- - - ">="
51
- - !ruby/object:Gem::Version
52
- version: '1.1'
53
- - !ruby/object:Gem::Dependency
54
- name: dry-configurable
55
- requirement: !ruby/object:Gem::Requirement
56
- requirements:
57
- - - "~>"
58
- - !ruby/object:Gem::Version
59
- version: '0.13'
60
- type: :runtime
61
- prerelease: false
62
- version_requirements: !ruby/object:Gem::Requirement
63
- requirements:
64
- - - "~>"
46
+ version: 2.2.3
47
+ - - "<"
65
48
  - !ruby/object:Gem::Version
66
- version: '0.13'
67
- - !ruby/object:Gem::Dependency
68
- name: dry-monitor
69
- requirement: !ruby/object:Gem::Requirement
70
- requirements:
71
- - - "~>"
72
- - !ruby/object:Gem::Version
73
- version: '0.5'
49
+ version: 3.0.0
74
50
  type: :runtime
75
51
  prerelease: false
76
52
  version_requirements: !ruby/object:Gem::Requirement
77
- requirements:
78
- - - "~>"
79
- - !ruby/object:Gem::Version
80
- version: '0.5'
81
- - !ruby/object:Gem::Dependency
82
- name: dry-validation
83
- requirement: !ruby/object:Gem::Requirement
84
- requirements:
85
- - - "~>"
86
- - !ruby/object:Gem::Version
87
- version: '1.7'
88
- type: :runtime
89
- prerelease: false
90
- version_requirements: !ruby/object:Gem::Requirement
91
- requirements:
92
- - - "~>"
93
- - !ruby/object:Gem::Version
94
- version: '1.7'
95
- - !ruby/object:Gem::Dependency
96
- name: rdkafka
97
- requirement: !ruby/object:Gem::Requirement
98
53
  requirements:
99
54
  - - ">="
100
55
  - !ruby/object:Gem::Version
101
- version: '0.10'
102
- type: :runtime
103
- prerelease: false
104
- version_requirements: !ruby/object:Gem::Requirement
105
- requirements:
106
- - - ">="
56
+ version: 2.2.3
57
+ - - "<"
107
58
  - !ruby/object:Gem::Version
108
- version: '0.10'
59
+ version: 3.0.0
109
60
  - !ruby/object:Gem::Dependency
110
61
  name: zeitwerk
111
62
  requirement: !ruby/object:Gem::Requirement
@@ -122,13 +73,14 @@ dependencies:
122
73
  version: '2.3'
123
74
  description: Kafka messaging made easy!
124
75
  email:
125
- - maciej@mensfeld.pl
76
+ - contact@karafka.io
126
77
  executables: []
127
78
  extensions: []
128
79
  extra_rdoc_files: []
129
80
  files:
130
81
  - ".coditsu/ci.yml"
131
82
  - ".diffend.yml"
83
+ - ".github/FUNDING.yml"
132
84
  - ".github/workflows/ci.yml"
133
85
  - ".gitignore"
134
86
  - ".rspec"
@@ -139,41 +91,49 @@ files:
139
91
  - Gemfile.lock
140
92
  - MIT-LICENSE
141
93
  - README.md
142
- - certs/mensfeld.pem
143
- - config/errors.yml
94
+ - certs/cert_chain.pem
95
+ - config/locales/errors.yml
144
96
  - docker-compose.yml
145
- - lib/water_drop.rb
146
- - lib/water_drop/config.rb
147
- - lib/water_drop/contracts.rb
148
- - lib/water_drop/contracts/config.rb
149
- - lib/water_drop/contracts/message.rb
150
- - lib/water_drop/errors.rb
151
- - lib/water_drop/instrumentation.rb
152
- - lib/water_drop/instrumentation/callbacks/delivery.rb
153
- - lib/water_drop/instrumentation/callbacks/error.rb
154
- - lib/water_drop/instrumentation/callbacks/statistics.rb
155
- - lib/water_drop/instrumentation/callbacks/statistics_decorator.rb
156
- - lib/water_drop/instrumentation/callbacks_manager.rb
157
- - lib/water_drop/instrumentation/monitor.rb
158
- - lib/water_drop/instrumentation/stdout_listener.rb
159
- - lib/water_drop/patches/rdkafka/bindings.rb
160
- - lib/water_drop/patches/rdkafka/producer.rb
161
- - lib/water_drop/producer.rb
162
- - lib/water_drop/producer/async.rb
163
- - lib/water_drop/producer/buffer.rb
164
- - lib/water_drop/producer/builder.rb
165
- - lib/water_drop/producer/dummy_client.rb
166
- - lib/water_drop/producer/status.rb
167
- - lib/water_drop/producer/sync.rb
168
- - lib/water_drop/version.rb
169
97
  - lib/waterdrop.rb
98
+ - lib/waterdrop/clients/buffered.rb
99
+ - lib/waterdrop/clients/dummy.rb
100
+ - lib/waterdrop/clients/rdkafka.rb
101
+ - lib/waterdrop/config.rb
102
+ - lib/waterdrop/contracts.rb
103
+ - lib/waterdrop/contracts/config.rb
104
+ - lib/waterdrop/contracts/message.rb
105
+ - lib/waterdrop/errors.rb
106
+ - lib/waterdrop/instrumentation/callbacks/delivery.rb
107
+ - lib/waterdrop/instrumentation/callbacks/error.rb
108
+ - lib/waterdrop/instrumentation/callbacks/statistics.rb
109
+ - lib/waterdrop/instrumentation/logger_listener.rb
110
+ - lib/waterdrop/instrumentation/monitor.rb
111
+ - lib/waterdrop/instrumentation/notifications.rb
112
+ - lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
113
+ - lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb
114
+ - lib/waterdrop/middleware.rb
115
+ - lib/waterdrop/producer.rb
116
+ - lib/waterdrop/producer/async.rb
117
+ - lib/waterdrop/producer/buffer.rb
118
+ - lib/waterdrop/producer/builder.rb
119
+ - lib/waterdrop/producer/status.rb
120
+ - lib/waterdrop/producer/sync.rb
121
+ - lib/waterdrop/producer/transactions.rb
122
+ - lib/waterdrop/version.rb
170
123
  - log/.gitkeep
124
+ - renovate.json
171
125
  - waterdrop.gemspec
172
126
  homepage: https://karafka.io
173
127
  licenses:
174
128
  - MIT
175
129
  metadata:
130
+ funding_uri: https://karafka.io/#become-pro
131
+ homepage_uri: https://karafka.io
132
+ changelog_uri: https://karafka.io/docs/Changelog-WaterDrop
133
+ bug_tracker_uri: https://github.com/karafka/waterdrop/issues
176
134
  source_code_uri: https://github.com/karafka/waterdrop
135
+ documentation_uri: https://karafka.io/docs/#waterdrop
136
+ rubygems_mfa_required: 'true'
177
137
  post_install_message:
178
138
  rdoc_options: []
179
139
  require_paths:
@@ -182,14 +142,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
182
142
  requirements:
183
143
  - - ">="
184
144
  - !ruby/object:Gem::Version
185
- version: 2.6.0
145
+ version: '0'
186
146
  required_rubygems_version: !ruby/object:Gem::Requirement
187
147
  requirements:
188
148
  - - ">="
189
149
  - !ruby/object:Gem::Version
190
150
  version: '0'
191
151
  requirements: []
192
- rubygems_version: 3.2.31
152
+ rubygems_version: 3.4.19
193
153
  signing_key:
194
154
  specification_version: 4
195
155
  summary: Kafka messaging made easy!
metadata.gz.sig CHANGED
Binary file
data/certs/mensfeld.pem DELETED
@@ -1,25 +0,0 @@
1
- -----BEGIN CERTIFICATE-----
2
- MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
3
- ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
4
- NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
5
- CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
6
- ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
7
- TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
8
- 1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
9
- vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
10
- qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
11
- tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
12
- g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
13
- LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
14
- BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
15
- EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
16
- c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
17
- W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
18
- yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
19
- 8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
20
- /rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
21
- fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
22
- 2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
23
- R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
24
- pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
25
- -----END CERTIFICATE-----
data/config/errors.yml DELETED
@@ -1,6 +0,0 @@
1
- en:
2
- dry_validation:
3
- errors:
4
- invalid_key_type: all keys need to be of type String
5
- invalid_value_type: all values need to be of type String
6
- max_payload_size: is more than `max_payload_size` config value
@@ -1,26 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Contracts
5
- # Contract with validation rules for WaterDrop configuration details
6
- class Config < Dry::Validation::Contract
7
- # Ensure valid format of each seed broker so that rdkafka doesn't fail silently
8
- SEED_BROKER_FORMAT_REGEXP = %r{\A([^:/,]+:[0-9]+)(,[^:/,]+:[0-9]+)*\z}.freeze
9
-
10
- private_constant :SEED_BROKER_FORMAT_REGEXP
11
-
12
- params do
13
- required(:id).filled(:str?)
14
- required(:logger).filled
15
- required(:deliver).filled(:bool?)
16
- required(:max_payload_size).filled(:int?, gteq?: 1)
17
- required(:max_wait_timeout).filled(:number?, gteq?: 0)
18
- required(:wait_timeout).filled(:number?, gt?: 0)
19
-
20
- required(:kafka).schema do
21
- required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
22
- end
23
- end
24
- end
25
- end
26
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Contracts
5
- # Contract with validation rules for validating that all the message options that
6
- # we provide to producer ale valid and usable
7
- class Message < Dry::Validation::Contract
8
- # Regex to check that topic has a valid format
9
- TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
10
-
11
- # Checks, that the given value is a string
12
- STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
13
-
14
- private_constant :TOPIC_REGEXP, :STRING_ASSERTION
15
-
16
- config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
17
-
18
- option :max_payload_size
19
-
20
- params do
21
- required(:topic).filled(:str?, format?: TOPIC_REGEXP)
22
- required(:payload).filled(:str?)
23
- optional(:key).maybe(:str?, :filled?)
24
- optional(:partition).filled(:int?, gteq?: -1)
25
- optional(:partition_key).maybe(:str?, :filled?)
26
- optional(:timestamp).maybe { time? | int? }
27
- optional(:headers).maybe(:hash?)
28
- end
29
-
30
- rule(:headers) do
31
- next unless value.is_a?(Hash)
32
-
33
- key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
34
- key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
35
- end
36
-
37
- rule(:payload) do
38
- key.failure(:max_payload_size) if value.bytesize > max_payload_size
39
- end
40
- end
41
- end
42
- end
@@ -1,30 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Instrumentation
5
- module Callbacks
6
- # Creates a callable that we want to run upon each message delivery or failure
7
- #
8
- # @note We don't have to provide client_name here as this callback is per client instance
9
- class Delivery
10
- # @param producer_id [String] id of the current producer
11
- # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
12
- def initialize(producer_id, monitor)
13
- @producer_id = producer_id
14
- @monitor = monitor
15
- end
16
-
17
- # Emits delivery details to the monitor
18
- # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
19
- def call(delivery_report)
20
- @monitor.instrument(
21
- 'message.acknowledged',
22
- producer_id: @producer_id,
23
- offset: delivery_report.offset,
24
- partition: delivery_report.partition
25
- )
26
- end
27
- end
28
- end
29
- end
30
- end
@@ -1,77 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Instrumentation
5
- module Callbacks
6
- # Many of the librdkafka statistics are absolute values instead of a gauge.
7
- # This means, that for example number of messages sent is an absolute growing value
8
- # instead of being a value of messages sent from the last statistics report.
9
- # This decorator calculates the diff against previously emited stats, so we get also
10
- # the diff together with the original values
11
- class StatisticsDecorator
12
- def initialize
13
- @previous = {}.freeze
14
- end
15
-
16
- # @param emited_stats [Hash] original emited statistics
17
- # @return [Hash] emited statistics extended with the diff data
18
- # @note We modify the emited statistics, instead of creating new. Since we don't expose
19
- # any API to get raw data, users can just assume that the result of this decoration is
20
- # the proper raw stats that they can use
21
- def call(emited_stats)
22
- diff(
23
- @previous,
24
- emited_stats
25
- )
26
-
27
- @previous = emited_stats
28
-
29
- emited_stats.freeze
30
- end
31
-
32
- private
33
-
34
- # Calculates the diff of the provided values and modifies in place the emited statistics
35
- #
36
- # @param previous [Object] previous value from the given scope in which
37
- # we are
38
- # @param current [Object] current scope from emitted statistics
39
- # @return [Object] the diff if the values were numerics or the current scope
40
- def diff(previous, current)
41
- if current.is_a?(Hash)
42
- # @note We cannot use #each_key as we modify the content of the current scope
43
- # in place (in case it's a hash)
44
- current.keys.each do |key|
45
- append(
46
- current,
47
- key,
48
- diff((previous || {})[key], (current || {})[key])
49
- )
50
- end
51
- end
52
-
53
- # Diff can be computed only for numerics
54
- return current unless current.is_a?(Numeric)
55
- # If there was no previous value, delta is always zero
56
- return 0 unless previous
57
- # Should never happen but just in case, a type changed in between stats
58
- return current unless previous.is_a?(Numeric)
59
-
60
- current - previous
61
- end
62
-
63
- # Appends the result of the diff to a given key as long as the result is numeric
64
- #
65
- # @param current [Hash] current scope
66
- # @param key [Symbol] key based on which we were diffing
67
- # @param result [Object] diff result
68
- def append(current, key, result)
69
- return unless result.is_a?(Numeric)
70
- return if current.frozen?
71
-
72
- current["#{key}_d"] = result
73
- end
74
- end
75
- end
76
- end
77
- end
@@ -1,39 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Instrumentation
5
- # This manager allows us to register multiple callbacks into a hook that is suppose to support
6
- # a single callback
7
- class CallbacksManager
8
- # @return [::WaterDrop::Instrumentation::CallbacksManager]
9
- def initialize
10
- @callbacks = Concurrent::Hash.new
11
- end
12
-
13
- # Invokes all the callbacks registered one after another
14
- #
15
- # @param args [Object] any args that should go to the callbacks
16
- # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
17
- # callbacks and add new at the same time. Since we don't know when and in what thread
18
- # things are going to be added to the manager, we need to extract values into an array and
19
- # run it. That way we can add new things the same time.
20
- def call(*args)
21
- @callbacks.values.each { |callback| callback.call(*args) }
22
- end
23
-
24
- # Adds a callback to the manager
25
- #
26
- # @param id [String] id of the callback (used when deleting it)
27
- # @param callable [#call] object that responds to a `#call` method
28
- def add(id, callable)
29
- @callbacks[id] = callable
30
- end
31
-
32
- # Removes the callback from the manager
33
- # @param id [String] id of the callback we want to remove
34
- def delete(id)
35
- @callbacks.delete(id)
36
- end
37
- end
38
- end
39
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- # Namespace for all the things related with WaterDrop instrumentation process
5
- module Instrumentation
6
- class << self
7
- # Builds a manager for statistics callbacks
8
- # @return [WaterDrop::CallbacksManager]
9
- def statistics_callbacks
10
- @statistics_callbacks ||= CallbacksManager.new
11
- end
12
-
13
- # Builds a manager for error callbacks
14
- # @return [WaterDrop::CallbacksManager]
15
- def error_callbacks
16
- @error_callbacks ||= CallbacksManager.new
17
- end
18
- end
19
- end
20
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Patches
5
- module Rdkafka
6
- # Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
7
- # to work with in a bit different way than rdkafka itself
8
- module Bindings
9
- class << self
10
- # Add extra methods that we need
11
- # @param mod [::Rdkafka::Bindings] rdkafka bindings module
12
- def included(mod)
13
- mod.attach_function :rd_kafka_name, [:pointer], :string
14
-
15
- # Default rdkafka setup for errors doest not propagate client details, thus it always
16
- # publishes all the stuff for all rdkafka instances. We change that by providing
17
- # function that fetches the instance name, allowing us to have better notifications
18
- mod.send(:remove_const, :ErrorCallback)
19
- mod.const_set(:ErrorCallback, build_error_callback)
20
- end
21
-
22
- # @return [FFI::Function] overwritten callback function
23
- def build_error_callback
24
- FFI::Function.new(
25
- :void, %i[pointer int string pointer]
26
- ) do |client_prr, err_code, reason, _opaque|
27
- return nil unless ::Rdkafka::Config.error_callback
28
-
29
- name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
30
-
31
- error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
32
-
33
- ::Rdkafka::Config.error_callback.call(name, error)
34
- end
35
- end
36
- end
37
- end
38
- end
39
- end
40
- end
41
-
42
- ::Rdkafka::Bindings.include(::WaterDrop::Patches::Rdkafka::Bindings)
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- # Patches to external components
5
- module Patches
6
- # Rdkafka related patches
7
- module Rdkafka
8
- # Rdkafka::Producer patches
9
- module Producer
10
- # Adds a method that allows us to get the native kafka producer name
11
- # @return [String] producer instance name
12
- def name
13
- ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
14
- end
15
- end
16
- end
17
- end
18
- end
19
-
20
- ::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer
@@ -1,32 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- class Producer
5
- # A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
6
- # want to dispatch anything to Kafka
7
- class DummyClient
8
- # @return [DummyClient] dummy instance
9
- def initialize
10
- @counter = -1
11
- end
12
-
13
- # Dummy method for returning the delivery report
14
- # @param _args [Object] anything that the delivery handle accepts
15
- # @return [::Rdkafka::Producer::DeliveryReport]
16
- def wait(*_args)
17
- ::Rdkafka::Producer::DeliveryReport.new(0, @counter += 1)
18
- end
19
-
20
- # @param _args [Object] anything really, this dummy is suppose to support anything
21
- def respond_to_missing?(*_args)
22
- true
23
- end
24
-
25
- # @param _args [Object] anything really, this dummy is suppose to support anything
26
- # @return [self] returns self for chaining cases
27
- def method_missing(*_args)
28
- self || super
29
- end
30
- end
31
- end
32
- end