waterdrop 2.0.7 → 2.6.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/.github/FUNDING.yml +1 -0
  4. data/.github/workflows/ci.yml +39 -13
  5. data/.ruby-version +1 -1
  6. data/CHANGELOG.md +212 -0
  7. data/Gemfile +0 -2
  8. data/Gemfile.lock +45 -75
  9. data/README.md +22 -275
  10. data/certs/cert_chain.pem +26 -0
  11. data/config/locales/errors.yml +39 -0
  12. data/docker-compose.yml +21 -12
  13. data/lib/waterdrop/clients/buffered.rb +95 -0
  14. data/lib/waterdrop/clients/dummy.rb +69 -0
  15. data/lib/waterdrop/clients/rdkafka.rb +34 -0
  16. data/lib/{water_drop → waterdrop}/config.rb +39 -16
  17. data/lib/waterdrop/contracts/config.rb +43 -0
  18. data/lib/waterdrop/contracts/message.rb +64 -0
  19. data/lib/waterdrop/contracts/transactional_offset.rb +21 -0
  20. data/lib/{water_drop → waterdrop}/errors.rb +23 -7
  21. data/lib/waterdrop/helpers/counter.rb +27 -0
  22. data/lib/waterdrop/instrumentation/callbacks/delivery.rb +106 -0
  23. data/lib/{water_drop → waterdrop}/instrumentation/callbacks/error.rb +6 -2
  24. data/lib/{water_drop → waterdrop}/instrumentation/callbacks/statistics.rb +1 -1
  25. data/lib/{water_drop/instrumentation/stdout_listener.rb → waterdrop/instrumentation/logger_listener.rb} +91 -21
  26. data/lib/waterdrop/instrumentation/monitor.rb +20 -0
  27. data/lib/{water_drop/instrumentation/monitor.rb → waterdrop/instrumentation/notifications.rb} +15 -14
  28. data/lib/waterdrop/instrumentation/vendors/datadog/dashboard.json +1 -0
  29. data/lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb +210 -0
  30. data/lib/waterdrop/middleware.rb +50 -0
  31. data/lib/{water_drop → waterdrop}/producer/async.rb +40 -4
  32. data/lib/{water_drop → waterdrop}/producer/buffer.rb +13 -31
  33. data/lib/{water_drop → waterdrop}/producer/builder.rb +6 -11
  34. data/lib/{water_drop → waterdrop}/producer/sync.rb +44 -15
  35. data/lib/waterdrop/producer/transactions.rb +219 -0
  36. data/lib/waterdrop/producer.rb +324 -0
  37. data/lib/{water_drop → waterdrop}/version.rb +1 -1
  38. data/lib/waterdrop.rb +27 -2
  39. data/renovate.json +6 -0
  40. data/waterdrop.gemspec +14 -11
  41. data.tar.gz.sig +0 -0
  42. metadata +73 -111
  43. metadata.gz.sig +0 -0
  44. data/certs/mensfeld.pem +0 -25
  45. data/config/errors.yml +0 -6
  46. data/lib/water_drop/contracts/config.rb +0 -26
  47. data/lib/water_drop/contracts/message.rb +0 -42
  48. data/lib/water_drop/instrumentation/callbacks/delivery.rb +0 -30
  49. data/lib/water_drop/instrumentation/callbacks/statistics_decorator.rb +0 -77
  50. data/lib/water_drop/instrumentation/callbacks_manager.rb +0 -39
  51. data/lib/water_drop/instrumentation.rb +0 -20
  52. data/lib/water_drop/patches/rdkafka/bindings.rb +0 -42
  53. data/lib/water_drop/patches/rdkafka/producer.rb +0 -20
  54. data/lib/water_drop/producer/dummy_client.rb +0 -32
  55. data/lib/water_drop/producer.rb +0 -162
  56. data/lib/water_drop.rb +0 -36
  57. /data/lib/{water_drop → waterdrop}/contracts.rb +0 -0
  58. /data/lib/{water_drop → waterdrop}/producer/status.rb +0 -0
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: waterdrop
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.7
4
+ version: 2.6.14
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -10,102 +10,53 @@ bindir: bin
10
10
  cert_chain:
11
11
  - |
12
12
  -----BEGIN CERTIFICATE-----
13
- MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
14
- ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
15
- NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
16
- CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
17
- ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
18
- TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
19
- 1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
20
- vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
21
- qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
22
- tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
23
- g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
24
- LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
25
- BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
26
- EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
27
- c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
28
- W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
29
- yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
30
- 8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
31
- /rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
32
- fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
33
- 2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
34
- R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
35
- pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
13
+ MIIEcDCCAtigAwIBAgIBATANBgkqhkiG9w0BAQsFADA/MRAwDgYDVQQDDAdjb250
14
+ YWN0MRcwFQYKCZImiZPyLGQBGRYHa2FyYWZrYTESMBAGCgmSJomT8ixkARkWAmlv
15
+ MB4XDTIzMDgyMTA3MjU1NFoXDTI0MDgyMDA3MjU1NFowPzEQMA4GA1UEAwwHY29u
16
+ dGFjdDEXMBUGCgmSJomT8ixkARkWB2thcmFma2ExEjAQBgoJkiaJk/IsZAEZFgJp
17
+ bzCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAOuZpyQKEwsTG9plLat7
18
+ 8bUaNuNBEnouTsNMr6X+XTgvyrAxTuocdsyP1sNCjdS1B8RiiDH1/Nt9qpvlBWon
19
+ sdJ1SYhaWNVfqiYStTDnCx3PRMmHRdD4KqUWKpN6VpZ1O/Zu+9Mw0COmvXgZuuO9
20
+ wMSJkXRo6dTCfMedLAIxjMeBIxtoLR2e6Jm6MR8+8WYYVWrO9kSOOt5eKQLBY7aK
21
+ b/Dc40EcJKPg3Z30Pia1M9ZyRlb6SOj6SKpHRqc7vbVQxjEw6Jjal1lZ49m3YZMd
22
+ ArMAs9lQZNdSw5/UX6HWWURLowg6k10RnhTUtYyzO9BFev0JFJftHnmuk8vtb+SD
23
+ 5VPmjFXg2VOcw0B7FtG75Vackk8QKfgVe3nSPhVpew2CSPlbJzH80wChbr19+e3+
24
+ YGr1tOiaJrL6c+PNmb0F31NXMKpj/r+n15HwlTMRxQrzFcgjBlxf2XFGnPQXHhBm
25
+ kp1OFnEq4GG9sON4glRldkwzi/f/fGcZmo5fm3d+0ZdNgwIDAQABo3cwdTAJBgNV
26
+ HRMEAjAAMAsGA1UdDwQEAwIEsDAdBgNVHQ4EFgQUPVH5+dLA80A1kJ2Uz5iGwfOa
27
+ 1+swHQYDVR0RBBYwFIESY29udGFjdEBrYXJhZmthLmlvMB0GA1UdEgQWMBSBEmNv
28
+ bnRhY3RAa2FyYWZrYS5pbzANBgkqhkiG9w0BAQsFAAOCAYEAnpa0jcN7JzREHMTQ
29
+ bfZ+xcvlrzuROMY6A3zIZmQgbnoZZNuX4cMRrT1p1HuwXpxdpHPw7dDjYqWw3+1h
30
+ 3mXLeMuk7amjQpYoSWU/OIZMhIsARra22UN8qkkUlUj3AwTaChVKN/bPJOM2DzfU
31
+ kz9vUgLeYYFfQbZqeI6SsM7ltilRV4W8D9yNUQQvOxCFxtLOetJ00fC/E7zMUzbK
32
+ IBwYFQYsbI6XQzgAIPW6nGSYKgRhkfpmquXSNKZRIQ4V6bFrufa+DzD0bt2ZA3ah
33
+ fMmJguyb5L2Gf1zpDXzFSPMG7YQFLzwYz1zZZvOU7/UCpQsHpID/YxqDp4+Dgb+Y
34
+ qma0whX8UG/gXFV2pYWpYOfpatvahwi+A1TwPQsuZwkkhi1OyF1At3RY+hjSXyav
35
+ AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
36
+ msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
36
37
  -----END CERTIFICATE-----
37
- date: 2021-12-03 00:00:00.000000000 Z
38
+ date: 2024-02-06 00:00:00.000000000 Z
38
39
  dependencies:
39
40
  - !ruby/object:Gem::Dependency
40
- name: concurrent-ruby
41
+ name: karafka-core
41
42
  requirement: !ruby/object:Gem::Requirement
42
43
  requirements:
43
44
  - - ">="
44
45
  - !ruby/object:Gem::Version
45
- version: '1.1'
46
- type: :runtime
47
- prerelease: false
48
- version_requirements: !ruby/object:Gem::Requirement
49
- requirements:
50
- - - ">="
51
- - !ruby/object:Gem::Version
52
- version: '1.1'
53
- - !ruby/object:Gem::Dependency
54
- name: dry-configurable
55
- requirement: !ruby/object:Gem::Requirement
56
- requirements:
57
- - - "~>"
58
- - !ruby/object:Gem::Version
59
- version: '0.13'
60
- type: :runtime
61
- prerelease: false
62
- version_requirements: !ruby/object:Gem::Requirement
63
- requirements:
64
- - - "~>"
46
+ version: 2.2.3
47
+ - - "<"
65
48
  - !ruby/object:Gem::Version
66
- version: '0.13'
67
- - !ruby/object:Gem::Dependency
68
- name: dry-monitor
69
- requirement: !ruby/object:Gem::Requirement
70
- requirements:
71
- - - "~>"
72
- - !ruby/object:Gem::Version
73
- version: '0.5'
49
+ version: 3.0.0
74
50
  type: :runtime
75
51
  prerelease: false
76
52
  version_requirements: !ruby/object:Gem::Requirement
77
- requirements:
78
- - - "~>"
79
- - !ruby/object:Gem::Version
80
- version: '0.5'
81
- - !ruby/object:Gem::Dependency
82
- name: dry-validation
83
- requirement: !ruby/object:Gem::Requirement
84
- requirements:
85
- - - "~>"
86
- - !ruby/object:Gem::Version
87
- version: '1.7'
88
- type: :runtime
89
- prerelease: false
90
- version_requirements: !ruby/object:Gem::Requirement
91
- requirements:
92
- - - "~>"
93
- - !ruby/object:Gem::Version
94
- version: '1.7'
95
- - !ruby/object:Gem::Dependency
96
- name: rdkafka
97
- requirement: !ruby/object:Gem::Requirement
98
53
  requirements:
99
54
  - - ">="
100
55
  - !ruby/object:Gem::Version
101
- version: '0.10'
102
- type: :runtime
103
- prerelease: false
104
- version_requirements: !ruby/object:Gem::Requirement
105
- requirements:
106
- - - ">="
56
+ version: 2.2.3
57
+ - - "<"
107
58
  - !ruby/object:Gem::Version
108
- version: '0.10'
59
+ version: 3.0.0
109
60
  - !ruby/object:Gem::Dependency
110
61
  name: zeitwerk
111
62
  requirement: !ruby/object:Gem::Requirement
@@ -122,13 +73,14 @@ dependencies:
122
73
  version: '2.3'
123
74
  description: Kafka messaging made easy!
124
75
  email:
125
- - maciej@mensfeld.pl
76
+ - contact@karafka.io
126
77
  executables: []
127
78
  extensions: []
128
79
  extra_rdoc_files: []
129
80
  files:
130
81
  - ".coditsu/ci.yml"
131
82
  - ".diffend.yml"
83
+ - ".github/FUNDING.yml"
132
84
  - ".github/workflows/ci.yml"
133
85
  - ".gitignore"
134
86
  - ".rspec"
@@ -139,41 +91,51 @@ files:
139
91
  - Gemfile.lock
140
92
  - MIT-LICENSE
141
93
  - README.md
142
- - certs/mensfeld.pem
143
- - config/errors.yml
94
+ - certs/cert_chain.pem
95
+ - config/locales/errors.yml
144
96
  - docker-compose.yml
145
- - lib/water_drop.rb
146
- - lib/water_drop/config.rb
147
- - lib/water_drop/contracts.rb
148
- - lib/water_drop/contracts/config.rb
149
- - lib/water_drop/contracts/message.rb
150
- - lib/water_drop/errors.rb
151
- - lib/water_drop/instrumentation.rb
152
- - lib/water_drop/instrumentation/callbacks/delivery.rb
153
- - lib/water_drop/instrumentation/callbacks/error.rb
154
- - lib/water_drop/instrumentation/callbacks/statistics.rb
155
- - lib/water_drop/instrumentation/callbacks/statistics_decorator.rb
156
- - lib/water_drop/instrumentation/callbacks_manager.rb
157
- - lib/water_drop/instrumentation/monitor.rb
158
- - lib/water_drop/instrumentation/stdout_listener.rb
159
- - lib/water_drop/patches/rdkafka/bindings.rb
160
- - lib/water_drop/patches/rdkafka/producer.rb
161
- - lib/water_drop/producer.rb
162
- - lib/water_drop/producer/async.rb
163
- - lib/water_drop/producer/buffer.rb
164
- - lib/water_drop/producer/builder.rb
165
- - lib/water_drop/producer/dummy_client.rb
166
- - lib/water_drop/producer/status.rb
167
- - lib/water_drop/producer/sync.rb
168
- - lib/water_drop/version.rb
169
97
  - lib/waterdrop.rb
98
+ - lib/waterdrop/clients/buffered.rb
99
+ - lib/waterdrop/clients/dummy.rb
100
+ - lib/waterdrop/clients/rdkafka.rb
101
+ - lib/waterdrop/config.rb
102
+ - lib/waterdrop/contracts.rb
103
+ - lib/waterdrop/contracts/config.rb
104
+ - lib/waterdrop/contracts/message.rb
105
+ - lib/waterdrop/contracts/transactional_offset.rb
106
+ - lib/waterdrop/errors.rb
107
+ - lib/waterdrop/helpers/counter.rb
108
+ - lib/waterdrop/instrumentation/callbacks/delivery.rb
109
+ - lib/waterdrop/instrumentation/callbacks/error.rb
110
+ - lib/waterdrop/instrumentation/callbacks/statistics.rb
111
+ - lib/waterdrop/instrumentation/logger_listener.rb
112
+ - lib/waterdrop/instrumentation/monitor.rb
113
+ - lib/waterdrop/instrumentation/notifications.rb
114
+ - lib/waterdrop/instrumentation/vendors/datadog/dashboard.json
115
+ - lib/waterdrop/instrumentation/vendors/datadog/metrics_listener.rb
116
+ - lib/waterdrop/middleware.rb
117
+ - lib/waterdrop/producer.rb
118
+ - lib/waterdrop/producer/async.rb
119
+ - lib/waterdrop/producer/buffer.rb
120
+ - lib/waterdrop/producer/builder.rb
121
+ - lib/waterdrop/producer/status.rb
122
+ - lib/waterdrop/producer/sync.rb
123
+ - lib/waterdrop/producer/transactions.rb
124
+ - lib/waterdrop/version.rb
170
125
  - log/.gitkeep
126
+ - renovate.json
171
127
  - waterdrop.gemspec
172
128
  homepage: https://karafka.io
173
129
  licenses:
174
130
  - MIT
175
131
  metadata:
132
+ funding_uri: https://karafka.io/#become-pro
133
+ homepage_uri: https://karafka.io
134
+ changelog_uri: https://karafka.io/docs/Changelog-WaterDrop
135
+ bug_tracker_uri: https://github.com/karafka/waterdrop/issues
176
136
  source_code_uri: https://github.com/karafka/waterdrop
137
+ documentation_uri: https://karafka.io/docs/#waterdrop
138
+ rubygems_mfa_required: 'true'
177
139
  post_install_message:
178
140
  rdoc_options: []
179
141
  require_paths:
@@ -182,14 +144,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
182
144
  requirements:
183
145
  - - ">="
184
146
  - !ruby/object:Gem::Version
185
- version: 2.6.0
147
+ version: '0'
186
148
  required_rubygems_version: !ruby/object:Gem::Requirement
187
149
  requirements:
188
150
  - - ">="
189
151
  - !ruby/object:Gem::Version
190
152
  version: '0'
191
153
  requirements: []
192
- rubygems_version: 3.2.31
154
+ rubygems_version: 3.5.3
193
155
  signing_key:
194
156
  specification_version: 4
195
157
  summary: Kafka messaging made easy!
metadata.gz.sig CHANGED
Binary file
data/certs/mensfeld.pem DELETED
@@ -1,25 +0,0 @@
1
- -----BEGIN CERTIFICATE-----
2
- MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
3
- ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
4
- NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
5
- CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
6
- ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
7
- TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
8
- 1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
9
- vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
10
- qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
11
- tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
12
- g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
13
- LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
14
- BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
15
- EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
16
- c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
17
- W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
18
- yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
19
- 8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
20
- /rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
21
- fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
22
- 2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
23
- R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
24
- pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
25
- -----END CERTIFICATE-----
data/config/errors.yml DELETED
@@ -1,6 +0,0 @@
1
- en:
2
- dry_validation:
3
- errors:
4
- invalid_key_type: all keys need to be of type String
5
- invalid_value_type: all values need to be of type String
6
- max_payload_size: is more than `max_payload_size` config value
@@ -1,26 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Contracts
5
- # Contract with validation rules for WaterDrop configuration details
6
- class Config < Dry::Validation::Contract
7
- # Ensure valid format of each seed broker so that rdkafka doesn't fail silently
8
- SEED_BROKER_FORMAT_REGEXP = %r{\A([^:/,]+:[0-9]+)(,[^:/,]+:[0-9]+)*\z}.freeze
9
-
10
- private_constant :SEED_BROKER_FORMAT_REGEXP
11
-
12
- params do
13
- required(:id).filled(:str?)
14
- required(:logger).filled
15
- required(:deliver).filled(:bool?)
16
- required(:max_payload_size).filled(:int?, gteq?: 1)
17
- required(:max_wait_timeout).filled(:number?, gteq?: 0)
18
- required(:wait_timeout).filled(:number?, gt?: 0)
19
-
20
- required(:kafka).schema do
21
- required(:'bootstrap.servers').filled(:str?, format?: SEED_BROKER_FORMAT_REGEXP)
22
- end
23
- end
24
- end
25
- end
26
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Contracts
5
- # Contract with validation rules for validating that all the message options that
6
- # we provide to producer ale valid and usable
7
- class Message < Dry::Validation::Contract
8
- # Regex to check that topic has a valid format
9
- TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
10
-
11
- # Checks, that the given value is a string
12
- STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
13
-
14
- private_constant :TOPIC_REGEXP, :STRING_ASSERTION
15
-
16
- config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
17
-
18
- option :max_payload_size
19
-
20
- params do
21
- required(:topic).filled(:str?, format?: TOPIC_REGEXP)
22
- required(:payload).filled(:str?)
23
- optional(:key).maybe(:str?, :filled?)
24
- optional(:partition).filled(:int?, gteq?: -1)
25
- optional(:partition_key).maybe(:str?, :filled?)
26
- optional(:timestamp).maybe { time? | int? }
27
- optional(:headers).maybe(:hash?)
28
- end
29
-
30
- rule(:headers) do
31
- next unless value.is_a?(Hash)
32
-
33
- key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
34
- key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
35
- end
36
-
37
- rule(:payload) do
38
- key.failure(:max_payload_size) if value.bytesize > max_payload_size
39
- end
40
- end
41
- end
42
- end
@@ -1,30 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Instrumentation
5
- module Callbacks
6
- # Creates a callable that we want to run upon each message delivery or failure
7
- #
8
- # @note We don't have to provide client_name here as this callback is per client instance
9
- class Delivery
10
- # @param producer_id [String] id of the current producer
11
- # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
12
- def initialize(producer_id, monitor)
13
- @producer_id = producer_id
14
- @monitor = monitor
15
- end
16
-
17
- # Emits delivery details to the monitor
18
- # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
19
- def call(delivery_report)
20
- @monitor.instrument(
21
- 'message.acknowledged',
22
- producer_id: @producer_id,
23
- offset: delivery_report.offset,
24
- partition: delivery_report.partition
25
- )
26
- end
27
- end
28
- end
29
- end
30
- end
@@ -1,77 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Instrumentation
5
- module Callbacks
6
- # Many of the librdkafka statistics are absolute values instead of a gauge.
7
- # This means, that for example number of messages sent is an absolute growing value
8
- # instead of being a value of messages sent from the last statistics report.
9
- # This decorator calculates the diff against previously emited stats, so we get also
10
- # the diff together with the original values
11
- class StatisticsDecorator
12
- def initialize
13
- @previous = {}.freeze
14
- end
15
-
16
- # @param emited_stats [Hash] original emited statistics
17
- # @return [Hash] emited statistics extended with the diff data
18
- # @note We modify the emited statistics, instead of creating new. Since we don't expose
19
- # any API to get raw data, users can just assume that the result of this decoration is
20
- # the proper raw stats that they can use
21
- def call(emited_stats)
22
- diff(
23
- @previous,
24
- emited_stats
25
- )
26
-
27
- @previous = emited_stats
28
-
29
- emited_stats.freeze
30
- end
31
-
32
- private
33
-
34
- # Calculates the diff of the provided values and modifies in place the emited statistics
35
- #
36
- # @param previous [Object] previous value from the given scope in which
37
- # we are
38
- # @param current [Object] current scope from emitted statistics
39
- # @return [Object] the diff if the values were numerics or the current scope
40
- def diff(previous, current)
41
- if current.is_a?(Hash)
42
- # @note We cannot use #each_key as we modify the content of the current scope
43
- # in place (in case it's a hash)
44
- current.keys.each do |key|
45
- append(
46
- current,
47
- key,
48
- diff((previous || {})[key], (current || {})[key])
49
- )
50
- end
51
- end
52
-
53
- # Diff can be computed only for numerics
54
- return current unless current.is_a?(Numeric)
55
- # If there was no previous value, delta is always zero
56
- return 0 unless previous
57
- # Should never happen but just in case, a type changed in between stats
58
- return current unless previous.is_a?(Numeric)
59
-
60
- current - previous
61
- end
62
-
63
- # Appends the result of the diff to a given key as long as the result is numeric
64
- #
65
- # @param current [Hash] current scope
66
- # @param key [Symbol] key based on which we were diffing
67
- # @param result [Object] diff result
68
- def append(current, key, result)
69
- return unless result.is_a?(Numeric)
70
- return if current.frozen?
71
-
72
- current["#{key}_d"] = result
73
- end
74
- end
75
- end
76
- end
77
- end
@@ -1,39 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Instrumentation
5
- # This manager allows us to register multiple callbacks into a hook that is suppose to support
6
- # a single callback
7
- class CallbacksManager
8
- # @return [::WaterDrop::Instrumentation::CallbacksManager]
9
- def initialize
10
- @callbacks = Concurrent::Hash.new
11
- end
12
-
13
- # Invokes all the callbacks registered one after another
14
- #
15
- # @param args [Object] any args that should go to the callbacks
16
- # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
17
- # callbacks and add new at the same time. Since we don't know when and in what thread
18
- # things are going to be added to the manager, we need to extract values into an array and
19
- # run it. That way we can add new things the same time.
20
- def call(*args)
21
- @callbacks.values.each { |callback| callback.call(*args) }
22
- end
23
-
24
- # Adds a callback to the manager
25
- #
26
- # @param id [String] id of the callback (used when deleting it)
27
- # @param callable [#call] object that responds to a `#call` method
28
- def add(id, callable)
29
- @callbacks[id] = callable
30
- end
31
-
32
- # Removes the callback from the manager
33
- # @param id [String] id of the callback we want to remove
34
- def delete(id)
35
- @callbacks.delete(id)
36
- end
37
- end
38
- end
39
- end
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- # Namespace for all the things related with WaterDrop instrumentation process
5
- module Instrumentation
6
- class << self
7
- # Builds a manager for statistics callbacks
8
- # @return [WaterDrop::CallbacksManager]
9
- def statistics_callbacks
10
- @statistics_callbacks ||= CallbacksManager.new
11
- end
12
-
13
- # Builds a manager for error callbacks
14
- # @return [WaterDrop::CallbacksManager]
15
- def error_callbacks
16
- @error_callbacks ||= CallbacksManager.new
17
- end
18
- end
19
- end
20
- end
@@ -1,42 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Patches
5
- module Rdkafka
6
- # Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
7
- # to work with in a bit different way than rdkafka itself
8
- module Bindings
9
- class << self
10
- # Add extra methods that we need
11
- # @param mod [::Rdkafka::Bindings] rdkafka bindings module
12
- def included(mod)
13
- mod.attach_function :rd_kafka_name, [:pointer], :string
14
-
15
- # Default rdkafka setup for errors doest not propagate client details, thus it always
16
- # publishes all the stuff for all rdkafka instances. We change that by providing
17
- # function that fetches the instance name, allowing us to have better notifications
18
- mod.send(:remove_const, :ErrorCallback)
19
- mod.const_set(:ErrorCallback, build_error_callback)
20
- end
21
-
22
- # @return [FFI::Function] overwritten callback function
23
- def build_error_callback
24
- FFI::Function.new(
25
- :void, %i[pointer int string pointer]
26
- ) do |client_prr, err_code, reason, _opaque|
27
- return nil unless ::Rdkafka::Config.error_callback
28
-
29
- name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
30
-
31
- error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
32
-
33
- ::Rdkafka::Config.error_callback.call(name, error)
34
- end
35
- end
36
- end
37
- end
38
- end
39
- end
40
- end
41
-
42
- ::Rdkafka::Bindings.include(::WaterDrop::Patches::Rdkafka::Bindings)
@@ -1,20 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- # Patches to external components
5
- module Patches
6
- # Rdkafka related patches
7
- module Rdkafka
8
- # Rdkafka::Producer patches
9
- module Producer
10
- # Adds a method that allows us to get the native kafka producer name
11
- # @return [String] producer instance name
12
- def name
13
- ::Rdkafka::Bindings.rd_kafka_name(@native_kafka)
14
- end
15
- end
16
- end
17
- end
18
- end
19
-
20
- ::Rdkafka::Producer.include ::WaterDrop::Patches::Rdkafka::Producer
@@ -1,32 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- class Producer
5
- # A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
6
- # want to dispatch anything to Kafka
7
- class DummyClient
8
- # @return [DummyClient] dummy instance
9
- def initialize
10
- @counter = -1
11
- end
12
-
13
- # Dummy method for returning the delivery report
14
- # @param _args [Object] anything that the delivery handle accepts
15
- # @return [::Rdkafka::Producer::DeliveryReport]
16
- def wait(*_args)
17
- ::Rdkafka::Producer::DeliveryReport.new(0, @counter += 1)
18
- end
19
-
20
- # @param _args [Object] anything really, this dummy is suppose to support anything
21
- def respond_to_missing?(*_args)
22
- true
23
- end
24
-
25
- # @param _args [Object] anything really, this dummy is suppose to support anything
26
- # @return [self] returns self for chaining cases
27
- def method_missing(*_args)
28
- self || super
29
- end
30
- end
31
- end
32
- end