dionysus-rb 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +61 -0
  3. data/.github/workflows/ci.yml +77 -0
  4. data/.gitignore +12 -0
  5. data/.rspec +3 -0
  6. data/.rubocop.yml +175 -0
  7. data/.rubocop_todo.yml +53 -0
  8. data/CHANGELOG.md +227 -0
  9. data/Gemfile +10 -0
  10. data/Gemfile.lock +258 -0
  11. data/LICENSE.txt +21 -0
  12. data/README.md +1206 -0
  13. data/Rakefile +10 -0
  14. data/assets/logo.svg +51 -0
  15. data/bin/console +11 -0
  16. data/bin/karafka_health_check +14 -0
  17. data/bin/outbox_worker_health_check +12 -0
  18. data/bin/setup +8 -0
  19. data/dionysus-rb.gemspec +64 -0
  20. data/docker-compose.yml +44 -0
  21. data/lib/dionysus/checks/health_check.rb +50 -0
  22. data/lib/dionysus/checks.rb +7 -0
  23. data/lib/dionysus/consumer/batch_events_publisher.rb +33 -0
  24. data/lib/dionysus/consumer/config.rb +97 -0
  25. data/lib/dionysus/consumer/deserializer.rb +231 -0
  26. data/lib/dionysus/consumer/dionysus_event.rb +42 -0
  27. data/lib/dionysus/consumer/karafka_consumer_generator.rb +56 -0
  28. data/lib/dionysus/consumer/params_batch_processor.rb +65 -0
  29. data/lib/dionysus/consumer/params_batch_transformations/remove_duplicates_strategy.rb +54 -0
  30. data/lib/dionysus/consumer/params_batch_transformations.rb +4 -0
  31. data/lib/dionysus/consumer/persistor.rb +157 -0
  32. data/lib/dionysus/consumer/registry.rb +84 -0
  33. data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data.rb +27 -0
  34. data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data_job.rb +26 -0
  35. data/lib/dionysus/consumer/synced_data.rb +4 -0
  36. data/lib/dionysus/consumer/synchronizable_model.rb +93 -0
  37. data/lib/dionysus/consumer/workers_group.rb +18 -0
  38. data/lib/dionysus/consumer.rb +36 -0
  39. data/lib/dionysus/monitor.rb +48 -0
  40. data/lib/dionysus/producer/base_responder.rb +46 -0
  41. data/lib/dionysus/producer/config.rb +104 -0
  42. data/lib/dionysus/producer/deleted_record_serializer.rb +17 -0
  43. data/lib/dionysus/producer/genesis/performed.rb +11 -0
  44. data/lib/dionysus/producer/genesis/stream_job.rb +13 -0
  45. data/lib/dionysus/producer/genesis/streamer/base_job.rb +44 -0
  46. data/lib/dionysus/producer/genesis/streamer/standard_job.rb +43 -0
  47. data/lib/dionysus/producer/genesis/streamer.rb +40 -0
  48. data/lib/dionysus/producer/genesis.rb +62 -0
  49. data/lib/dionysus/producer/karafka_responder_generator.rb +133 -0
  50. data/lib/dionysus/producer/key.rb +14 -0
  51. data/lib/dionysus/producer/model_serializer.rb +105 -0
  52. data/lib/dionysus/producer/outbox/active_record_publishable.rb +74 -0
  53. data/lib/dionysus/producer/outbox/datadog_latency_reporter.rb +26 -0
  54. data/lib/dionysus/producer/outbox/datadog_latency_reporter_job.rb +11 -0
  55. data/lib/dionysus/producer/outbox/datadog_latency_reporter_scheduler.rb +47 -0
  56. data/lib/dionysus/producer/outbox/datadog_tracer.rb +32 -0
  57. data/lib/dionysus/producer/outbox/duplicates_filter.rb +26 -0
  58. data/lib/dionysus/producer/outbox/event_name.rb +26 -0
  59. data/lib/dionysus/producer/outbox/health_check.rb +48 -0
  60. data/lib/dionysus/producer/outbox/latency_tracker.rb +43 -0
  61. data/lib/dionysus/producer/outbox/model.rb +117 -0
  62. data/lib/dionysus/producer/outbox/producer.rb +26 -0
  63. data/lib/dionysus/producer/outbox/publishable.rb +106 -0
  64. data/lib/dionysus/producer/outbox/publisher.rb +131 -0
  65. data/lib/dionysus/producer/outbox/records_processor.rb +56 -0
  66. data/lib/dionysus/producer/outbox/runner.rb +120 -0
  67. data/lib/dionysus/producer/outbox/tombstone_publisher.rb +22 -0
  68. data/lib/dionysus/producer/outbox.rb +103 -0
  69. data/lib/dionysus/producer/partition_key.rb +42 -0
  70. data/lib/dionysus/producer/registry/validator.rb +32 -0
  71. data/lib/dionysus/producer/registry.rb +165 -0
  72. data/lib/dionysus/producer/serializer.rb +52 -0
  73. data/lib/dionysus/producer/suppressor.rb +18 -0
  74. data/lib/dionysus/producer.rb +121 -0
  75. data/lib/dionysus/railtie.rb +9 -0
  76. data/lib/dionysus/rb/version.rb +5 -0
  77. data/lib/dionysus/rb.rb +8 -0
  78. data/lib/dionysus/support/rspec/outbox_publishable.rb +78 -0
  79. data/lib/dionysus/topic_name.rb +15 -0
  80. data/lib/dionysus/utils/default_message_filter.rb +25 -0
  81. data/lib/dionysus/utils/exponential_backoff.rb +7 -0
  82. data/lib/dionysus/utils/karafka_datadog_listener.rb +20 -0
  83. data/lib/dionysus/utils/karafka_sentry_listener.rb +9 -0
  84. data/lib/dionysus/utils/null_error_handler.rb +6 -0
  85. data/lib/dionysus/utils/null_event_bus.rb +5 -0
  86. data/lib/dionysus/utils/null_hermes_event_producer.rb +5 -0
  87. data/lib/dionysus/utils/null_instrumenter.rb +7 -0
  88. data/lib/dionysus/utils/null_lock_client.rb +13 -0
  89. data/lib/dionysus/utils/null_model_factory.rb +5 -0
  90. data/lib/dionysus/utils/null_mutex_provider.rb +7 -0
  91. data/lib/dionysus/utils/null_retry_provider.rb +7 -0
  92. data/lib/dionysus/utils/null_tracer.rb +5 -0
  93. data/lib/dionysus/utils/null_transaction_provider.rb +15 -0
  94. data/lib/dionysus/utils/sidekiq_batched_job_distributor.rb +24 -0
  95. data/lib/dionysus/utils.rb +6 -0
  96. data/lib/dionysus/version.rb +7 -0
  97. data/lib/dionysus-rb.rb +3 -0
  98. data/lib/dionysus.rb +133 -0
  99. data/lib/tasks/dionysus.rake +18 -0
  100. data/log/development.log +0 -0
  101. data/sig/dionysus/rb.rbs +6 -0
  102. metadata +585 -0
data/Gemfile.lock ADDED
@@ -0,0 +1,258 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ dionysus-rb (0.1.0)
5
+ activerecord (>= 5)
6
+ activesupport (>= 3.2)
7
+ concurrent-ruby
8
+ dry-monitor
9
+ file-based-healthcheck
10
+ karafka (~> 2.0)
11
+ sidekiq
12
+ sidekiq-cron
13
+ sigurd
14
+ waterdrop (~> 2.0)
15
+ zeitwerk
16
+
17
+ GEM
18
+ remote: https://rubygems.org/
19
+ specs:
20
+ activemodel (7.1.1)
21
+ activesupport (= 7.1.1)
22
+ activerecord (7.1.1)
23
+ activemodel (= 7.1.1)
24
+ activesupport (= 7.1.1)
25
+ timeout (>= 0.4.0)
26
+ activesupport (7.1.1)
27
+ base64
28
+ bigdecimal
29
+ concurrent-ruby (~> 1.0, >= 1.0.2)
30
+ connection_pool (>= 2.2.5)
31
+ drb
32
+ i18n (>= 1.6, < 2)
33
+ minitest (>= 5.1)
34
+ mutex_m
35
+ tzinfo (~> 2.0)
36
+ amq-protocol (2.3.2)
37
+ ast (2.4.2)
38
+ base64 (0.1.1)
39
+ bigdecimal (3.1.4)
40
+ bunny (2.22.0)
41
+ amq-protocol (~> 2.3, >= 2.3.1)
42
+ sorted_set (~> 1, >= 1.0.2)
43
+ byebug (11.1.3)
44
+ carrot-top (0.0.7)
45
+ json
46
+ concurrent-ruby (1.2.2)
47
+ connection_pool (2.4.1)
48
+ crypt_keeper (0.13.1)
49
+ activerecord (>= 3.0)
50
+ activesupport (>= 3.0)
51
+ datadog-ci (0.5.1)
52
+ msgpack
53
+ ddtrace (1.18.0)
54
+ datadog-ci (~> 0.5.0)
55
+ debase-ruby_core_source (= 3.2.3)
56
+ libdatadog (~> 5.0.0.1.0)
57
+ libddwaf (~> 1.14.0.0.0)
58
+ msgpack
59
+ debase-ruby_core_source (3.2.3)
60
+ diff-lcs (1.5.0)
61
+ dogstatsd-ruby (5.5.0)
62
+ drb (2.1.1)
63
+ ruby2_keywords
64
+ dry-configurable (1.0.1)
65
+ dry-core (~> 1.0, < 2)
66
+ zeitwerk (~> 2.6)
67
+ dry-container (0.11.0)
68
+ concurrent-ruby (~> 1.0)
69
+ dry-core (1.0.0)
70
+ concurrent-ruby (~> 1.0)
71
+ zeitwerk (~> 2.6)
72
+ dry-events (1.0.1)
73
+ concurrent-ruby (~> 1.0)
74
+ dry-core (~> 1.0, < 2)
75
+ dry-inflector (1.0.0)
76
+ dry-logic (1.5.0)
77
+ concurrent-ruby (~> 1.0)
78
+ dry-core (~> 1.0, < 2)
79
+ zeitwerk (~> 2.6)
80
+ dry-monitor (1.0.1)
81
+ dry-configurable (~> 1.0, < 2)
82
+ dry-core (~> 1.0, < 2)
83
+ dry-events (~> 1.0, < 2)
84
+ dry-struct (1.6.0)
85
+ dry-core (~> 1.0, < 2)
86
+ dry-types (>= 1.7, < 2)
87
+ ice_nine (~> 0.11)
88
+ zeitwerk (~> 2.6)
89
+ dry-types (1.7.1)
90
+ concurrent-ruby (~> 1.0)
91
+ dry-core (~> 1.0)
92
+ dry-inflector (~> 1.0)
93
+ dry-logic (~> 1.4)
94
+ zeitwerk (~> 2.6)
95
+ et-orbi (1.2.7)
96
+ tzinfo
97
+ exponential-backoff (0.0.4)
98
+ ffi (1.16.3)
99
+ file-based-healthcheck (0.2.0)
100
+ activesupport (>= 3.2)
101
+ fugit (1.8.1)
102
+ et-orbi (~> 1, >= 1.2.7)
103
+ raabro (~> 1.4)
104
+ globalid (1.2.1)
105
+ activesupport (>= 6.1)
106
+ hermes-rb (0.8.0)
107
+ activerecord (>= 5)
108
+ activesupport (>= 5)
109
+ dry-container (~> 0)
110
+ dry-struct (~> 1)
111
+ hutch (~> 1.0)
112
+ request_store (~> 1)
113
+ hutch (1.2.0)
114
+ activesupport (>= 4.2, < 8)
115
+ bunny (>= 2.19, < 3.0)
116
+ carrot-top (~> 0.0.7)
117
+ multi_json (~> 1.15)
118
+ i18n (1.14.1)
119
+ concurrent-ruby (~> 1.0)
120
+ ice_nine (0.11.2)
121
+ json (2.6.2)
122
+ karafka (2.2.7)
123
+ karafka-core (>= 2.2.2, < 2.3.0)
124
+ thor (>= 0.20)
125
+ waterdrop (>= 2.6.6, < 3.0.0)
126
+ zeitwerk (~> 2.3)
127
+ karafka-core (2.2.2)
128
+ concurrent-ruby (>= 1.1)
129
+ karafka-rdkafka (>= 0.13.1, < 0.14.0)
130
+ karafka-rdkafka (0.13.5)
131
+ ffi (~> 1.15)
132
+ mini_portile2 (~> 2.6)
133
+ rake (> 12)
134
+ libdatadog (5.0.0.1.0)
135
+ libddwaf (1.14.0.0.0)
136
+ ffi (~> 1.0)
137
+ mini_portile2 (2.8.4)
138
+ minitest (5.20.0)
139
+ msgpack (1.7.2)
140
+ multi_json (1.15.0)
141
+ mutex_m (0.1.2)
142
+ parallel (1.22.1)
143
+ parser (3.1.2.1)
144
+ ast (~> 2.4.1)
145
+ pg (1.3.5)
146
+ raabro (1.4.0)
147
+ rack (2.2.4)
148
+ rainbow (3.1.1)
149
+ rake (13.0.6)
150
+ rbtree (0.4.6)
151
+ redis (4.2.5)
152
+ redis-namespace (1.8.2)
153
+ redis (>= 3.0.4)
154
+ redlock (1.2.2)
155
+ redis (>= 3.0.0, < 5.0)
156
+ regexp_parser (2.5.0)
157
+ request_store (1.5.1)
158
+ rack (>= 1.4)
159
+ rexml (3.2.5)
160
+ rspec (3.12.0)
161
+ rspec-core (~> 3.12.0)
162
+ rspec-expectations (~> 3.12.0)
163
+ rspec-mocks (~> 3.12.0)
164
+ rspec-core (3.12.2)
165
+ rspec-support (~> 3.12.0)
166
+ rspec-expectations (3.12.3)
167
+ diff-lcs (>= 1.2.0, < 2.0)
168
+ rspec-support (~> 3.12.0)
169
+ rspec-mocks (3.12.6)
170
+ diff-lcs (>= 1.2.0, < 2.0)
171
+ rspec-support (~> 3.12.0)
172
+ rspec-retry (0.6.2)
173
+ rspec-core (> 3.3)
174
+ rspec-sidekiq (3.1.0)
175
+ rspec-core (~> 3.0, >= 3.0.0)
176
+ sidekiq (>= 2.4.0)
177
+ rspec-support (3.12.1)
178
+ rubocop (1.35.0)
179
+ json (~> 2.3)
180
+ parallel (~> 1.10)
181
+ parser (>= 3.1.2.1)
182
+ rainbow (>= 2.2.2, < 4.0)
183
+ regexp_parser (>= 1.8, < 3.0)
184
+ rexml (>= 3.2.5, < 4.0)
185
+ rubocop-ast (>= 1.20.1, < 2.0)
186
+ ruby-progressbar (~> 1.7)
187
+ unicode-display_width (>= 1.4.0, < 3.0)
188
+ rubocop-ast (1.21.0)
189
+ parser (>= 3.1.1.0)
190
+ rubocop-performance (1.14.3)
191
+ rubocop (>= 1.7.0, < 2.0)
192
+ rubocop-ast (>= 0.4.0)
193
+ rubocop-rake (0.6.0)
194
+ rubocop (~> 1.0)
195
+ rubocop-rspec (2.12.1)
196
+ rubocop (~> 1.31)
197
+ ruby-progressbar (1.11.0)
198
+ ruby2_keywords (0.0.5)
199
+ sentry-ruby (5.4.1)
200
+ concurrent-ruby (~> 1.0, >= 1.0.2)
201
+ set (1.0.3)
202
+ shoulda-matchers (5.2.0)
203
+ activesupport (>= 5.2.0)
204
+ sidekiq (6.5.1)
205
+ connection_pool (>= 2.2.2)
206
+ rack (~> 2.0)
207
+ redis (>= 4.2.0)
208
+ sidekiq-cron (1.10.1)
209
+ fugit (~> 1.8)
210
+ globalid (>= 1.0.1)
211
+ sidekiq (>= 6)
212
+ sigurd (0.1.0)
213
+ concurrent-ruby (~> 1)
214
+ exponential-backoff
215
+ sorted_set (1.0.3)
216
+ rbtree
217
+ set (~> 1.0)
218
+ thor (1.2.2)
219
+ timecop (0.9.5)
220
+ timeout (0.4.0)
221
+ tzinfo (2.0.6)
222
+ concurrent-ruby (~> 1.0)
223
+ unicode-display_width (2.2.0)
224
+ waterdrop (2.6.7)
225
+ karafka-core (>= 2.1.1, < 3.0.0)
226
+ zeitwerk (~> 2.3)
227
+ zeitwerk (2.6.12)
228
+
229
+ PLATFORMS
230
+ ruby
231
+
232
+ DEPENDENCIES
233
+ activerecord (~> 7.1)
234
+ bundler
235
+ byebug
236
+ crypt_keeper
237
+ ddtrace
238
+ dionysus-rb!
239
+ dogstatsd-ruby
240
+ hermes-rb
241
+ pg
242
+ rake (~> 13.0)
243
+ redis
244
+ redis-namespace
245
+ redlock
246
+ rspec (~> 3.0)
247
+ rspec-retry
248
+ rspec-sidekiq
249
+ rubocop
250
+ rubocop-performance
251
+ rubocop-rake
252
+ rubocop-rspec
253
+ sentry-ruby
254
+ shoulda-matchers
255
+ timecop
256
+
257
+ BUNDLED WITH
258
+ 2.4.14
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2023 Karol Galanciak
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.