dionysus-rb 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (102) hide show
  1. checksums.yaml +7 -0
  2. data/.circleci/config.yml +61 -0
  3. data/.github/workflows/ci.yml +77 -0
  4. data/.gitignore +12 -0
  5. data/.rspec +3 -0
  6. data/.rubocop.yml +175 -0
  7. data/.rubocop_todo.yml +53 -0
  8. data/CHANGELOG.md +227 -0
  9. data/Gemfile +10 -0
  10. data/Gemfile.lock +258 -0
  11. data/LICENSE.txt +21 -0
  12. data/README.md +1206 -0
  13. data/Rakefile +10 -0
  14. data/assets/logo.svg +51 -0
  15. data/bin/console +11 -0
  16. data/bin/karafka_health_check +14 -0
  17. data/bin/outbox_worker_health_check +12 -0
  18. data/bin/setup +8 -0
  19. data/dionysus-rb.gemspec +64 -0
  20. data/docker-compose.yml +44 -0
  21. data/lib/dionysus/checks/health_check.rb +50 -0
  22. data/lib/dionysus/checks.rb +7 -0
  23. data/lib/dionysus/consumer/batch_events_publisher.rb +33 -0
  24. data/lib/dionysus/consumer/config.rb +97 -0
  25. data/lib/dionysus/consumer/deserializer.rb +231 -0
  26. data/lib/dionysus/consumer/dionysus_event.rb +42 -0
  27. data/lib/dionysus/consumer/karafka_consumer_generator.rb +56 -0
  28. data/lib/dionysus/consumer/params_batch_processor.rb +65 -0
  29. data/lib/dionysus/consumer/params_batch_transformations/remove_duplicates_strategy.rb +54 -0
  30. data/lib/dionysus/consumer/params_batch_transformations.rb +4 -0
  31. data/lib/dionysus/consumer/persistor.rb +157 -0
  32. data/lib/dionysus/consumer/registry.rb +84 -0
  33. data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data.rb +27 -0
  34. data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data_job.rb +26 -0
  35. data/lib/dionysus/consumer/synced_data.rb +4 -0
  36. data/lib/dionysus/consumer/synchronizable_model.rb +93 -0
  37. data/lib/dionysus/consumer/workers_group.rb +18 -0
  38. data/lib/dionysus/consumer.rb +36 -0
  39. data/lib/dionysus/monitor.rb +48 -0
  40. data/lib/dionysus/producer/base_responder.rb +46 -0
  41. data/lib/dionysus/producer/config.rb +104 -0
  42. data/lib/dionysus/producer/deleted_record_serializer.rb +17 -0
  43. data/lib/dionysus/producer/genesis/performed.rb +11 -0
  44. data/lib/dionysus/producer/genesis/stream_job.rb +13 -0
  45. data/lib/dionysus/producer/genesis/streamer/base_job.rb +44 -0
  46. data/lib/dionysus/producer/genesis/streamer/standard_job.rb +43 -0
  47. data/lib/dionysus/producer/genesis/streamer.rb +40 -0
  48. data/lib/dionysus/producer/genesis.rb +62 -0
  49. data/lib/dionysus/producer/karafka_responder_generator.rb +133 -0
  50. data/lib/dionysus/producer/key.rb +14 -0
  51. data/lib/dionysus/producer/model_serializer.rb +105 -0
  52. data/lib/dionysus/producer/outbox/active_record_publishable.rb +74 -0
  53. data/lib/dionysus/producer/outbox/datadog_latency_reporter.rb +26 -0
  54. data/lib/dionysus/producer/outbox/datadog_latency_reporter_job.rb +11 -0
  55. data/lib/dionysus/producer/outbox/datadog_latency_reporter_scheduler.rb +47 -0
  56. data/lib/dionysus/producer/outbox/datadog_tracer.rb +32 -0
  57. data/lib/dionysus/producer/outbox/duplicates_filter.rb +26 -0
  58. data/lib/dionysus/producer/outbox/event_name.rb +26 -0
  59. data/lib/dionysus/producer/outbox/health_check.rb +48 -0
  60. data/lib/dionysus/producer/outbox/latency_tracker.rb +43 -0
  61. data/lib/dionysus/producer/outbox/model.rb +117 -0
  62. data/lib/dionysus/producer/outbox/producer.rb +26 -0
  63. data/lib/dionysus/producer/outbox/publishable.rb +106 -0
  64. data/lib/dionysus/producer/outbox/publisher.rb +131 -0
  65. data/lib/dionysus/producer/outbox/records_processor.rb +56 -0
  66. data/lib/dionysus/producer/outbox/runner.rb +120 -0
  67. data/lib/dionysus/producer/outbox/tombstone_publisher.rb +22 -0
  68. data/lib/dionysus/producer/outbox.rb +103 -0
  69. data/lib/dionysus/producer/partition_key.rb +42 -0
  70. data/lib/dionysus/producer/registry/validator.rb +32 -0
  71. data/lib/dionysus/producer/registry.rb +165 -0
  72. data/lib/dionysus/producer/serializer.rb +52 -0
  73. data/lib/dionysus/producer/suppressor.rb +18 -0
  74. data/lib/dionysus/producer.rb +121 -0
  75. data/lib/dionysus/railtie.rb +9 -0
  76. data/lib/dionysus/rb/version.rb +5 -0
  77. data/lib/dionysus/rb.rb +8 -0
  78. data/lib/dionysus/support/rspec/outbox_publishable.rb +78 -0
  79. data/lib/dionysus/topic_name.rb +15 -0
  80. data/lib/dionysus/utils/default_message_filter.rb +25 -0
  81. data/lib/dionysus/utils/exponential_backoff.rb +7 -0
  82. data/lib/dionysus/utils/karafka_datadog_listener.rb +20 -0
  83. data/lib/dionysus/utils/karafka_sentry_listener.rb +9 -0
  84. data/lib/dionysus/utils/null_error_handler.rb +6 -0
  85. data/lib/dionysus/utils/null_event_bus.rb +5 -0
  86. data/lib/dionysus/utils/null_hermes_event_producer.rb +5 -0
  87. data/lib/dionysus/utils/null_instrumenter.rb +7 -0
  88. data/lib/dionysus/utils/null_lock_client.rb +13 -0
  89. data/lib/dionysus/utils/null_model_factory.rb +5 -0
  90. data/lib/dionysus/utils/null_mutex_provider.rb +7 -0
  91. data/lib/dionysus/utils/null_retry_provider.rb +7 -0
  92. data/lib/dionysus/utils/null_tracer.rb +5 -0
  93. data/lib/dionysus/utils/null_transaction_provider.rb +15 -0
  94. data/lib/dionysus/utils/sidekiq_batched_job_distributor.rb +24 -0
  95. data/lib/dionysus/utils.rb +6 -0
  96. data/lib/dionysus/version.rb +7 -0
  97. data/lib/dionysus-rb.rb +3 -0
  98. data/lib/dionysus.rb +133 -0
  99. data/lib/tasks/dionysus.rake +18 -0
  100. data/log/development.log +0 -0
  101. data/sig/dionysus/rb.rbs +6 -0
  102. metadata +585 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 51b0fe501a9678e2fc887a1525a40a0ddc6e04d9a2d49e8211305370eef4e843
4
+ data.tar.gz: 82597c73fa710cbac12f091e6f8c26462aae3dfc7572de180cbc474c70cedf1a
5
+ SHA512:
6
+ metadata.gz: c7f05381d3fc7e3eba08e4666a46b71e1bc52425d56a3a0d46c5fde4287fdcc77bc64c4521d4f76307849e1cb017e07ab9d612dc127c72bc3fe5e23d919c994e
7
+ data.tar.gz: 13e9c0664deb5453e9510c0cba835222cb487ff3511696bdcfce4ec09f8677eb16fd4b25bafd4748275cc0533bd62ac3a524761d673d368666153ff79892cb40
@@ -0,0 +1,61 @@
1
+ version: 2
2
+ jobs:
3
+ build:
4
+ working_directory: ~/BookingSync/dionysus-rb
5
+ parallelism: 1
6
+ docker:
7
+ - image: cimg/ruby:2.7.6
8
+ environment:
9
+ POSTGRES_URL: "postgres://docker:docker@127.0.0.1:5432"
10
+ DATABASE_URL: "postgres://docker:docker@127.0.0.1:5432/circle_test"
11
+ DATABASE_NAME: "dionysus-rb"
12
+ - image: redis
13
+ - image: wurstmeister/zookeeper
14
+ - image: wurstmeister/kafka:2.11-0.11.0.3
15
+ environment:
16
+ KAFKA_ADVERTISED_HOST_NAME: localhost
17
+ KAFKA_ADVERTISED_PORT: 9092
18
+ KAFKA_PORT: 9092
19
+ KAFKA_DELETE_TOPIC_ENABLE: true
20
+ KAFKA_CREATE_TOPICS: "v102_rentals:1:1"
21
+ REDIS_URL: redis://localhost:6379/1
22
+ - image: circleci/postgres:12.6
23
+ environment:
24
+ POSTGRES_USER: docker
25
+ POSTGRES_PASSWORD: docker
26
+ steps:
27
+ - checkout
28
+ - run: mkdir /tmp/circleci-test-results
29
+ - restore_cache:
30
+ keys:
31
+ - v1-gemfile-lock-{{ checksum "Gemfile.lock" }}
32
+ - run:
33
+ name: Installing dependencies
34
+ command: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs 4 --retry 3
35
+ - save_cache:
36
+ key: v1-gemfile-lock-{{ checksum "Gemfile.lock" }}
37
+ paths:
38
+ - vendor/bundle
39
+ - ~/.bundle
40
+ - run:
41
+ name: Installing dockerize
42
+ command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz
43
+ environment:
44
+ DOCKERIZE_VERSION: v0.6.1
45
+ - run: sudo apt update && sudo apt install -y postgresql-client || true
46
+ - run:
47
+ name: Wait for db
48
+ command: dockerize -wait tcp://localhost:5432 -timeout 2m
49
+ - run:
50
+ name: Running Rubocop
51
+ command: bundle exec rubocop
52
+ - run:
53
+ name: Running specs
54
+ command: bundle exec rspec
55
+ environment:
56
+ RAILS_ENV: test
57
+ RACK_ENV: test
58
+ POSTGRES_USER: docker
59
+ POSTGRES_PASSWORD: docker
60
+ - store_test_results:
61
+ path: /tmp/circleci-test-results
@@ -0,0 +1,77 @@
1
+ name: CI
2
+ on: [pull_request]
3
+ jobs:
4
+ rubocop:
5
+ strategy:
6
+ fail-fast: true
7
+ runs-on: ubuntu-latest
8
+ steps:
9
+ - uses: actions/checkout@v2
10
+ - uses: ruby/setup-ruby@v1
11
+ with:
12
+ ruby-version: 2.7
13
+ bundler-cache: true
14
+ - run: bundle exec rubocop
15
+ rspec:
16
+ timeout-minutes: 10
17
+ strategy:
18
+ fail-fast: false
19
+ matrix:
20
+ ruby: ['2.7', '3.0.6', '3.1', '3.2']
21
+ runs-on: ubuntu-latest
22
+ env:
23
+ DD_PROFILING_NO_EXTENSION: true
24
+ RAILS_ENV: test
25
+ RACK_ENV: test
26
+ REDIS_URL: redis://localhost:6379/1
27
+ POSTGRES_USER: postgres
28
+ POSTGRES_PASSWORD: postgres
29
+ POSTGRES_DB: dionysus-rb-test
30
+ CLUSTER_ID: kafka-docker-cluster-1
31
+ KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
32
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
33
+ KAFKA_PROCESS_ROLES: broker,controller
34
+ KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
35
+ KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
36
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
37
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
38
+ KAFKA_BROKER_ID: 1
39
+ KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
40
+ ALLOW_PLAINTEXT_LISTENER: 'yes'
41
+ KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
42
+ KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
43
+ KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
44
+ KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
45
+ KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
46
+ KAFKA_CREATE_TOPICS: "v102_rentals:1:1"
47
+ services:
48
+ postgres:
49
+ image: postgres:14
50
+ env:
51
+ POSTGRES_USER: postgres
52
+ POSTGRES_PASSWORD: postgres
53
+ POSTGRES_DB: dionysus-rb-test
54
+ ports:
55
+ - 5432:5432
56
+ options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
57
+ redis:
58
+ image: redis
59
+ ports:
60
+ - 6379:6379
61
+ options: >-
62
+ --health-cmd "redis-cli ping"
63
+ --health-interval 10s
64
+ --health-timeout 5s
65
+ --health-retries 5
66
+ steps:
67
+ - uses: actions/checkout@v2
68
+ - name: Start Kafka with docker-compose
69
+ run: |
70
+ docker-compose up -d || (sleep 5 && docker-compose up -d)
71
+
72
+ - uses: ruby/setup-ruby@v1
73
+
74
+ with:
75
+ ruby-version: ${{ matrix.ruby }}
76
+ bundler-cache: true
77
+ - run: bundle exec rspec
data/.gitignore ADDED
@@ -0,0 +1,12 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
12
+ .byebug_history
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.rubocop.yml ADDED
@@ -0,0 +1,175 @@
1
+ inherit_from: .rubocop_todo.yml
2
+
3
+ require:
4
+ - rubocop-performance
5
+ - rubocop-rspec
6
+ - rubocop-rake
7
+
8
+ inherit_mode:
9
+ merge:
10
+ - Exclude
11
+
12
+ AllCops:
13
+ NewCops: enable
14
+ TargetRubyVersion: 2.7
15
+ Exclude:
16
+ - "db/**/*"
17
+ - "bin/**/*"
18
+ - "tmp/**/*"
19
+ - "log/**/*"
20
+ - "vendor/**/*"
21
+ - "spec/spec_helper.rb"
22
+
23
+ Layout/ArgumentAlignment:
24
+ EnforcedStyle: with_fixed_indentation
25
+
26
+ Layout/ArrayAlignment:
27
+ EnforcedStyle: with_fixed_indentation
28
+
29
+ Layout/EmptyLineBetweenDefs:
30
+ AllowAdjacentOneLineDefs: true
31
+
32
+ Layout/EndAlignment:
33
+ EnforcedStyleAlignWith: variable
34
+
35
+ Layout/FirstArgumentIndentation:
36
+ EnforcedStyle: consistent
37
+
38
+ Layout/FirstArrayElementIndentation:
39
+ EnforcedStyle: consistent
40
+
41
+ Layout/FirstHashElementIndentation:
42
+ EnforcedStyle: consistent
43
+
44
+ Layout/MultilineMethodCallIndentation:
45
+ EnforcedStyle: indented
46
+
47
+ Layout/MultilineOperationIndentation:
48
+ EnforcedStyle: indented
49
+
50
+ Layout/ParameterAlignment:
51
+ EnforcedStyle: with_fixed_indentation
52
+
53
+ Layout/SpaceBeforeBrackets:
54
+ Enabled: false
55
+
56
+ Lint/UnusedMethodArgument:
57
+ AllowUnusedKeywordArguments: true
58
+
59
+ Metrics/ParameterLists:
60
+ MaxOptionalParameters: 4
61
+ Exclude:
62
+ - 'lib/dionysus.rb'
63
+ - 'lib/dionysus/producer/genesis/stream_job.rb'
64
+ - 'lib/dionysus/producer/genesis.rb'
65
+
66
+ RSpec/ExpectChange:
67
+ EnforcedStyle: block
68
+
69
+ RSpec/LetSetup:
70
+ Enabled: false
71
+
72
+ RSpec/MultipleExpectations:
73
+ Max: 25
74
+ Exclude:
75
+ - 'spec/dionysus/consumer/registry_spec.rb'
76
+ - 'spec/dionysus/producer_spec.rb'
77
+ - 'spec/dionysus/producer/registry_spec.rb'
78
+
79
+ Style/Alias:
80
+ EnforcedStyle: prefer_alias_method
81
+
82
+ Style/ClassAndModuleChildren:
83
+ EnforcedStyle: compact
84
+ Exclude:
85
+ - 'lib/dionysus/version.rb'
86
+ - 'lib/dionysus/checks.rb'
87
+ - 'lib/dionysus.rb'
88
+
89
+ Style/CommandLiteral:
90
+ EnforcedStyle: percent_x
91
+
92
+ Style/RescueStandardError:
93
+ EnforcedStyle: implicit
94
+
95
+ Style/StringLiterals:
96
+ EnforcedStyle: double_quotes
97
+ ConsistentQuotesInMultiline: true
98
+
99
+ Style/StringLiteralsInInterpolation:
100
+ EnforcedStyle: double_quotes
101
+
102
+ Style/RaiseArgs:
103
+ EnforcedStyle: compact
104
+
105
+ Style/RegexpLiteral:
106
+ EnforcedStyle: percent_r
107
+
108
+ Style/Documentation:
109
+ Enabled: false
110
+
111
+ RSpec/NestedGroups:
112
+ Max: 10
113
+
114
+ RSpec/MultipleMemoizedHelpers:
115
+ Enabled: false
116
+
117
+ Lint/AmbiguousBlockAssociation:
118
+ Exclude:
119
+ - 'spec/**/*'
120
+ - 'lib/dionysus/support/rspec/outbox_publishable.rb'
121
+
122
+ RSpec/ExampleLength:
123
+ Max: 50
124
+ Exclude:
125
+ - 'spec/dionysus/consumer/registry_spec.rb'
126
+ - 'spec/dionysus/producer_spec.rb'
127
+ - 'spec/dionysus/producer/registry_spec.rb'
128
+
129
+ Naming/VariableNumber:
130
+ Enabled: false
131
+
132
+ Metrics/BlockLength:
133
+ Max: 30
134
+ Exclude:
135
+ - 'spec/**/*'
136
+ - 'lib/dionysus/support/rspec/outbox_publishable.rb'
137
+ - 'lib/dionysus/consumer/persistor.rb'
138
+
139
+ Naming/PredicateName:
140
+ Enabled: false
141
+
142
+ Metrics/AbcSize:
143
+ Enabled: false
144
+
145
+ Metrics/CyclomaticComplexity:
146
+ Max: 15
147
+
148
+ Metrics/MethodLength:
149
+ Enabled: false
150
+
151
+ Naming/AccessorMethodName:
152
+ Exclude:
153
+ - 'spec/dionysus/utils/karafka_datadog_listener_spec.rb'
154
+
155
+ RSpec/DescribeClass:
156
+ Exclude:
157
+ - 'spec/integration_spec.rb'
158
+
159
+ Style/MultilineBlockChain:
160
+ Exclude:
161
+ - 'spec/dionysus/producer/model_serializer_spec.rb'
162
+ - 'spec/dionysus/producer/serializer_spec.rb'
163
+
164
+ Performance/MethodObjectAsBlock:
165
+ Enabled: false
166
+
167
+ Metrics/ClassLength:
168
+ Max: 150
169
+
170
+ Layout/LineLength:
171
+ Max: 125
172
+
173
+ Naming/FileName:
174
+ Exclude:
175
+ - 'lib/dionysus-rb.rb'
data/.rubocop_todo.yml ADDED
@@ -0,0 +1,53 @@
1
+ RSpec/VerifiedDoubles:
2
+ Enabled: false
3
+
4
+ RSpec/AnyInstance:
5
+ Exclude:
6
+ - 'spec/dionysus/consumer/karafka_consumer_generator_spec.rb'
7
+ - 'spec/dionysus/producer/outbox/datadog_latency_reporter_job_spec.rb'
8
+ - 'spec/dionysus/producer/outbox/active_record_publishable_spec.rb'
9
+
10
+ Layout/LineLength:
11
+ Exclude:
12
+ - 'lib/dionysus.rb'
13
+ - 'lib/dionysus/consumer/persistor.rb'
14
+ - 'lib/dionysus/producer/genesis/streamer/standard_job.rb'
15
+ - 'spec/dionysus/consumer/karafka_consumer_generator_spec.rb'
16
+ - 'spec/dionysus/producer/outbox/publisher_spec.rb'
17
+ - 'spec/dionysus/producer/genesis_spec.rb'
18
+
19
+ Lint/NonLocalExitFromIterator:
20
+ Exclude:
21
+ - 'lib/dionysus/producer/karafka_responder_generator.rb'
22
+ - 'lib/dionysus/consumer/persistor.rb'
23
+
24
+ Metrics/PerceivedComplexity:
25
+ Exclude:
26
+ - 'lib/dionysus/producer/karafka_responder_generator.rb'
27
+ - 'lib/dionysus/consumer/karafka_consumer_generator.rb'
28
+ - 'lib/dionysus/consumer/persistor.rb'
29
+
30
+ RSpec/LeakyConstantDeclaration:
31
+ Exclude:
32
+ - 'spec/dionysus/producer_spec.rb'
33
+
34
+ Lint/ConstantDefinitionInBlock:
35
+ Exclude:
36
+ - 'spec/dionysus/producer_spec.rb'
37
+
38
+ Lint/EmptyClass:
39
+ Exclude:
40
+ - 'lib/dionysus/consumer/synced_data.rb'
41
+ - 'lib/dionysus/consumer/params_batch_transformations.rb'
42
+
43
+ RSpec/ScatteredSetup:
44
+ Exclude:
45
+ - 'spec/dionysus/producer/outbox/records_processor_spec.rb'
46
+
47
+ Metrics/CyclomaticComplexity:
48
+ Exclude:
49
+ - 'lib/dionysus/producer/karafka_responder_generator.rb'
50
+
51
+ Metrics/BlockLength:
52
+ Exclude:
53
+ - 'lib/dionysus/producer/karafka_responder_generator.rb'
data/CHANGELOG.md ADDED
@@ -0,0 +1,227 @@
1
+ ## [Unreleased]
2
+
3
+ ## [0.1.0]
4
+
5
+ - Turn private gem `bookingsync-prometheus` into `dionysus-rb` and release it publicly
6
+ - Allow to customize `consumer_group_prefix`
7
+ - Support Ruby 3.2
8
+
9
+
10
+ Migration path from `bookingsync-prometheus`:
11
+ - Rename all references of `Bookingsync::Prometheus` to `Dionysus`
12
+ - **CRITICAL** - when calling `Dionysus.initialize_application!` (previously `Bookingsync::Prometheus.initialize_application!`) provide a new keyword argument: `consumer_group_prefix` with a name of `prometheus_consumer_group_for` to preserve the same consumer group
13
+ - For producers, `config.hermes_event_producer` no longer defaults to `Hermes::EventProducer`, if you want to preserve the behavior make sure to configure this attribute explicitly: `config.hermes_event_producer = Hermes::EventProducer`
14
+ - `Bookingsync::Prometheus::Producer::Genesis::Performed` event was renamed to `Dionysus::Producer::Genesis::Performed`
15
+ - In the instrumentation context, all references to `prometheus`/`bookingsync_prometheus` were replaced to `dionysus`, which might require some renaming in the apps or observability tools.
16
+ - In the Datadog context (e.g. for `Dionysus::Producer::Outbox::DatadogLatencyReporter`), all references to `bookingsync_prometheus` were replaced to `dionysus`, which might require some renaming in the apps or observability tools, especially if you have a monitor based on this metric.
17
+ - In the Event Bus context, all references to `prometheus` were replaced to `dionysus`, which might require some renaming in the apps.
18
+
19
+
20
+ ## Changelog from bookingsync-prometheus gem (original private gem that was renamed to dionysus-rb) - keeping it for historical reasons as the commits are not preserved since it's a new repo
21
+
22
+ ## [1.17.2]
23
+ - Get back to using `in_batches` as the combination with `lazy` does the job and looks better han `find_in_batches`
24
+
25
+ ## [1.17.1]
26
+ - Address memory leak in `Bookingsync::Prometheus::Producer::Genesis::Streamer::BaseJob` caused by the fact that the way `in_batches` was used was returning all the records in the end which was not needed.
27
+
28
+ ## [1.17.0]
29
+ - Change batch size for Genesis to 1000 as it's a wiser default than 100 (and it's the same as ActiveRecord's default for `in_batches`).
30
+
31
+ ## [1.16.1]
32
+ - Do not create outbox records for Genesis Replica topics as that would mean the same record would be published twice to Genesis Replica (first time with the standard topic, the second time with the Genesis one)
33
+
34
+ ## [1.16.0]
35
+ - Introduce `genesis_replica: true` option for topics which assumes existence of twin topic where the storage would be infinite.
36
+ - Due to internal changes in responders to handle `genesis_replica`, it no longer makes sense to have an instrumentation in the responder following the pattern `prometheus.respond.#{topic_name}"`, but rather `"prometheus.respond.#{responder_class_name}"`, which should have been the case from the beginning.
37
+
38
+ ## [1.15.0]
39
+ - Allow lambda (or an object responding to `call` method) for `partition_key` that takes a single argument - the resource
40
+
41
+ ## [1.14.0]
42
+ - Introduce message filter on consumer level to allow filtering out messages that are not desired for some reason (e.g. the are too big and cause OOMs).
43
+
44
+ ## [1.13.0]
45
+ - Provide an option to chain methods when resolving `association` for Observers (via `association_name`).
46
+
47
+ ## [1.12.0]
48
+ - Introduce `Bookingsync::Prometheus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy` and `remove_consecutive_duplicates_before_publishing` config option.
49
+
50
+ ## [1.11.0]
51
+ - Adjust `Bookingsync::Prometheus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy` to sort messages by timestamps before dropping duplicates to not assume that the messages are ordered.
52
+
53
+ ## [1.10.2]
54
+ - Make `bin/karafka_health_check` more lightweight by loading only the check instead of whole prometheus and rails
55
+
56
+ ## [1.10.1]
57
+ - Make `resource` to-one relationship optional to not blow up the Rails validations
58
+
59
+ ## [1.10.0]
60
+
61
+ - Introduce `Bookingsync::Prometheus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy` and use it by default
62
+ - Do not set @transactional_outbox_enabled when calling the reader in the scenario where the variable is not set, just return the default
63
+
64
+ ## [1.9.2]
65
+
66
+ - Make `Bookingsync::Prometheus::Checks::HealthCheck` even less aggressive - increase `expiry_time_in_seconds`.
67
+
68
+ ## [1.9.1]
69
+
70
+ - Make `Bookingsync::Prometheus::Checks::HealthCheck` less aggressive - increase `expiry_time_in_seconds`.
71
+
72
+ ## [1.9.0]
73
+
74
+ - Provide a service to assign values to columns from `synced_data`
75
+
76
+ ## [1.8.0]
77
+
78
+ - Do not allow to execute Genesis for a model that is only a dependency to prevent enqueuing of duplicates.
79
+
80
+ ## [1.7.0]
81
+
82
+ - Rescue from `publish_outbox_records` (inline handling of outbox publishing) and let the worker retry later.
83
+ - Do not send excessive notifications via error handler from `Bookingsync::Prometheus::Producer::Outbox::Publisher`, use logger instead.
84
+
85
+ ## [1.6.0]
86
+
87
+ - Handle the case of a massive number of observers to be published - instead of publishing them inline (beyond a threshold), execute Genesis for them
88
+
89
+ ## [1.5.0]
90
+
91
+ - Do not blow up with an exception when outbox publishes _updated event for a deleted record, send a notification instead
92
+
93
+ ## [1.4.1]
94
+
95
+ - Register a heartbeat for a healthcheck when the messages are consumed
96
+
97
+ ## [1.4.0]
98
+
99
+ - Move to file-based healthchecks, instead of using Redis-based ones.
100
+
101
+ ## [1.3.0]
102
+
103
+ - Make health checks for Karafka work with liveness probe as well if statistics emission is enabled (check Readme for more details)
104
+
105
+ ## [1.2.0]
106
+
107
+ - report `"#{namespace}.bookingsync_prometheus.producer.outbox.latency.highest_since_creation_date` to Datadog
108
+
109
+ ## [1.1.1]
110
+
111
+ - Do not publish records for _created events if the they don't exist
112
+ - Do not cancel the dependent records of the aggregate for _destroyed event, only do this via attributes assignment in such a case
113
+
114
+ ## [1.1.0]
115
+
116
+ - Allow extra configuration of topics to take advantage of Karafka 2.0 features like Dead Letter Queue
117
+ - Fix specs for KarafkaSentryListener and KarafkaDatadogListener for unknown events, as the behavior of Karafka monitor changed
118
+
119
+ ## [1.0.0]
120
+
121
+ - [BREAKING CHANGES] Require Karafka 2.0 which has a lot of consequences
122
+ - `Bookingsync::Prometheus.initialize_application!` no longer takes `batch_fetching` and `backend` arguments
123
+ - `sidekiq-backend` is no longer supported
124
+ - batch fetching/batch consuming is always enabled. `config.batch_consuming` option is no longer available.
125
+ - `seed_brokers` argument provided to `Bookingsync::Prometheus.initialize_application!` should no longer contain the protocol, i.e. use `["localhost:9092"]` instead of `["kafka://localhost:9092"]`
126
+ - `ruby-kafka` is no longer used under the hood, it's replaced by `librdkafka`. Which implies a lot of config options are no longer available, e.g `kafka.fetcher_max_queue_size`, `kafka.offset_commit_interval`, `kafka.offset_commit_threshold`.
127
+ - when overriding kafka-specific settings in a block provided to `Bookingsync::Prometheus.initialize_application!`, you no longer can use writers to modify the config, instead, you modify the `kafka` hash settings. For example, instead of `config.kafka.heartbeat_interval = 120` you should use `config.kafka[:"heartbeat.interval.ms"] = 120_000`. Notice that the keys are symbols and the unit might be different (seconds vs. milliseconds for example).
128
+ - custom partition assignment strategy is no longer supported
129
+ - metrics from `ruby-kafka` are no longer available
130
+ - Use `kafka://localhost:9092` instead of `kafka://127.0.0.1:9092` in development environment
131
+ - There is no more `ruby-kafka` so the Datadog integration it was providing will also not work. Karafka/WaterDrop 2.0 comes with its own implementation: https://karafka.io/docs/Monitoring-and-logging/#datadog-and-statsd-integration, https://github.com/karafka/waterdrop#datadog-and-statsd-integration
132
+ - health check needs to be outside the `Rails.application.config.to_prepare` block
133
+ - Make sure you use `ddtrace` gem at 1.0 or higher
134
+ - Follow the guide: https://karafka.io/docs/Upgrades-2.0/
135
+ - If the application is just a producer, use the following config:
136
+
137
+ ``` rb
138
+ Karafka::App.setup do |config|
139
+ config.producer = ::WaterDrop::Producer.new do |producer_config|
140
+ producer_config.kafka = {
141
+ 'bootstrap.servers': ENV.fetch("PROMETHEUS_SEED_BROKER").gsub("kafka://", "").tr(";", ","), # this needs to be comma-separates list of brokers
142
+ 'request.required.acks': 1,
143
+ "client.id": :id_of_the_producer_goes_here.to_s
144
+ }
145
+ producer_config.id = :id_of_the_producer_goes_here.to_s
146
+ producer_config.deliver = true
147
+ end
148
+ end
149
+ ```
150
+
151
+ - Migration for some popular ruby-kafka options:
152
+ - `config.kafka.max_bytes_per_partition` => `config.kafka[:"max.partition.fetch.bytes"]`
153
+ - `config.kafka.socket_timeout` => `config.kafka[:"socket.timeout.ms"]` (the unit changes to milliseconds, you might need to multiply it by 1000)
154
+ - `config.kafka.session_timeout` => `config.kafka[:"session.timeout.ms"]` (the unit changes to milliseconds, you might need to multiply it by 1000)
155
+ - `config.kafka.heartbeat_interval` => `config.kafka[:"heartbeat.interval.ms"]` (the unit changes to milliseconds, you might need to multiply it by 1000)
156
+
157
+ ## [0.3.7]
158
+
159
+ - Fix inferring error inside `Bookingsync::Prometheus::Producer::Outbox::Model` if arity is different than -1 or 1
160
+
161
+
162
+ ## [0.3.6]
163
+
164
+ - Ensure publishing outbox observers won't blow up if to-one relationship is nil
165
+
166
+ ## [0.3.5]
167
+
168
+ - Remove `datadog_statsd` prefix in favour of `namespace`.
169
+
170
+ ## [0.3.4]
171
+
172
+ - Add latency tracking via Datadog for publishing via outbox
173
+
174
+ ## [0.3.3]
175
+
176
+ - Enforce order by resource's `created_at` when publishing from after_commit
177
+
178
+ ## [0.3.2]
179
+
180
+ - Explicitly require `concurrent/array` to avoid `uninitialized constant Concurrent::Array`.
181
+
182
+ ## [0.3.1]
183
+
184
+ - Fix serialization of deleted records
185
+ - Fix issue in the outbox publisher for the scenario where record is a dependency of the parent but no parents exist
186
+
187
+ ## [0.3.0]
188
+
189
+ - Allow publishing outbox records right after the transaction is committed
190
+ - Require `partition_key` column as otherwise, it's not possible resolve the partition key for deleted records
191
+
192
+ ## [0.2.7]
193
+
194
+ - Add column validation rake task
195
+
196
+ ## [0.2.6]
197
+
198
+ - Remove attributes validation in `Bookingsync::Prometheus::Producer::Registry` as it was messing up the initializers in Rails apps and required hacks for rake tasks. Replace it with a service to validate the models.
199
+
200
+ ## [0.2.5]
201
+
202
+ - Handle auto-inclusion of `Bookingsync::Prometheus::Producer::Outbox::ActiveRecordPublishable`
203
+
204
+ ## [0.2.4]
205
+
206
+ - Fix unlocking behavior (use `lock` with a block to automatically unlock upon completion)
207
+
208
+ ## [0.2.3]
209
+
210
+ - Fix outbox records creation for models that are just a dependency for parents
211
+ - Improve exposed shared behavior to make it possible to pass extra attributes
212
+
213
+ ## [0.2.2]
214
+
215
+ - Fix shared behavior
216
+
217
+ ## [0.2.1]
218
+
219
+ - Fix constants redefinition errors
220
+
221
+ ## [0.2.0]
222
+
223
+ - Add Transactional Outbox and standardize usage of the gem
224
+
225
+ ## [0.1.0]
226
+
227
+ - Initial release
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ source "https://rubygems.org"
4
+
5
+ # Specify your gem's dependencies in dionysus-rb.gemspec
6
+ gemspec
7
+
8
+ gem "rake", "~> 13.0"
9
+
10
+ gem "rspec", "~> 3.0"