dionysus-rb 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.circleci/config.yml +61 -0
- data/.github/workflows/ci.yml +77 -0
- data/.gitignore +12 -0
- data/.rspec +3 -0
- data/.rubocop.yml +175 -0
- data/.rubocop_todo.yml +53 -0
- data/CHANGELOG.md +227 -0
- data/Gemfile +10 -0
- data/Gemfile.lock +258 -0
- data/LICENSE.txt +21 -0
- data/README.md +1206 -0
- data/Rakefile +10 -0
- data/assets/logo.svg +51 -0
- data/bin/console +11 -0
- data/bin/karafka_health_check +14 -0
- data/bin/outbox_worker_health_check +12 -0
- data/bin/setup +8 -0
- data/dionysus-rb.gemspec +64 -0
- data/docker-compose.yml +44 -0
- data/lib/dionysus/checks/health_check.rb +50 -0
- data/lib/dionysus/checks.rb +7 -0
- data/lib/dionysus/consumer/batch_events_publisher.rb +33 -0
- data/lib/dionysus/consumer/config.rb +97 -0
- data/lib/dionysus/consumer/deserializer.rb +231 -0
- data/lib/dionysus/consumer/dionysus_event.rb +42 -0
- data/lib/dionysus/consumer/karafka_consumer_generator.rb +56 -0
- data/lib/dionysus/consumer/params_batch_processor.rb +65 -0
- data/lib/dionysus/consumer/params_batch_transformations/remove_duplicates_strategy.rb +54 -0
- data/lib/dionysus/consumer/params_batch_transformations.rb +4 -0
- data/lib/dionysus/consumer/persistor.rb +157 -0
- data/lib/dionysus/consumer/registry.rb +84 -0
- data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data.rb +27 -0
- data/lib/dionysus/consumer/synced_data/assign_columns_from_synced_data_job.rb +26 -0
- data/lib/dionysus/consumer/synced_data.rb +4 -0
- data/lib/dionysus/consumer/synchronizable_model.rb +93 -0
- data/lib/dionysus/consumer/workers_group.rb +18 -0
- data/lib/dionysus/consumer.rb +36 -0
- data/lib/dionysus/monitor.rb +48 -0
- data/lib/dionysus/producer/base_responder.rb +46 -0
- data/lib/dionysus/producer/config.rb +104 -0
- data/lib/dionysus/producer/deleted_record_serializer.rb +17 -0
- data/lib/dionysus/producer/genesis/performed.rb +11 -0
- data/lib/dionysus/producer/genesis/stream_job.rb +13 -0
- data/lib/dionysus/producer/genesis/streamer/base_job.rb +44 -0
- data/lib/dionysus/producer/genesis/streamer/standard_job.rb +43 -0
- data/lib/dionysus/producer/genesis/streamer.rb +40 -0
- data/lib/dionysus/producer/genesis.rb +62 -0
- data/lib/dionysus/producer/karafka_responder_generator.rb +133 -0
- data/lib/dionysus/producer/key.rb +14 -0
- data/lib/dionysus/producer/model_serializer.rb +105 -0
- data/lib/dionysus/producer/outbox/active_record_publishable.rb +74 -0
- data/lib/dionysus/producer/outbox/datadog_latency_reporter.rb +26 -0
- data/lib/dionysus/producer/outbox/datadog_latency_reporter_job.rb +11 -0
- data/lib/dionysus/producer/outbox/datadog_latency_reporter_scheduler.rb +47 -0
- data/lib/dionysus/producer/outbox/datadog_tracer.rb +32 -0
- data/lib/dionysus/producer/outbox/duplicates_filter.rb +26 -0
- data/lib/dionysus/producer/outbox/event_name.rb +26 -0
- data/lib/dionysus/producer/outbox/health_check.rb +48 -0
- data/lib/dionysus/producer/outbox/latency_tracker.rb +43 -0
- data/lib/dionysus/producer/outbox/model.rb +117 -0
- data/lib/dionysus/producer/outbox/producer.rb +26 -0
- data/lib/dionysus/producer/outbox/publishable.rb +106 -0
- data/lib/dionysus/producer/outbox/publisher.rb +131 -0
- data/lib/dionysus/producer/outbox/records_processor.rb +56 -0
- data/lib/dionysus/producer/outbox/runner.rb +120 -0
- data/lib/dionysus/producer/outbox/tombstone_publisher.rb +22 -0
- data/lib/dionysus/producer/outbox.rb +103 -0
- data/lib/dionysus/producer/partition_key.rb +42 -0
- data/lib/dionysus/producer/registry/validator.rb +32 -0
- data/lib/dionysus/producer/registry.rb +165 -0
- data/lib/dionysus/producer/serializer.rb +52 -0
- data/lib/dionysus/producer/suppressor.rb +18 -0
- data/lib/dionysus/producer.rb +121 -0
- data/lib/dionysus/railtie.rb +9 -0
- data/lib/dionysus/rb/version.rb +5 -0
- data/lib/dionysus/rb.rb +8 -0
- data/lib/dionysus/support/rspec/outbox_publishable.rb +78 -0
- data/lib/dionysus/topic_name.rb +15 -0
- data/lib/dionysus/utils/default_message_filter.rb +25 -0
- data/lib/dionysus/utils/exponential_backoff.rb +7 -0
- data/lib/dionysus/utils/karafka_datadog_listener.rb +20 -0
- data/lib/dionysus/utils/karafka_sentry_listener.rb +9 -0
- data/lib/dionysus/utils/null_error_handler.rb +6 -0
- data/lib/dionysus/utils/null_event_bus.rb +5 -0
- data/lib/dionysus/utils/null_hermes_event_producer.rb +5 -0
- data/lib/dionysus/utils/null_instrumenter.rb +7 -0
- data/lib/dionysus/utils/null_lock_client.rb +13 -0
- data/lib/dionysus/utils/null_model_factory.rb +5 -0
- data/lib/dionysus/utils/null_mutex_provider.rb +7 -0
- data/lib/dionysus/utils/null_retry_provider.rb +7 -0
- data/lib/dionysus/utils/null_tracer.rb +5 -0
- data/lib/dionysus/utils/null_transaction_provider.rb +15 -0
- data/lib/dionysus/utils/sidekiq_batched_job_distributor.rb +24 -0
- data/lib/dionysus/utils.rb +6 -0
- data/lib/dionysus/version.rb +7 -0
- data/lib/dionysus-rb.rb +3 -0
- data/lib/dionysus.rb +133 -0
- data/lib/tasks/dionysus.rake +18 -0
- data/log/development.log +0 -0
- data/sig/dionysus/rb.rbs +6 -0
- metadata +585 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 51b0fe501a9678e2fc887a1525a40a0ddc6e04d9a2d49e8211305370eef4e843
|
4
|
+
data.tar.gz: 82597c73fa710cbac12f091e6f8c26462aae3dfc7572de180cbc474c70cedf1a
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c7f05381d3fc7e3eba08e4666a46b71e1bc52425d56a3a0d46c5fde4287fdcc77bc64c4521d4f76307849e1cb017e07ab9d612dc127c72bc3fe5e23d919c994e
|
7
|
+
data.tar.gz: 13e9c0664deb5453e9510c0cba835222cb487ff3511696bdcfce4ec09f8677eb16fd4b25bafd4748275cc0533bd62ac3a524761d673d368666153ff79892cb40
|
@@ -0,0 +1,61 @@
|
|
1
|
+
version: 2
|
2
|
+
jobs:
|
3
|
+
build:
|
4
|
+
working_directory: ~/BookingSync/dionysus-rb
|
5
|
+
parallelism: 1
|
6
|
+
docker:
|
7
|
+
- image: cimg/ruby:2.7.6
|
8
|
+
environment:
|
9
|
+
POSTGRES_URL: "postgres://docker:docker@127.0.0.1:5432"
|
10
|
+
DATABASE_URL: "postgres://docker:docker@127.0.0.1:5432/circle_test"
|
11
|
+
DATABASE_NAME: "dionysus-rb"
|
12
|
+
- image: redis
|
13
|
+
- image: wurstmeister/zookeeper
|
14
|
+
- image: wurstmeister/kafka:2.11-0.11.0.3
|
15
|
+
environment:
|
16
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
17
|
+
KAFKA_ADVERTISED_PORT: 9092
|
18
|
+
KAFKA_PORT: 9092
|
19
|
+
KAFKA_DELETE_TOPIC_ENABLE: true
|
20
|
+
KAFKA_CREATE_TOPICS: "v102_rentals:1:1"
|
21
|
+
REDIS_URL: redis://localhost:6379/1
|
22
|
+
- image: circleci/postgres:12.6
|
23
|
+
environment:
|
24
|
+
POSTGRES_USER: docker
|
25
|
+
POSTGRES_PASSWORD: docker
|
26
|
+
steps:
|
27
|
+
- checkout
|
28
|
+
- run: mkdir /tmp/circleci-test-results
|
29
|
+
- restore_cache:
|
30
|
+
keys:
|
31
|
+
- v1-gemfile-lock-{{ checksum "Gemfile.lock" }}
|
32
|
+
- run:
|
33
|
+
name: Installing dependencies
|
34
|
+
command: bundle check --path=vendor/bundle || bundle install --path=vendor/bundle --jobs 4 --retry 3
|
35
|
+
- save_cache:
|
36
|
+
key: v1-gemfile-lock-{{ checksum "Gemfile.lock" }}
|
37
|
+
paths:
|
38
|
+
- vendor/bundle
|
39
|
+
- ~/.bundle
|
40
|
+
- run:
|
41
|
+
name: Installing dockerize
|
42
|
+
command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-alpine-linux-amd64-$DOCKERIZE_VERSION.tar.gz
|
43
|
+
environment:
|
44
|
+
DOCKERIZE_VERSION: v0.6.1
|
45
|
+
- run: sudo apt update && sudo apt install -y postgresql-client || true
|
46
|
+
- run:
|
47
|
+
name: Wait for db
|
48
|
+
command: dockerize -wait tcp://localhost:5432 -timeout 2m
|
49
|
+
- run:
|
50
|
+
name: Running Rubocop
|
51
|
+
command: bundle exec rubocop
|
52
|
+
- run:
|
53
|
+
name: Running specs
|
54
|
+
command: bundle exec rspec
|
55
|
+
environment:
|
56
|
+
RAILS_ENV: test
|
57
|
+
RACK_ENV: test
|
58
|
+
POSTGRES_USER: docker
|
59
|
+
POSTGRES_PASSWORD: docker
|
60
|
+
- store_test_results:
|
61
|
+
path: /tmp/circleci-test-results
|
@@ -0,0 +1,77 @@
|
|
1
|
+
name: CI
|
2
|
+
on: [pull_request]
|
3
|
+
jobs:
|
4
|
+
rubocop:
|
5
|
+
strategy:
|
6
|
+
fail-fast: true
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
steps:
|
9
|
+
- uses: actions/checkout@v2
|
10
|
+
- uses: ruby/setup-ruby@v1
|
11
|
+
with:
|
12
|
+
ruby-version: 2.7
|
13
|
+
bundler-cache: true
|
14
|
+
- run: bundle exec rubocop
|
15
|
+
rspec:
|
16
|
+
timeout-minutes: 10
|
17
|
+
strategy:
|
18
|
+
fail-fast: false
|
19
|
+
matrix:
|
20
|
+
ruby: ['2.7', '3.0.6', '3.1', '3.2']
|
21
|
+
runs-on: ubuntu-latest
|
22
|
+
env:
|
23
|
+
DD_PROFILING_NO_EXTENSION: true
|
24
|
+
RAILS_ENV: test
|
25
|
+
RACK_ENV: test
|
26
|
+
REDIS_URL: redis://localhost:6379/1
|
27
|
+
POSTGRES_USER: postgres
|
28
|
+
POSTGRES_PASSWORD: postgres
|
29
|
+
POSTGRES_DB: dionysus-rb-test
|
30
|
+
CLUSTER_ID: kafka-docker-cluster-1
|
31
|
+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
32
|
+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
33
|
+
KAFKA_PROCESS_ROLES: broker,controller
|
34
|
+
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
|
35
|
+
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
|
36
|
+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
|
37
|
+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://127.0.0.1:9092
|
38
|
+
KAFKA_BROKER_ID: 1
|
39
|
+
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@127.0.0.1:9093
|
40
|
+
ALLOW_PLAINTEXT_LISTENER: 'yes'
|
41
|
+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'true'
|
42
|
+
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
|
43
|
+
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
|
44
|
+
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
|
45
|
+
KAFKA_AUTHORIZER_CLASS_NAME: org.apache.kafka.metadata.authorizer.StandardAuthorizer
|
46
|
+
KAFKA_CREATE_TOPICS: "v102_rentals:1:1"
|
47
|
+
services:
|
48
|
+
postgres:
|
49
|
+
image: postgres:14
|
50
|
+
env:
|
51
|
+
POSTGRES_USER: postgres
|
52
|
+
POSTGRES_PASSWORD: postgres
|
53
|
+
POSTGRES_DB: dionysus-rb-test
|
54
|
+
ports:
|
55
|
+
- 5432:5432
|
56
|
+
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
57
|
+
redis:
|
58
|
+
image: redis
|
59
|
+
ports:
|
60
|
+
- 6379:6379
|
61
|
+
options: >-
|
62
|
+
--health-cmd "redis-cli ping"
|
63
|
+
--health-interval 10s
|
64
|
+
--health-timeout 5s
|
65
|
+
--health-retries 5
|
66
|
+
steps:
|
67
|
+
- uses: actions/checkout@v2
|
68
|
+
- name: Start Kafka with docker-compose
|
69
|
+
run: |
|
70
|
+
docker-compose up -d || (sleep 5 && docker-compose up -d)
|
71
|
+
|
72
|
+
- uses: ruby/setup-ruby@v1
|
73
|
+
|
74
|
+
with:
|
75
|
+
ruby-version: ${{ matrix.ruby }}
|
76
|
+
bundler-cache: true
|
77
|
+
- run: bundle exec rspec
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,175 @@
|
|
1
|
+
inherit_from: .rubocop_todo.yml
|
2
|
+
|
3
|
+
require:
|
4
|
+
- rubocop-performance
|
5
|
+
- rubocop-rspec
|
6
|
+
- rubocop-rake
|
7
|
+
|
8
|
+
inherit_mode:
|
9
|
+
merge:
|
10
|
+
- Exclude
|
11
|
+
|
12
|
+
AllCops:
|
13
|
+
NewCops: enable
|
14
|
+
TargetRubyVersion: 2.7
|
15
|
+
Exclude:
|
16
|
+
- "db/**/*"
|
17
|
+
- "bin/**/*"
|
18
|
+
- "tmp/**/*"
|
19
|
+
- "log/**/*"
|
20
|
+
- "vendor/**/*"
|
21
|
+
- "spec/spec_helper.rb"
|
22
|
+
|
23
|
+
Layout/ArgumentAlignment:
|
24
|
+
EnforcedStyle: with_fixed_indentation
|
25
|
+
|
26
|
+
Layout/ArrayAlignment:
|
27
|
+
EnforcedStyle: with_fixed_indentation
|
28
|
+
|
29
|
+
Layout/EmptyLineBetweenDefs:
|
30
|
+
AllowAdjacentOneLineDefs: true
|
31
|
+
|
32
|
+
Layout/EndAlignment:
|
33
|
+
EnforcedStyleAlignWith: variable
|
34
|
+
|
35
|
+
Layout/FirstArgumentIndentation:
|
36
|
+
EnforcedStyle: consistent
|
37
|
+
|
38
|
+
Layout/FirstArrayElementIndentation:
|
39
|
+
EnforcedStyle: consistent
|
40
|
+
|
41
|
+
Layout/FirstHashElementIndentation:
|
42
|
+
EnforcedStyle: consistent
|
43
|
+
|
44
|
+
Layout/MultilineMethodCallIndentation:
|
45
|
+
EnforcedStyle: indented
|
46
|
+
|
47
|
+
Layout/MultilineOperationIndentation:
|
48
|
+
EnforcedStyle: indented
|
49
|
+
|
50
|
+
Layout/ParameterAlignment:
|
51
|
+
EnforcedStyle: with_fixed_indentation
|
52
|
+
|
53
|
+
Layout/SpaceBeforeBrackets:
|
54
|
+
Enabled: false
|
55
|
+
|
56
|
+
Lint/UnusedMethodArgument:
|
57
|
+
AllowUnusedKeywordArguments: true
|
58
|
+
|
59
|
+
Metrics/ParameterLists:
|
60
|
+
MaxOptionalParameters: 4
|
61
|
+
Exclude:
|
62
|
+
- 'lib/dionysus.rb'
|
63
|
+
- 'lib/dionysus/producer/genesis/stream_job.rb'
|
64
|
+
- 'lib/dionysus/producer/genesis.rb'
|
65
|
+
|
66
|
+
RSpec/ExpectChange:
|
67
|
+
EnforcedStyle: block
|
68
|
+
|
69
|
+
RSpec/LetSetup:
|
70
|
+
Enabled: false
|
71
|
+
|
72
|
+
RSpec/MultipleExpectations:
|
73
|
+
Max: 25
|
74
|
+
Exclude:
|
75
|
+
- 'spec/dionysus/consumer/registry_spec.rb'
|
76
|
+
- 'spec/dionysus/producer_spec.rb'
|
77
|
+
- 'spec/dionysus/producer/registry_spec.rb'
|
78
|
+
|
79
|
+
Style/Alias:
|
80
|
+
EnforcedStyle: prefer_alias_method
|
81
|
+
|
82
|
+
Style/ClassAndModuleChildren:
|
83
|
+
EnforcedStyle: compact
|
84
|
+
Exclude:
|
85
|
+
- 'lib/dionysus/version.rb'
|
86
|
+
- 'lib/dionysus/checks.rb'
|
87
|
+
- 'lib/dionysus.rb'
|
88
|
+
|
89
|
+
Style/CommandLiteral:
|
90
|
+
EnforcedStyle: percent_x
|
91
|
+
|
92
|
+
Style/RescueStandardError:
|
93
|
+
EnforcedStyle: implicit
|
94
|
+
|
95
|
+
Style/StringLiterals:
|
96
|
+
EnforcedStyle: double_quotes
|
97
|
+
ConsistentQuotesInMultiline: true
|
98
|
+
|
99
|
+
Style/StringLiteralsInInterpolation:
|
100
|
+
EnforcedStyle: double_quotes
|
101
|
+
|
102
|
+
Style/RaiseArgs:
|
103
|
+
EnforcedStyle: compact
|
104
|
+
|
105
|
+
Style/RegexpLiteral:
|
106
|
+
EnforcedStyle: percent_r
|
107
|
+
|
108
|
+
Style/Documentation:
|
109
|
+
Enabled: false
|
110
|
+
|
111
|
+
RSpec/NestedGroups:
|
112
|
+
Max: 10
|
113
|
+
|
114
|
+
RSpec/MultipleMemoizedHelpers:
|
115
|
+
Enabled: false
|
116
|
+
|
117
|
+
Lint/AmbiguousBlockAssociation:
|
118
|
+
Exclude:
|
119
|
+
- 'spec/**/*'
|
120
|
+
- 'lib/dionysus/support/rspec/outbox_publishable.rb'
|
121
|
+
|
122
|
+
RSpec/ExampleLength:
|
123
|
+
Max: 50
|
124
|
+
Exclude:
|
125
|
+
- 'spec/dionysus/consumer/registry_spec.rb'
|
126
|
+
- 'spec/dionysus/producer_spec.rb'
|
127
|
+
- 'spec/dionysus/producer/registry_spec.rb'
|
128
|
+
|
129
|
+
Naming/VariableNumber:
|
130
|
+
Enabled: false
|
131
|
+
|
132
|
+
Metrics/BlockLength:
|
133
|
+
Max: 30
|
134
|
+
Exclude:
|
135
|
+
- 'spec/**/*'
|
136
|
+
- 'lib/dionysus/support/rspec/outbox_publishable.rb'
|
137
|
+
- 'lib/dionysus/consumer/persistor.rb'
|
138
|
+
|
139
|
+
Naming/PredicateName:
|
140
|
+
Enabled: false
|
141
|
+
|
142
|
+
Metrics/AbcSize:
|
143
|
+
Enabled: false
|
144
|
+
|
145
|
+
Metrics/CyclomaticComplexity:
|
146
|
+
Max: 15
|
147
|
+
|
148
|
+
Metrics/MethodLength:
|
149
|
+
Enabled: false
|
150
|
+
|
151
|
+
Naming/AccessorMethodName:
|
152
|
+
Exclude:
|
153
|
+
- 'spec/dionysus/utils/karafka_datadog_listener_spec.rb'
|
154
|
+
|
155
|
+
RSpec/DescribeClass:
|
156
|
+
Exclude:
|
157
|
+
- 'spec/integration_spec.rb'
|
158
|
+
|
159
|
+
Style/MultilineBlockChain:
|
160
|
+
Exclude:
|
161
|
+
- 'spec/dionysus/producer/model_serializer_spec.rb'
|
162
|
+
- 'spec/dionysus/producer/serializer_spec.rb'
|
163
|
+
|
164
|
+
Performance/MethodObjectAsBlock:
|
165
|
+
Enabled: false
|
166
|
+
|
167
|
+
Metrics/ClassLength:
|
168
|
+
Max: 150
|
169
|
+
|
170
|
+
Layout/LineLength:
|
171
|
+
Max: 125
|
172
|
+
|
173
|
+
Naming/FileName:
|
174
|
+
Exclude:
|
175
|
+
- 'lib/dionysus-rb.rb'
|
data/.rubocop_todo.yml
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
RSpec/VerifiedDoubles:
|
2
|
+
Enabled: false
|
3
|
+
|
4
|
+
RSpec/AnyInstance:
|
5
|
+
Exclude:
|
6
|
+
- 'spec/dionysus/consumer/karafka_consumer_generator_spec.rb'
|
7
|
+
- 'spec/dionysus/producer/outbox/datadog_latency_reporter_job_spec.rb'
|
8
|
+
- 'spec/dionysus/producer/outbox/active_record_publishable_spec.rb'
|
9
|
+
|
10
|
+
Layout/LineLength:
|
11
|
+
Exclude:
|
12
|
+
- 'lib/dionysus.rb'
|
13
|
+
- 'lib/dionysus/consumer/persistor.rb'
|
14
|
+
- 'lib/dionysus/producer/genesis/streamer/standard_job.rb'
|
15
|
+
- 'spec/dionysus/consumer/karafka_consumer_generator_spec.rb'
|
16
|
+
- 'spec/dionysus/producer/outbox/publisher_spec.rb'
|
17
|
+
- 'spec/dionysus/producer/genesis_spec.rb'
|
18
|
+
|
19
|
+
Lint/NonLocalExitFromIterator:
|
20
|
+
Exclude:
|
21
|
+
- 'lib/dionysus/producer/karafka_responder_generator.rb'
|
22
|
+
- 'lib/dionysus/consumer/persistor.rb'
|
23
|
+
|
24
|
+
Metrics/PerceivedComplexity:
|
25
|
+
Exclude:
|
26
|
+
- 'lib/dionysus/producer/karafka_responder_generator.rb'
|
27
|
+
- 'lib/dionysus/consumer/karafka_consumer_generator.rb'
|
28
|
+
- 'lib/dionysus/consumer/persistor.rb'
|
29
|
+
|
30
|
+
RSpec/LeakyConstantDeclaration:
|
31
|
+
Exclude:
|
32
|
+
- 'spec/dionysus/producer_spec.rb'
|
33
|
+
|
34
|
+
Lint/ConstantDefinitionInBlock:
|
35
|
+
Exclude:
|
36
|
+
- 'spec/dionysus/producer_spec.rb'
|
37
|
+
|
38
|
+
Lint/EmptyClass:
|
39
|
+
Exclude:
|
40
|
+
- 'lib/dionysus/consumer/synced_data.rb'
|
41
|
+
- 'lib/dionysus/consumer/params_batch_transformations.rb'
|
42
|
+
|
43
|
+
RSpec/ScatteredSetup:
|
44
|
+
Exclude:
|
45
|
+
- 'spec/dionysus/producer/outbox/records_processor_spec.rb'
|
46
|
+
|
47
|
+
Metrics/CyclomaticComplexity:
|
48
|
+
Exclude:
|
49
|
+
- 'lib/dionysus/producer/karafka_responder_generator.rb'
|
50
|
+
|
51
|
+
Metrics/BlockLength:
|
52
|
+
Exclude:
|
53
|
+
- 'lib/dionysus/producer/karafka_responder_generator.rb'
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,227 @@
|
|
1
|
+
## [Unreleased]
|
2
|
+
|
3
|
+
## [0.1.0]
|
4
|
+
|
5
|
+
- Turn private gem `bookingsync-prometheus` into `dionysus-rb` and release it publicly
|
6
|
+
- Allow to customize `consumer_group_prefix`
|
7
|
+
- Support Ruby 3.2
|
8
|
+
|
9
|
+
|
10
|
+
Migration path from `bookingsync-prometheus`:
|
11
|
+
- Rename all references of `Bookingsync::Prometheus` to `Dionysus`
|
12
|
+
- **CRITICAL** - when calling `Dionysus.initialize_application!` (previously `Bookingsync::Prometheus.initialize_application!`) provide a new keyword argument: `consumer_group_prefix` with a name of `prometheus_consumer_group_for` to preserve the same consumer group
|
13
|
+
- For producers, `config.hermes_event_producer` no longer defaults to `Hermes::EventProducer`, if you want to preserve the behavior make sure to configure this attribute explicitly: `config.hermes_event_producer = Hermes::EventProducer`
|
14
|
+
- `Bookingsync::Prometheus::Producer::Genesis::Performed` event was renamed to `Dionysus::Producer::Genesis::Performed`
|
15
|
+
- In the instrumentation context, all references to `prometheus`/`bookingsync_prometheus` were replaced to `dionysus`, which might require some renaming in the apps or observability tools.
|
16
|
+
- In the Datadog context (e.g. for `Dionysus::Producer::Outbox::DatadogLatencyReporter`), all references to `bookingsync_prometheus` were replaced to `dionysus`, which might require some renaming in the apps or observability tools, especially if you have a monitor based on this metric.
|
17
|
+
- In the Event Bus context, all references to `prometheus` were replaced to `dionysus`, which might require some renaming in the apps.
|
18
|
+
|
19
|
+
|
20
|
+
## Changelog from bookingsync-prometheus gem (original private gem that was renamed to dionysus-rb) - keeping it for historical reasons as the commits are not preserved since it's a new repo
|
21
|
+
|
22
|
+
## [1.17.2]
|
23
|
+
- Get back to using `in_batches` as the combination with `lazy` does the job and looks better han `find_in_batches`
|
24
|
+
|
25
|
+
## [1.17.1]
|
26
|
+
- Address memory leak in `Bookingsync::Prometheus::Producer::Genesis::Streamer::BaseJob` caused by the fact that the way `in_batches` was used was returning all the records in the end which was not needed.
|
27
|
+
|
28
|
+
## [1.17.0]
|
29
|
+
- Change batch size for Genesis to 1000 as it's a wiser default than 100 (and it's the same as ActiveRecord's default for `in_batches`).
|
30
|
+
|
31
|
+
## [1.16.1]
|
32
|
+
- Do not create outbox records for Genesis Replica topics as that would mean the same record would be published twice to Genesis Replica (first time with the standard topic, the second time with the Genesis one)
|
33
|
+
|
34
|
+
## [1.16.0]
|
35
|
+
- Introduce `genesis_replica: true` option for topics which assumes existence of twin topic where the storage would be infinite.
|
36
|
+
- Due to internal changes in responders to handle `genesis_replica`, it no longer makes sense to have an instrumentation in the responder following the pattern `prometheus.respond.#{topic_name}"`, but rather `"prometheus.respond.#{responder_class_name}"`, which should have been the case from the beginning.
|
37
|
+
|
38
|
+
## [1.15.0]
|
39
|
+
- Allow lambda (or an object responding to `call` method) for `partition_key` that takes a single argument - the resource
|
40
|
+
|
41
|
+
## [1.14.0]
|
42
|
+
- Introduce message filter on consumer level to allow filtering out messages that are not desired for some reason (e.g. the are too big and cause OOMs).
|
43
|
+
|
44
|
+
## [1.13.0]
|
45
|
+
- Provide an option to chain methods when resolving `association` for Observers (via `association_name`).
|
46
|
+
|
47
|
+
## [1.12.0]
|
48
|
+
- Introduce `Bookingsync::Prometheus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy` and `remove_consecutive_duplicates_before_publishing` config option.
|
49
|
+
|
50
|
+
## [1.11.0]
|
51
|
+
- Adjust `Bookingsync::Prometheus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy` to sort messages by timestamps before dropping duplicates to not assume that the messages are ordered.
|
52
|
+
|
53
|
+
## [1.10.2]
|
54
|
+
- Make `bin/karafka_health_check` more lightweight by loading only the check instead of whole prometheus and rails
|
55
|
+
|
56
|
+
## [1.10.1]
|
57
|
+
- Make `resource` to-one relationship optional to not blow up the Rails validations
|
58
|
+
|
59
|
+
## [1.10.0]
|
60
|
+
|
61
|
+
- Introduce `Bookingsync::Prometheus::Consumer::ParamsBatchTransformations::RemoveDuplicatesStrategy` and use it by default
|
62
|
+
- Do not set @transactional_outbox_enabled when calling the reader in the scenario where the variable is not set, just return the default
|
63
|
+
|
64
|
+
## [1.9.2]
|
65
|
+
|
66
|
+
- Make `Bookingsync::Prometheus::Checks::HealthCheck` even less aggressive - increase `expiry_time_in_seconds`.
|
67
|
+
|
68
|
+
## [1.9.1]
|
69
|
+
|
70
|
+
- Make `Bookingsync::Prometheus::Checks::HealthCheck` less aggressive - increase `expiry_time_in_seconds`.
|
71
|
+
|
72
|
+
## [1.9.0]
|
73
|
+
|
74
|
+
- Provide a service to assign values to columns from `synced_data`
|
75
|
+
|
76
|
+
## [1.8.0]
|
77
|
+
|
78
|
+
- Do not allow to execute Genesis for a model that is only a dependency to prevent enqueuing of duplicates.
|
79
|
+
|
80
|
+
## [1.7.0]
|
81
|
+
|
82
|
+
- Rescue from `publish_outbox_records` (inline handling of outbox publishing) and let the worker retry later.
|
83
|
+
- Do not send excessive notifications via error handler from `Bookingsync::Prometheus::Producer::Outbox::Publisher`, use logger instead.
|
84
|
+
|
85
|
+
## [1.6.0]
|
86
|
+
|
87
|
+
- Handle the case of a massive number of observers to be published - instead of publishing them inline (beyond a threshold), execute Genesis for them
|
88
|
+
|
89
|
+
## [1.5.0]
|
90
|
+
|
91
|
+
- Do not blow up with an exception when outbox publishes _updated event for a deleted record, send a notification instead
|
92
|
+
|
93
|
+
## [1.4.1]
|
94
|
+
|
95
|
+
- Register a heartbeat for a healthcheck when the messages are consumed
|
96
|
+
|
97
|
+
## [1.4.0]
|
98
|
+
|
99
|
+
- Move to file-based healthchecks, instead of using Redis-based ones.
|
100
|
+
|
101
|
+
## [1.3.0]
|
102
|
+
|
103
|
+
- Make health checks for Karafka work with liveness probe as well if statistics emission is enabled (check Readme for more details)
|
104
|
+
|
105
|
+
## [1.2.0]
|
106
|
+
|
107
|
+
- report `"#{namespace}.bookingsync_prometheus.producer.outbox.latency.highest_since_creation_date` to Datadog
|
108
|
+
|
109
|
+
## [1.1.1]
|
110
|
+
|
111
|
+
- Do not publish records for _created events if the they don't exist
|
112
|
+
- Do not cancel the dependent records of the aggregate for _destroyed event, only do this via attributes assignment in such a case
|
113
|
+
|
114
|
+
## [1.1.0]
|
115
|
+
|
116
|
+
- Allow extra configuration of topics to take advantage of Karafka 2.0 features like Dead Letter Queue
|
117
|
+
- Fix specs for KarafkaSentryListener and KarafkaDatadogListener for unknown events, as the behavior of Karafka monitor changed
|
118
|
+
|
119
|
+
## [1.0.0]
|
120
|
+
|
121
|
+
- [BREAKING CHANGES] Require Karafka 2.0 which has a lot of consequences
|
122
|
+
- `Bookingsync::Prometheus.initialize_application!` no longer takes `batch_fetching` and `backend` arguments
|
123
|
+
- `sidekiq-backend` is no longer supported
|
124
|
+
- batch fetching/batch consuming is always enabled. `config.batch_consuming` option is no longer available.
|
125
|
+
- `seed_brokers` argument provided to `Bookingsync::Prometheus.initialize_application!` should no longer contain the protocol, i.e. use `["localhost:9092"]` instead of `["kafka://localhost:9092"]`
|
126
|
+
- `ruby-kafka` is no longer used under the hood, it's replaced by `librdkafka`. Which implies a lot of config options are no longer available, e.g `kafka.fetcher_max_queue_size`, `kafka.offset_commit_interval`, `kafka.offset_commit_threshold`.
|
127
|
+
- when overriding kafka-specific settings in a block provided to `Bookingsync::Prometheus.initialize_application!`, you no longer can use writers to modify the config, instead, you modify the `kafka` hash settings. For example, instead of `config.kafka.heartbeat_interval = 120` you should use `config.kafka[:"heartbeat.interval.ms"] = 120_000`. Notice that the keys are symbols and the unit might be different (seconds vs. milliseconds for example).
|
128
|
+
- custom partition assignment strategy is no longer supported
|
129
|
+
- metrics from `ruby-kafka` are no longer available
|
130
|
+
- Use `kafka://localhost:9092` instead of `kafka://127.0.0.1:9092` in development environment
|
131
|
+
- There is no more `ruby-kafka` so the Datadog integration it was providing will also not work. Karafka/WaterDrop 2.0 comes with its own implementation: https://karafka.io/docs/Monitoring-and-logging/#datadog-and-statsd-integration, https://github.com/karafka/waterdrop#datadog-and-statsd-integration
|
132
|
+
- health check needs to be outside the `Rails.application.config.to_prepare` block
|
133
|
+
- Make sure you use `ddtrace` gem at 1.0 or higher
|
134
|
+
- Follow the guide: https://karafka.io/docs/Upgrades-2.0/
|
135
|
+
- If the application is just a producer, use the following config:
|
136
|
+
|
137
|
+
``` rb
|
138
|
+
Karafka::App.setup do |config|
|
139
|
+
config.producer = ::WaterDrop::Producer.new do |producer_config|
|
140
|
+
producer_config.kafka = {
|
141
|
+
'bootstrap.servers': ENV.fetch("PROMETHEUS_SEED_BROKER").gsub("kafka://", "").tr(";", ","), # this needs to be comma-separates list of brokers
|
142
|
+
'request.required.acks': 1,
|
143
|
+
"client.id": :id_of_the_producer_goes_here.to_s
|
144
|
+
}
|
145
|
+
producer_config.id = :id_of_the_producer_goes_here.to_s
|
146
|
+
producer_config.deliver = true
|
147
|
+
end
|
148
|
+
end
|
149
|
+
```
|
150
|
+
|
151
|
+
- Migration for some popular ruby-kafka options:
|
152
|
+
- `config.kafka.max_bytes_per_partition` => `config.kafka[:"max.partition.fetch.bytes"]`
|
153
|
+
- `config.kafka.socket_timeout` => `config.kafka[:"socket.timeout.ms"]` (the unit changes to milliseconds, you might need to multiply it by 1000)
|
154
|
+
- `config.kafka.session_timeout` => `config.kafka[:"session.timeout.ms"]` (the unit changes to milliseconds, you might need to multiply it by 1000)
|
155
|
+
- `config.kafka.heartbeat_interval` => `config.kafka[:"heartbeat.interval.ms"]` (the unit changes to milliseconds, you might need to multiply it by 1000)
|
156
|
+
|
157
|
+
## [0.3.7]
|
158
|
+
|
159
|
+
- Fix inferring error inside `Bookingsync::Prometheus::Producer::Outbox::Model` if arity is different than -1 or 1
|
160
|
+
|
161
|
+
|
162
|
+
## [0.3.6]
|
163
|
+
|
164
|
+
- Ensure publishing outbox observers won't blow up if to-one relationship is nil
|
165
|
+
|
166
|
+
## [0.3.5]
|
167
|
+
|
168
|
+
- Remove `datadog_statsd` prefix in favour of `namespace`.
|
169
|
+
|
170
|
+
## [0.3.4]
|
171
|
+
|
172
|
+
- Add latency tracking via Datadog for publishing via outbox
|
173
|
+
|
174
|
+
## [0.3.3]
|
175
|
+
|
176
|
+
- Enforce order by resource's `created_at` when publishing from after_commit
|
177
|
+
|
178
|
+
## [0.3.2]
|
179
|
+
|
180
|
+
- Explicitly require `concurrent/array` to avoid `uninitialized constant Concurrent::Array`.
|
181
|
+
|
182
|
+
## [0.3.1]
|
183
|
+
|
184
|
+
- Fix serialization of deleted records
|
185
|
+
- Fix issue in the outbox publisher for the scenario where record is a dependency of the parent but no parents exist
|
186
|
+
|
187
|
+
## [0.3.0]
|
188
|
+
|
189
|
+
- Allow publishing outbox records right after the transaction is committed
|
190
|
+
- Require `partition_key` column as otherwise, it's not possible resolve the partition key for deleted records
|
191
|
+
|
192
|
+
## [0.2.7]
|
193
|
+
|
194
|
+
- Add column validation rake task
|
195
|
+
|
196
|
+
## [0.2.6]
|
197
|
+
|
198
|
+
- Remove attributes validation in `Bookingsync::Prometheus::Producer::Registry` as it was messing up the initializers in Rails apps and required hacks for rake tasks. Replace it with a service to validate the models.
|
199
|
+
|
200
|
+
## [0.2.5]
|
201
|
+
|
202
|
+
- Handle auto-inclusion of `Bookingsync::Prometheus::Producer::Outbox::ActiveRecordPublishable`
|
203
|
+
|
204
|
+
## [0.2.4]
|
205
|
+
|
206
|
+
- Fix unlocking behavior (use `lock` with a block to automatically unlock upon completion)
|
207
|
+
|
208
|
+
## [0.2.3]
|
209
|
+
|
210
|
+
- Fix outbox records creation for models that are just a dependency for parents
|
211
|
+
- Improve exposed shared behavior to make it possible to pass extra attributes
|
212
|
+
|
213
|
+
## [0.2.2]
|
214
|
+
|
215
|
+
- Fix shared behavior
|
216
|
+
|
217
|
+
## [0.2.1]
|
218
|
+
|
219
|
+
- Fix constants redefinition errors
|
220
|
+
|
221
|
+
## [0.2.0]
|
222
|
+
|
223
|
+
- Add Transactional Outbox and standardize usage of the gem
|
224
|
+
|
225
|
+
## [0.1.0]
|
226
|
+
|
227
|
+
- Initial release
|