sbmt-kafka_producer 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rspec +3 -0
- data/.rubocop.yml +34 -0
- data/Appraisals +24 -0
- data/CHANGELOG.md +166 -0
- data/Gemfile +5 -0
- data/LICENSE +21 -0
- data/README.md +167 -0
- data/Rakefile +12 -0
- data/dip.yml +67 -0
- data/docker-compose.yml +15 -0
- data/lefthook-local.dip_example.yml +4 -0
- data/lefthook.yml +6 -0
- data/lib/generators/kafka_producer/concerns/configuration.rb +30 -0
- data/lib/generators/kafka_producer/install/USAGE +8 -0
- data/lib/generators/kafka_producer/install/install_generator.rb +18 -0
- data/lib/generators/kafka_producer/install/templates/kafka_producer.yml +36 -0
- data/lib/generators/kafka_producer/outbox_producer/USAGE +9 -0
- data/lib/generators/kafka_producer/outbox_producer/outbox_producer_generator.rb +24 -0
- data/lib/generators/kafka_producer/producer/USAGE +10 -0
- data/lib/generators/kafka_producer/producer/producer_generator.rb +18 -0
- data/lib/generators/kafka_producer/producer/templates/producer.rb.erb +11 -0
- data/lib/sbmt/kafka_producer/base_producer.rb +103 -0
- data/lib/sbmt/kafka_producer/config/auth.rb +62 -0
- data/lib/sbmt/kafka_producer/config/kafka.rb +37 -0
- data/lib/sbmt/kafka_producer/config/producer.rb +51 -0
- data/lib/sbmt/kafka_producer/error_tracker.rb +31 -0
- data/lib/sbmt/kafka_producer/instrumentation/open_telemetry_loader.rb +23 -0
- data/lib/sbmt/kafka_producer/instrumentation/open_telemetry_tracer.rb +58 -0
- data/lib/sbmt/kafka_producer/instrumentation/tracing_middleware.rb +15 -0
- data/lib/sbmt/kafka_producer/instrumentation/yabeda_metrics_listener.rb +88 -0
- data/lib/sbmt/kafka_producer/kafka_client_factory.rb +61 -0
- data/lib/sbmt/kafka_producer/logger.rb +25 -0
- data/lib/sbmt/kafka_producer/outbox_producer.rb +11 -0
- data/lib/sbmt/kafka_producer/outbox_transport_factory.rb +13 -0
- data/lib/sbmt/kafka_producer/railtie.rb +16 -0
- data/lib/sbmt/kafka_producer/testing/configure_producer_client.rb +13 -0
- data/lib/sbmt/kafka_producer/testing.rb +5 -0
- data/lib/sbmt/kafka_producer/types.rb +12 -0
- data/lib/sbmt/kafka_producer/version.rb +7 -0
- data/lib/sbmt/kafka_producer/yabeda_configurer.rb +62 -0
- data/lib/sbmt/kafka_producer.rb +42 -0
- data/rubocop/rspec.yml +29 -0
- data/sbmt-kafka_producer.gemspec +59 -0
- metadata +427 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 3b6c10d4e49a462afad448f607d1d9c202b59a7b70661ef17cf7ec8dd4d8112e
|
4
|
+
data.tar.gz: 4cf15927ca33194dc5639afcf7967b3965ac817ad8813380e33f7dc003aa11d0
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 467ccc822998a5a9bb174557b6a72be74855060cb5e24e3097efb9ac9be746adfede6c4c77721e637ae6b14db4dc8983c7b76a23c6127016345f3f7fdcf1f681
|
7
|
+
data.tar.gz: bd1df75213ff393acf379fe4d3bbf013747af8f193a0970effbff0782e6c7dd1adeb8cd8cec4704cebb6fc214a1631dc330a679d77ca83c786067786adda329f
|
data/.rspec
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
inherit_mode:
|
2
|
+
merge:
|
3
|
+
- Exclude
|
4
|
+
|
5
|
+
require:
|
6
|
+
- rubocop-performance
|
7
|
+
- rubocop-rails
|
8
|
+
- rubocop-rspec
|
9
|
+
- standard
|
10
|
+
|
11
|
+
inherit_gem:
|
12
|
+
standard: config/base.yml
|
13
|
+
|
14
|
+
inherit_from:
|
15
|
+
- rubocop/rspec.yml
|
16
|
+
|
17
|
+
AllCops:
|
18
|
+
NewCops: enable
|
19
|
+
SuggestExtensions: false
|
20
|
+
TargetRubyVersion: 2.7
|
21
|
+
TargetRailsVersion: 5.2
|
22
|
+
|
23
|
+
RSpec/FilePath:
|
24
|
+
Enabled: false
|
25
|
+
|
26
|
+
RSpec/VerifiedDoubles:
|
27
|
+
Exclude:
|
28
|
+
- spec/**/*_spec.rb
|
29
|
+
|
30
|
+
Style/SingleLineMethods:
|
31
|
+
Enabled: false
|
32
|
+
|
33
|
+
Style/EmptyMethod:
|
34
|
+
Enabled: false
|
data/Appraisals
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# See compatibility table at https://www.fastruby.io/blog/ruby/rails/versions/compatibility-table.html
|
4
|
+
|
5
|
+
versions_map = {
|
6
|
+
"5.1" => %w[2.7],
|
7
|
+
"5.2" => %w[2.7],
|
8
|
+
"6.0" => %w[2.7],
|
9
|
+
"6.1" => %w[2.7 3.0],
|
10
|
+
"7.0" => %w[3.1],
|
11
|
+
"7.1" => %w[3.2]
|
12
|
+
}
|
13
|
+
|
14
|
+
current_ruby_version = RUBY_VERSION.split(".").first(2).join(".")
|
15
|
+
|
16
|
+
versions_map.each do |rails_version, ruby_versions|
|
17
|
+
ruby_versions.each do |ruby_version|
|
18
|
+
next if ruby_version != current_ruby_version
|
19
|
+
|
20
|
+
appraise "rails-#{rails_version}" do
|
21
|
+
gem "rails", "~> #{rails_version}.0"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,166 @@
|
|
1
|
+
# Change Log
|
2
|
+
|
3
|
+
All notable changes to this project will be documented in this file.
|
4
|
+
|
5
|
+
The format is based on [Keep a Changelog](http://keepachangelog.com/)
|
6
|
+
and this project adheres to [Semantic Versioning](http://semver.org/).
|
7
|
+
|
8
|
+
## [Unreleased] - yyyy-mm-dd
|
9
|
+
|
10
|
+
### Added
|
11
|
+
|
12
|
+
### Changed
|
13
|
+
|
14
|
+
### Fixed
|
15
|
+
|
16
|
+
## [2.0.0] - 2024-01-29
|
17
|
+
|
18
|
+
### Changed
|
19
|
+
|
20
|
+
- Remove `sbmt-dev`
|
21
|
+
|
22
|
+
## [1.0.0] - 2024-01-12
|
23
|
+
|
24
|
+
### Added
|
25
|
+
|
26
|
+
- Use mainstream karafka instead of custom fork
|
27
|
+
|
28
|
+
## [0.8.0] - 2023-10-05
|
29
|
+
|
30
|
+
### Added
|
31
|
+
|
32
|
+
- Errors' `cause` handling
|
33
|
+
|
34
|
+
### Fixed
|
35
|
+
|
36
|
+
- change from `double` to `instance_double`
|
37
|
+
|
38
|
+
## [0.7.0] - 2023-09-14
|
39
|
+
|
40
|
+
### Added
|
41
|
+
|
42
|
+
- Plug OpenTelemetry
|
43
|
+
|
44
|
+
## [0.6.3] - 2023-08-10
|
45
|
+
|
46
|
+
### Fixed
|
47
|
+
|
48
|
+
- Return True when publishing with bang methods
|
49
|
+
|
50
|
+
## [0.6.2] - 2023-08-08
|
51
|
+
|
52
|
+
### Added
|
53
|
+
|
54
|
+
- add ErrorTracker for Sentry
|
55
|
+
|
56
|
+
## [0.6.1] - 2023-08-07
|
57
|
+
|
58
|
+
### Fixed
|
59
|
+
|
60
|
+
- Don't catch an exception when publishing through the Sbmt::KafkaProducer::OutboxProducer
|
61
|
+
|
62
|
+
## [0.6.0] - 2023-07-23
|
63
|
+
|
64
|
+
### Added
|
65
|
+
|
66
|
+
- rails generator for initial configuration
|
67
|
+
- rails generator for producer/outbox_producer creation
|
68
|
+
|
69
|
+
## [0.5.1] - 2023-07-21
|
70
|
+
|
71
|
+
### Fixed
|
72
|
+
|
73
|
+
- change sentry method from capture_message to capture_exception
|
74
|
+
|
75
|
+
## [0.5.0] - 2023-06-26
|
76
|
+
|
77
|
+
### Fixed
|
78
|
+
- Mock BaseProducer for rspec
|
79
|
+
|
80
|
+
## [Unreleased] - 2023-06-21
|
81
|
+
|
82
|
+
### Changed
|
83
|
+
- update README
|
84
|
+
|
85
|
+
## [0.4.2] - 2023-06-20
|
86
|
+
|
87
|
+
### Fixed
|
88
|
+
- fixed version **sbmt-waterdrop**
|
89
|
+
|
90
|
+
## [0.4.1] - 2023-06-19
|
91
|
+
|
92
|
+
### Fixed
|
93
|
+
- fixed error handling in the method **on_error_occurred**
|
94
|
+
|
95
|
+
## [0.4.0] - 2023-06-13
|
96
|
+
|
97
|
+
### Changed
|
98
|
+
- config changed from anyway to Dry::Struct
|
99
|
+
|
100
|
+
## [0.3.0] - 2023-06-01
|
101
|
+
|
102
|
+
### Added
|
103
|
+
- implement producer metrics
|
104
|
+
|
105
|
+
## [0.2.3] - 2023-05-19
|
106
|
+
|
107
|
+
### Added
|
108
|
+
- for outbox, if the default settings for the kafka section are overridden, they are overwritten
|
109
|
+
|
110
|
+
### Changed
|
111
|
+
|
112
|
+
### Fixed
|
113
|
+
|
114
|
+
## [0.2.2] - 2023-05-18
|
115
|
+
|
116
|
+
### Added
|
117
|
+
- arbitrary parameters from kafka
|
118
|
+
|
119
|
+
### Changed
|
120
|
+
|
121
|
+
### Fixed
|
122
|
+
|
123
|
+
## [0.2.1] - 2023-05-16
|
124
|
+
|
125
|
+
### Added
|
126
|
+
- fix logger
|
127
|
+
|
128
|
+
### Changed
|
129
|
+
|
130
|
+
### Fixed
|
131
|
+
|
132
|
+
## [0.2.0] - 2023-05-16
|
133
|
+
|
134
|
+
### Added
|
135
|
+
- basic options for producer
|
136
|
+
|
137
|
+
### Changed
|
138
|
+
|
139
|
+
### Fixed
|
140
|
+
|
141
|
+
## [Unreleased] - 2023-05-04
|
142
|
+
|
143
|
+
### Added
|
144
|
+
- basic config for producer via gem anyway_config
|
145
|
+
|
146
|
+
### Changed
|
147
|
+
|
148
|
+
### Fixed
|
149
|
+
|
150
|
+
## [Unreleased] - 2023-05-02
|
151
|
+
|
152
|
+
### Added
|
153
|
+
- BaseProducer
|
154
|
+
- OutboxProducer
|
155
|
+
- Sentry, logger
|
156
|
+
|
157
|
+
### Changed
|
158
|
+
|
159
|
+
### Fixed
|
160
|
+
|
161
|
+
|
162
|
+
## [Unreleased]
|
163
|
+
|
164
|
+
## [0.1.0] - 2023-04-17
|
165
|
+
|
166
|
+
- Initial release
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
MIT License
|
2
|
+
|
3
|
+
Copyright (c) 2024 SberMarket Tech
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
13
|
+
copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,167 @@
|
|
1
|
+
[](https://badge.fury.io/rb/sbmt-kafka_producer)
|
2
|
+
[](https://github.com/SberMarket-Tech/sbmt-kafka_producer/actions?query=branch%3Amaster)
|
3
|
+
|
4
|
+
# Sbmt-KafkaProducer
|
5
|
+
|
6
|
+
This gem is used to produce Kafka messages. It is a wrapper over the [waterdrop](https://github.com/karafka/waterdrop) gem, and it is recommended for use as a transport with the [sbmt-outbox](https://github.com/SberMarket-Tech/sbmt-outbox) gem.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your app's Gemfile:
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
gem "sbmt-kafka_producer"
|
14
|
+
```
|
15
|
+
|
16
|
+
And then execute:
|
17
|
+
|
18
|
+
```bash
|
19
|
+
bundle install
|
20
|
+
```
|
21
|
+
|
22
|
+
## Auto configuration
|
23
|
+
|
24
|
+
We recommend going through the configuration and file creation process using the following Rails generators. Each generator can be run by using the `--help` option to learn more about the available arguments.
|
25
|
+
|
26
|
+
### Initial configuration
|
27
|
+
|
28
|
+
If you plug the gem into your application for the first time, you can generate the initial configuration:
|
29
|
+
|
30
|
+
```shell
|
31
|
+
rails g kafka_producer:install
|
32
|
+
```
|
33
|
+
|
34
|
+
As a result, the `config/kafka_producer.yml` file will be created.
|
35
|
+
|
36
|
+
### Producer class
|
37
|
+
|
38
|
+
A producer class can be generated with the following command:
|
39
|
+
|
40
|
+
```shell
|
41
|
+
rails g kafka_producer:producer MaybeNamespaced::Name sync topic
|
42
|
+
```
|
43
|
+
|
44
|
+
As the result, a sync producer will be created.
|
45
|
+
|
46
|
+
### Outbox producer
|
47
|
+
|
48
|
+
To generate an Outbox producer for use with Gem [sbmt-Outbox](https://github.com/SberMarket-Tech/sbmt-outbox), run the following command:
|
49
|
+
|
50
|
+
```shell
|
51
|
+
rails g kafka_producer:outbox_producer SomeOutboxItem
|
52
|
+
```
|
53
|
+
|
54
|
+
## Manual configuration
|
55
|
+
|
56
|
+
The `config/kafka_producer.yml` file is the main configuration for this gem.
|
57
|
+
|
58
|
+
```yaml
|
59
|
+
default: &default
|
60
|
+
deliver: true
|
61
|
+
ignore_kafka_error: true
|
62
|
+
# see more options at https://github.com/karafka/waterdrop/blob/master/lib/waterdrop/config.rb
|
63
|
+
wait_on_queue_full: true
|
64
|
+
max_payload_size: 1000012
|
65
|
+
max_wait_timeout: 5
|
66
|
+
wait_timeout: 0.005
|
67
|
+
auth:
|
68
|
+
kind: plaintext
|
69
|
+
kafka:
|
70
|
+
servers: "kafka:9092" # required
|
71
|
+
max_retries: 2 # optional, default: 2
|
72
|
+
required_acks: -1 # optional, default: -1
|
73
|
+
ack_timeout: 1 # in seconds, optional, default: 1
|
74
|
+
retry_backoff: 1 # in seconds, optional, default: 1
|
75
|
+
connect_timeout: 1 # in seconds, optional, default: 1
|
76
|
+
kafka_config: # low-level custom Kafka options
|
77
|
+
queue.buffering.max.messages: 1
|
78
|
+
queue.buffering.max.ms: 10_000
|
79
|
+
|
80
|
+
development:
|
81
|
+
<<: *default
|
82
|
+
|
83
|
+
test:
|
84
|
+
<<: *default
|
85
|
+
deliver: false
|
86
|
+
wait_on_queue_full: false
|
87
|
+
|
88
|
+
production:
|
89
|
+
<<: *default
|
90
|
+
```
|
91
|
+
|
92
|
+
### `auth` config section
|
93
|
+
|
94
|
+
The gem supports 2 variants: plaintext (default) and SASL-plaintext
|
95
|
+
|
96
|
+
SASL-plaintext:
|
97
|
+
|
98
|
+
```yaml
|
99
|
+
auth:
|
100
|
+
kind: sasl_plaintext
|
101
|
+
sasl_username: user
|
102
|
+
sasl_password: pwd
|
103
|
+
sasl_mechanism: SCRAM-SHA-512
|
104
|
+
```
|
105
|
+
|
106
|
+
### `kafka` config section
|
107
|
+
|
108
|
+
The `servers` key is required and should be in rdkafka format: without `kafka://` prefix, for example: `srv1:port1,srv2:port2,...`.
|
109
|
+
|
110
|
+
The `kafka_config` section may contain any [rdkafka option](https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md)
|
111
|
+
|
112
|
+
### Producer class
|
113
|
+
|
114
|
+
To create a producer that will be responsible for sending messages to Kafka, copy the following code:
|
115
|
+
|
116
|
+
```ruby
|
117
|
+
# app/producers/some_producer.rb
|
118
|
+
class SomeProducer < Sbmt::KafkaProducer::BaseProducer
|
119
|
+
option :topic, default: -> { "topic" }
|
120
|
+
|
121
|
+
def publish(payload, **options)
|
122
|
+
sync_publish(payload, options)
|
123
|
+
# async_publish(payload, options)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
```
|
127
|
+
|
128
|
+
### Outbox producer config
|
129
|
+
|
130
|
+
Add the following lines to your `config/outbox.yml` file in the `transports` section:
|
131
|
+
|
132
|
+
```yaml
|
133
|
+
outbox_items:
|
134
|
+
some_outbox_item:
|
135
|
+
transports:
|
136
|
+
sbmt/kafka_producer:
|
137
|
+
topic: 'topic'
|
138
|
+
kafka: # optional kafka options
|
139
|
+
required_acks: -1
|
140
|
+
```
|
141
|
+
|
142
|
+
## Usage
|
143
|
+
|
144
|
+
To send a message to a Kafka topic, execute the following command:
|
145
|
+
|
146
|
+
```ruby
|
147
|
+
SomeProducer.new.publish(payload, key: "123", headers: {"some-header" => "some-value"})
|
148
|
+
```
|
149
|
+
|
150
|
+
## Metrics
|
151
|
+
|
152
|
+
The gem collects base producing metrics using [Yabeda](https://github.com/yabeda-rb/yabeda). See metrics at [YabedaConfigurer](./lib/sbmt/kafka_producer/yabeda_configurer.rb).
|
153
|
+
|
154
|
+
## Testing
|
155
|
+
|
156
|
+
To stub a producer request to real Kafka broker, you can use a mock. To do this, please add `require "sbmt/kafka_producer/testing"` to the `spec/rails_helper.rb`.
|
157
|
+
|
158
|
+
## Development
|
159
|
+
|
160
|
+
Install [dip](https://github.com/bibendi/dip).
|
161
|
+
|
162
|
+
And run:
|
163
|
+
|
164
|
+
```shell
|
165
|
+
dip provision
|
166
|
+
dip rspec
|
167
|
+
```
|
data/Rakefile
ADDED
data/dip.yml
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
version: '7'
|
2
|
+
|
3
|
+
environment:
|
4
|
+
RUBY_VERSION: '3.2'
|
5
|
+
|
6
|
+
compose:
|
7
|
+
files:
|
8
|
+
- docker-compose.yml
|
9
|
+
|
10
|
+
interaction:
|
11
|
+
bash:
|
12
|
+
description: Open the Bash shell in app's container
|
13
|
+
service: ruby
|
14
|
+
command: /bin/bash
|
15
|
+
|
16
|
+
bundle:
|
17
|
+
description: Run Bundler commands
|
18
|
+
service: ruby
|
19
|
+
command: bundle
|
20
|
+
|
21
|
+
rails:
|
22
|
+
description: Run RoR commands
|
23
|
+
service: ruby
|
24
|
+
command: bundle exec rails
|
25
|
+
|
26
|
+
appraisal:
|
27
|
+
description: Run Appraisal commands
|
28
|
+
service: ruby
|
29
|
+
command: bundle exec appraisal
|
30
|
+
|
31
|
+
rspec:
|
32
|
+
description: Run Rspec commands
|
33
|
+
service: ruby
|
34
|
+
command: bundle exec rspec
|
35
|
+
subcommands:
|
36
|
+
all:
|
37
|
+
command: bundle exec appraisal rspec
|
38
|
+
rails-6.0:
|
39
|
+
command: bundle exec appraisal rails-6.0 rspec
|
40
|
+
rails-6.1:
|
41
|
+
command: bundle exec appraisal rails-6.1 rspec
|
42
|
+
rails-7.0:
|
43
|
+
command: bundle exec appraisal rails-7.0 rspec
|
44
|
+
rails-7.1:
|
45
|
+
command: bundle exec appraisal rails-7.1 rspec
|
46
|
+
|
47
|
+
rubocop:
|
48
|
+
description: Run Ruby linter
|
49
|
+
service: ruby
|
50
|
+
command: bundle exec rubocop
|
51
|
+
|
52
|
+
setup:
|
53
|
+
description: Install deps
|
54
|
+
service: ruby
|
55
|
+
command: bin/setup
|
56
|
+
|
57
|
+
test:
|
58
|
+
description: Run linters, run all tests
|
59
|
+
service: ruby
|
60
|
+
command: bin/test
|
61
|
+
|
62
|
+
provision:
|
63
|
+
- dip compose down --volumes
|
64
|
+
- cp -f lefthook-local.dip_example.yml lefthook-local.yml
|
65
|
+
- rm -f Gemfile.lock
|
66
|
+
- rm -f gemfiles/*gemfile*
|
67
|
+
- dip setup
|
data/docker-compose.yml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
services:
|
2
|
+
ruby:
|
3
|
+
image: ruby:${RUBY_VERSION:-3.2}
|
4
|
+
environment:
|
5
|
+
HISTFILE: /app/tmp/.bash_history
|
6
|
+
BUNDLE_PATH: /usr/local/bundle
|
7
|
+
BUNDLE_CONFIG: /app/.bundle/config
|
8
|
+
command: bash
|
9
|
+
working_dir: /app
|
10
|
+
volumes:
|
11
|
+
- .:/app:cached
|
12
|
+
- bundler_data:/usr/local/bundle
|
13
|
+
|
14
|
+
volumes:
|
15
|
+
bundler_data:
|
data/lefthook.yml
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module KafkaProducer
|
4
|
+
module Generators
|
5
|
+
module Concerns
|
6
|
+
module Configuration
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
CONFIG_PATH = "config/kafka_producer.yml"
|
10
|
+
|
11
|
+
def check_config_file!
|
12
|
+
config_path = File.expand_path(CONFIG_PATH)
|
13
|
+
return if File.exist?(config_path)
|
14
|
+
|
15
|
+
generator_name = "kafka_producer:install"
|
16
|
+
if yes?(
|
17
|
+
"The file #{config_path} does not appear to exist." \
|
18
|
+
" Would you like to generate it?"
|
19
|
+
)
|
20
|
+
generate generator_name
|
21
|
+
else
|
22
|
+
raise Rails::Generators::Error, "Please generate #{config_path} " \
|
23
|
+
"by running `bin/rails g #{generator_name}` " \
|
24
|
+
"or add this file manually."
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails/generators"
|
4
|
+
require "generators/kafka_producer/concerns/configuration"
|
5
|
+
|
6
|
+
module KafkaProducer
|
7
|
+
module Generators
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
9
|
+
include Concerns::Configuration
|
10
|
+
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
12
|
+
|
13
|
+
def create_kafka_producer_yml
|
14
|
+
copy_file "kafka_producer.yml", CONFIG_PATH
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
default: &default
|
2
|
+
deliver: true
|
3
|
+
wait_on_queue_full: true
|
4
|
+
max_payload_size: 1000012
|
5
|
+
max_wait_timeout: 5
|
6
|
+
wait_timeout: 0.005
|
7
|
+
ignore_kafka_error: true
|
8
|
+
|
9
|
+
auth:
|
10
|
+
sasl_username: <%= ENV.fetch('KAFKA_BROKERS'){ 'SCRAM-SHA-512:kafka_login:kafka_password' }.split(':').second %>
|
11
|
+
sasl_password: <%= ENV.fetch('KAFKA_BROKERS'){ 'SCRAM-SHA-512:kafka_login:kafka_password' }.split(':').last %>
|
12
|
+
sasl_mechanism: <%= ENV.fetch('KAFKA_BROKERS'){ 'SCRAM-SHA-512:kafka_login:kafka_password' }.split(':').first %>
|
13
|
+
kind: 'sasl_plaintext'
|
14
|
+
|
15
|
+
kafka:
|
16
|
+
servers: "kafka:9092"
|
17
|
+
max_retries: 2
|
18
|
+
required_acks: -1
|
19
|
+
ack_timeout: 1
|
20
|
+
retry_backoff: 1
|
21
|
+
connect_timeout: 1
|
22
|
+
|
23
|
+
development:
|
24
|
+
<<: *default
|
25
|
+
auth:
|
26
|
+
kind: plaintext
|
27
|
+
test:
|
28
|
+
<<: *default
|
29
|
+
deliver: false
|
30
|
+
wait_on_queue_full: false
|
31
|
+
auth:
|
32
|
+
kind: plaintext
|
33
|
+
staging: &staging
|
34
|
+
<<: *default
|
35
|
+
production:
|
36
|
+
<<: *staging
|
@@ -0,0 +1,9 @@
|
|
1
|
+
Description:
|
2
|
+
Modifies the config/outbox.yml file to work with the outbox gem
|
3
|
+
Pass in the outbox item, either CamelCased or under_scored
|
4
|
+
|
5
|
+
Example:
|
6
|
+
bin/rails generate kafka_producer:outbox_producer some_namespace/outbox_item
|
7
|
+
|
8
|
+
This will modify:
|
9
|
+
config/outbox.yml
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "rails/generators"
|
4
|
+
|
5
|
+
module KafkaProducer
|
6
|
+
module Generators
|
7
|
+
class OutboxProducerGenerator < Rails::Generators::NamedBase
|
8
|
+
source_root File.expand_path("templates", __dir__)
|
9
|
+
|
10
|
+
class_option :skip_item, type: :boolean, default: false, desc: "Skip creating InboxItem"
|
11
|
+
|
12
|
+
def insert_outbox_producer
|
13
|
+
generate "outbox:item", "#{item_name.underscore} --kind outbox" unless options[:skip_item]
|
14
|
+
generate "outbox:transport", "#{item_name.underscore} sbmt/kafka_producer --kind outbox"
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
|
19
|
+
def item_name
|
20
|
+
file_path
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
Description:
|
2
|
+
Stubs out a new non-outbox producer. Pass the producer name, either
|
3
|
+
CamelCased or under_scored
|
4
|
+
It takes the producer type, topic as arguments.
|
5
|
+
|
6
|
+
Example:
|
7
|
+
bin/rails generate kafka_producer:producer Test producer_type topic
|
8
|
+
|
9
|
+
This will create:
|
10
|
+
app/producers/test_producer.rb
|