transactional_outbox 1.0.0beta1 → 1.0.0beta2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/README.md +27 -2
- data/lib/generators/transactional_outbox/migration/templates/active_record.rb.erb +23 -4
- data/lib/generators/transactional_outbox/migration/templates/sequel.rb.erb +15 -5
- data/lib/transactional_outbox/adapters_container.rb +1 -1
- data/lib/transactional_outbox/constants.rb +3 -0
- data/lib/transactional_outbox/database/adapters/active_record.rb +24 -6
- data/lib/transactional_outbox/database/adapters/base.rb +14 -0
- data/lib/transactional_outbox/database/adapters/instance_methods.rb +21 -0
- data/lib/transactional_outbox/database/adapters/interface.rb +3 -12
- data/lib/transactional_outbox/database/adapters/null.rb +28 -12
- data/lib/transactional_outbox/database/adapters/sequel.rb +23 -4
- data/lib/transactional_outbox/database.rb +2 -1
- data/lib/transactional_outbox/event/builder.rb +4 -2
- data/lib/transactional_outbox/event/contextable.rb +0 -2
- data/lib/transactional_outbox/event/payloadable.rb +1 -1
- data/lib/transactional_outbox/event.rb +3 -2
- data/lib/transactional_outbox/producer/adapters/base.rb +14 -0
- data/lib/transactional_outbox/producer/adapters/instance_methods.rb +17 -0
- data/lib/transactional_outbox/producer/adapters/interface.rb +2 -9
- data/lib/transactional_outbox/producer/adapters/kafka.rb +4 -1
- data/lib/transactional_outbox/producer/adapters/null.rb +6 -4
- data/lib/transactional_outbox/producer.rb +2 -8
- data/lib/transactional_outbox/relay/event_processor.rb +25 -17
- data/lib/transactional_outbox/relay/graceful_shutdown.rb +9 -3
- data/lib/transactional_outbox/relay/runner.rb +4 -12
- data/lib/transactional_outbox/relay/worker_set/processor.rb +8 -8
- data/lib/transactional_outbox/relay/worker_set/worker.rb +11 -8
- data/lib/transactional_outbox/relay/worker_set.rb +8 -8
- data/lib/transactional_outbox/tasks/{relay.rake → event_relay.rake} +1 -1
- data/lib/transactional_outbox/version.rb +1 -1
- data/lib/transactional_outbox.rb +9 -10
- metadata +7 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 0fdc9de5705816c6e7a07d44d5ca43fa972d6d83935b861cc6ea42f477af6f24
|
|
4
|
+
data.tar.gz: c0b9b3743a73d142153b480a52260a4e030c6496d18c57adb0f0c6b2f43bee4b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2e588b796c4c5b22c9c0188a8d5ed857695c832d961df5c4ebfc3c4bbbe3237e61339a13b763412d4e52a3dd087e1f40749a0cba777518262486e95623cd952c
|
|
7
|
+
data.tar.gz: a2b31bcca2501c6f19ba593caa80badbfdc75301b4c97affd3e8e02bf2a6cac8cb835d105aad8d96825d8002d6c38ef6e618d84f68ff51e60b55377aad9a5ca8
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
|
@@ -8,13 +8,13 @@
|
|
|
8
8
|
|
|
9
9
|
## Installation
|
|
10
10
|
|
|
11
|
-
Add this line to your application's Gemfile:
|
|
11
|
+
1. Add this line to your application's Gemfile:
|
|
12
12
|
|
|
13
13
|
```ruby
|
|
14
14
|
gem 'transactional_outbox'
|
|
15
15
|
```
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
2. Execute:
|
|
18
18
|
|
|
19
19
|
$ bundle install
|
|
20
20
|
|
|
@@ -22,6 +22,31 @@ Or install it yourself as:
|
|
|
22
22
|
|
|
23
23
|
$ gem install transactional_outbox
|
|
24
24
|
|
|
25
|
+
## Using
|
|
26
|
+
|
|
27
|
+
1. Configure it (see [configuration](docs/configuration.md)).
|
|
28
|
+
|
|
29
|
+
2. Generate migrations (if you're using Rails) or use template from [here](./lib/generators/transactional_outbox/migration/templates/) and migrate it:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
$ rails g transactional_outbox:migration
|
|
33
|
+
$ rails db:migrate
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
3. Define your own events (see [events creation](./docs/events_creation.md)).
|
|
37
|
+
|
|
38
|
+
4. Run event relay (see [event relay](./docs/event_relay.md)) with rake task (if needed):
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
$ rake event_relay:run
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Or run it directly like [here](./examples/example-app/bin/relay):
|
|
45
|
+
|
|
46
|
+
```ruby
|
|
47
|
+
TransactionalOutbox::Relay::Runner.start
|
|
48
|
+
```
|
|
49
|
+
|
|
25
50
|
## Architecture
|
|
26
51
|
|
|
27
52
|

|
|
@@ -1,18 +1,37 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
class CreateOutboxEventsTable < ActiveRecord::Migration[7.0]
|
|
4
|
-
def
|
|
4
|
+
def up
|
|
5
|
+
execute <<-SQL
|
|
6
|
+
CREATE TYPE <%= TransactionalOutbox.config.outbox_table_name %>_status_type AS ENUM(
|
|
7
|
+
'<%= TransactionalOutbox::EVENT_NEW_STATUS %>',
|
|
8
|
+
'<%= TransactionalOutbox::EVENT_PROCESSING_STATUS %>'
|
|
9
|
+
);
|
|
10
|
+
SQL
|
|
11
|
+
|
|
5
12
|
create_table :<%= TransactionalOutbox.config.outbox_table_name %>, id: :uuid do |t|
|
|
6
13
|
t.string :aggregate_type, null: false
|
|
7
14
|
t.uuid :aggregate_id, null: false
|
|
8
15
|
t.string :event_type, null: false
|
|
9
|
-
t.string :
|
|
16
|
+
t.string :queue, null: false
|
|
17
|
+
t.column :status, :<%= TransactionalOutbox.config.outbox_table_name %>_status_type, default: "<%= TransactionalOutbox::EVENT_NEW_STATUS %>", null: false
|
|
18
|
+
t.jsonb :queue_extra_parameters
|
|
10
19
|
t.jsonb :headers
|
|
11
20
|
t.jsonb :payload
|
|
12
|
-
t.datetime :
|
|
21
|
+
t.datetime :processing_started_at
|
|
22
|
+
|
|
23
|
+
t.timestamps
|
|
13
24
|
end
|
|
14
25
|
|
|
15
|
-
add_index :<%= TransactionalOutbox.config.outbox_table_name %>, :
|
|
26
|
+
add_index :<%= TransactionalOutbox.config.outbox_table_name %>, :queue
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def down
|
|
30
|
+
drop_table :<%= TransactionalOutbox.config.outbox_table_name %>
|
|
31
|
+
|
|
32
|
+
execute <<-SQL
|
|
33
|
+
DROP TYPE <%= TransactionalOutbox.config.outbox_table_name %>_status_type;
|
|
34
|
+
SQL
|
|
16
35
|
end
|
|
17
36
|
end
|
|
18
37
|
|
|
@@ -3,24 +3,34 @@
|
|
|
3
3
|
Sequel.migration do
|
|
4
4
|
up do
|
|
5
5
|
run <<-SQL.squish
|
|
6
|
+
CREATE TYPE <%= TransactionalOutbox.config.outbox_table_name %>_status_type AS ENUM(
|
|
7
|
+
'<%= TransactionalOutbox::EVENT_NEW_STATUS %>',
|
|
8
|
+
'<%= TransactionalOutbox::EVENT_PROCESSING_STATUS %>'
|
|
9
|
+
);
|
|
10
|
+
|
|
6
11
|
CREATE TABLE IF NOT EXISTS <%= TransactionalOutbox.config.outbox_table_name %> (
|
|
7
12
|
id uuid NOT NULL DEFAULT gen_random_uuid() PRIMARY KEY,
|
|
8
|
-
aggregate_type
|
|
13
|
+
aggregate_type varchar(255) NOT NULL,
|
|
9
14
|
aggregate_id uuid NOT NULL,
|
|
10
|
-
event_type
|
|
11
|
-
|
|
15
|
+
event_type varchar(255) NOT NULL,
|
|
16
|
+
queue text NOT NULL,
|
|
17
|
+
queue_extra_parameters jsonb,
|
|
18
|
+
status <%= TransactionalOutbox.config.outbox_table_name %>_status_type NOT NULL DEFAULT '<%= TransactionalOutbox::EVENT_NEW_STATUS %>',
|
|
12
19
|
headers jsonb,
|
|
13
20
|
payload jsonb,
|
|
14
|
-
|
|
21
|
+
processing_started_at timestamp without time zone,
|
|
22
|
+
created_at timestamp without time zone DEFAULT now() NOT NULL,
|
|
23
|
+
updated_at timestamp without time zone DEFAULT now() NOT NULL
|
|
15
24
|
);
|
|
16
25
|
|
|
17
|
-
CREATE INDEX idx_<%= TransactionalOutbox.config.outbox_table_name %>
|
|
26
|
+
CREATE INDEX idx_<%= TransactionalOutbox.config.outbox_table_name %>_queue ON <%= TransactionalOutbox.config.outbox_table_name %>(queue);
|
|
18
27
|
SQL
|
|
19
28
|
end
|
|
20
29
|
|
|
21
30
|
down do
|
|
22
31
|
run <<-SQL.squish
|
|
23
32
|
DROP TABLE IF EXISTS <%= TransactionalOutbox.config.outbox_table_name %>;
|
|
33
|
+
DROP TYPE <%= TransactionalOutbox.config.outbox_table_name %>_status_type;
|
|
24
34
|
SQL
|
|
25
35
|
end
|
|
26
36
|
end
|
|
@@ -6,7 +6,7 @@ module TransactionalOutbox
|
|
|
6
6
|
def resolve(adapter)
|
|
7
7
|
return container.resolve(adapter) if exists?(adapter)
|
|
8
8
|
|
|
9
|
-
raise TransactionalOutbox::UnknownAdapterError, "Unknown
|
|
9
|
+
raise TransactionalOutbox::UnknownAdapterError, "Unknown adapter: #{adapter}"
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
def register(adapter, klass)
|
|
@@ -12,5 +12,8 @@ module TransactionalOutbox
|
|
|
12
12
|
WORKER_STOPPED_MONITOR_EVENT = "worker.stopped"
|
|
13
13
|
WORKER_EVENTS_PROCESSED_MONITOR_EVENT = "worker.events.processed"
|
|
14
14
|
WORKER_EXCEPTIONS_TOTAL_MONITOR_EVENT = "worker.exceptions_total"
|
|
15
|
+
|
|
16
|
+
EVENT_NEW_STATUS = "new"
|
|
17
|
+
EVENT_PROCESSING_STATUS = "processing"
|
|
15
18
|
end
|
|
16
19
|
end
|
|
@@ -3,22 +3,40 @@
|
|
|
3
3
|
module TransactionalOutbox
|
|
4
4
|
class Database
|
|
5
5
|
class Adapters
|
|
6
|
-
class ActiveRecord <
|
|
6
|
+
class ActiveRecord < Base
|
|
7
7
|
def transaction(*options, &) = model.transaction(*options, &)
|
|
8
8
|
|
|
9
|
-
def fetch_events(
|
|
10
|
-
model.where(
|
|
11
|
-
|
|
12
|
-
|
|
9
|
+
def fetch_events(queue, batch_size)
|
|
10
|
+
claimable_scope(model.where(queue:))
|
|
11
|
+
.order(:created_at)
|
|
12
|
+
.limit(batch_size)
|
|
13
|
+
.lock("FOR UPDATE SKIP LOCKED")
|
|
14
|
+
.map { |event| event.attributes.symbolize_keys }
|
|
13
15
|
end
|
|
14
16
|
|
|
15
17
|
def insert_events(attributes) = model.insert_all(attributes)
|
|
16
18
|
def delete_events(ids) = model.where(id: ids).delete_all
|
|
17
|
-
def
|
|
19
|
+
def fetch_queues = claimable_scope(model).select(:queue).distinct.map(&:queue)
|
|
20
|
+
|
|
21
|
+
def move_to_processing(ids)
|
|
22
|
+
model
|
|
23
|
+
.where(id: ids)
|
|
24
|
+
.update_all(status: TransactionalOutbox::EVENT_PROCESSING_STATUS, processing_started_at: current_time)
|
|
25
|
+
end
|
|
18
26
|
|
|
19
27
|
private
|
|
20
28
|
|
|
21
29
|
def model = @model ||= Object.const_get(config.db.connection_data[:model])
|
|
30
|
+
|
|
31
|
+
def claimable_scope(scope)
|
|
32
|
+
scope
|
|
33
|
+
.where(status: TransactionalOutbox::EVENT_NEW_STATUS)
|
|
34
|
+
.or(
|
|
35
|
+
scope
|
|
36
|
+
.where(status: TransactionalOutbox::EVENT_PROCESSING_STATUS)
|
|
37
|
+
.where(processing_started_at: ...outdated_events_timestamp)
|
|
38
|
+
)
|
|
39
|
+
end
|
|
22
40
|
end
|
|
23
41
|
end
|
|
24
42
|
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TransactionalOutbox
|
|
4
|
+
class Database
|
|
5
|
+
class Adapters
|
|
6
|
+
module InstanceMethods
|
|
7
|
+
def initialize
|
|
8
|
+
@outbox_table_name = TransactionalOutbox.config.outbox_table_name
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
attr_reader :outbox_table_name
|
|
14
|
+
|
|
15
|
+
def config = @config ||= TransactionalOutbox.config
|
|
16
|
+
def outdated_events_timestamp = Time.now.utc - config.relay.processing_events_claim_timeout_seconds
|
|
17
|
+
def current_time = Time.now.utc
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -4,21 +4,12 @@ module TransactionalOutbox
|
|
|
4
4
|
class Database
|
|
5
5
|
class Adapters
|
|
6
6
|
class Interface
|
|
7
|
-
def initialize
|
|
8
|
-
@outbox_table_name = TransactionalOutbox.config.outbox_table_name
|
|
9
|
-
end
|
|
10
|
-
|
|
11
7
|
def transaction(*_options, &) = raise NotImplementedError
|
|
12
8
|
def insert_events(_attributes) = raise NotImplementedError
|
|
13
|
-
def fetch_events(
|
|
9
|
+
def fetch_events(_queue_name, _batch_size) = raise NotImplementedError
|
|
14
10
|
def delete_events(_ids) = raise NotImplementedError
|
|
15
|
-
def
|
|
16
|
-
|
|
17
|
-
private
|
|
18
|
-
|
|
19
|
-
attr_reader :outbox_table_name
|
|
20
|
-
|
|
21
|
-
def config = @config ||= TransactionalOutbox.config
|
|
11
|
+
def fetch_queues = raise NotImplementedError
|
|
12
|
+
def move_to_processing(_ids) = raise NotImplementedError
|
|
22
13
|
end
|
|
23
14
|
end
|
|
24
15
|
end
|
|
@@ -3,34 +3,50 @@
|
|
|
3
3
|
module TransactionalOutbox
|
|
4
4
|
class Database
|
|
5
5
|
class Adapters
|
|
6
|
-
class Null <
|
|
6
|
+
class Null < Base
|
|
7
7
|
class << self
|
|
8
|
-
def dataset = @dataset ||=
|
|
9
|
-
def clear_store = @dataset =
|
|
8
|
+
def dataset = @dataset ||= {}
|
|
9
|
+
def clear_store = @dataset = {}
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
def transaction(*_options, &block) = block.call
|
|
13
13
|
|
|
14
14
|
def insert_events(rows)
|
|
15
|
-
rows
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
rows.each do |row|
|
|
16
|
+
dataset[row[:id]] = row.merge(
|
|
17
|
+
status: TransactionalOutbox::EVENT_NEW_STATUS, processing_started_at: current_time
|
|
18
|
+
)
|
|
19
|
+
end
|
|
18
20
|
end
|
|
19
21
|
|
|
20
|
-
def fetch_events(
|
|
21
|
-
|
|
22
|
+
def fetch_events(queue_name, batch_size)
|
|
23
|
+
dataset.values.select { |row| row[:queue] == queue_name }.then { claimable_scope(_1) }.first(batch_size)
|
|
24
|
+
end
|
|
22
25
|
|
|
23
|
-
def
|
|
24
|
-
ids_map = ids.to_h { |id| [id, true] }
|
|
26
|
+
def fetch_queues = dataset.values.then { claimable_scope(_1) }.map { |row| row[:queue] }.uniq
|
|
25
27
|
|
|
26
|
-
|
|
28
|
+
def move_to_processing(ids)
|
|
29
|
+
ids.each { |id| dataset[id]&.merge!(status: TransactionalOutbox::EVENT_PROCESSING_STATUS) }
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def delete_events(ids)
|
|
33
|
+
ids.each { |id| dataset.delete(id) }
|
|
27
34
|
|
|
28
35
|
true
|
|
29
36
|
end
|
|
30
37
|
|
|
31
38
|
private
|
|
32
39
|
|
|
33
|
-
def dataset = self.class.dataset ||=
|
|
40
|
+
def dataset = self.class.dataset ||= {}
|
|
41
|
+
|
|
42
|
+
def claimable_scope(scope)
|
|
43
|
+
scope.select do |row|
|
|
44
|
+
row[:status] == TransactionalOutbox::EVENT_NEW_STATUS || (
|
|
45
|
+
row[:status] == TransactionalOutbox::EVENT_PROCESSING_STATUS &&
|
|
46
|
+
row[:processing_started_at] < outdated_events_timestamp
|
|
47
|
+
)
|
|
48
|
+
end
|
|
49
|
+
end
|
|
34
50
|
end
|
|
35
51
|
end
|
|
36
52
|
end
|
|
@@ -3,14 +3,21 @@
|
|
|
3
3
|
module TransactionalOutbox
|
|
4
4
|
class Database
|
|
5
5
|
class Adapters
|
|
6
|
-
class Sequel <
|
|
6
|
+
class Sequel < Base
|
|
7
7
|
def transaction(*options, &) = db.transaction(*options, &)
|
|
8
8
|
def insert_events(attributes) = table_object.multi_insert(attributes)
|
|
9
|
-
def
|
|
9
|
+
def fetch_queues = claimable_scope(table_object).select(:queue).distinct.map { |event| event[:queue] }
|
|
10
|
+
|
|
10
11
|
def delete_events(ids) = table_object.where(id: ids).delete
|
|
11
12
|
|
|
12
|
-
def
|
|
13
|
-
|
|
13
|
+
def move_to_processing(ids)
|
|
14
|
+
table_object
|
|
15
|
+
.where(id: ids)
|
|
16
|
+
.update(status: TransactionalOutbox::EVENT_PROCESSING_STATUS, processing_started_at: current_time)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def fetch_events(queue, batch_size)
|
|
20
|
+
dataset = claimable_scope(table_object.where(queue:)).order(:created_at).limit(batch_size)
|
|
14
21
|
dataset = dataset.for_update.skip_locked unless config.test_environment
|
|
15
22
|
|
|
16
23
|
dataset.all
|
|
@@ -20,6 +27,18 @@ module TransactionalOutbox
|
|
|
20
27
|
|
|
21
28
|
def db = @db ||= config.db.connection_data[:db]
|
|
22
29
|
def table_object = @table_object ||= db.from(outbox_table_name)
|
|
30
|
+
|
|
31
|
+
def claimable_scope(scope)
|
|
32
|
+
scope.where(
|
|
33
|
+
::Sequel.|(
|
|
34
|
+
{ status: TransactionalOutbox::EVENT_NEW_STATUS },
|
|
35
|
+
::Sequel.&(
|
|
36
|
+
{ status: TransactionalOutbox::EVENT_PROCESSING_STATUS },
|
|
37
|
+
::Sequel.expr(:processing_started_at) < outdated_events_timestamp
|
|
38
|
+
)
|
|
39
|
+
)
|
|
40
|
+
)
|
|
41
|
+
end
|
|
23
42
|
end
|
|
24
43
|
end
|
|
25
44
|
end
|
|
@@ -4,7 +4,8 @@ module TransactionalOutbox
|
|
|
4
4
|
class Database
|
|
5
5
|
extend Forwardable
|
|
6
6
|
|
|
7
|
-
def_delegators :@adapter, :transaction, :insert_events, :fetch_events, :
|
|
7
|
+
def_delegators :@adapter, :transaction, :insert_events, :fetch_events, :fetch_queues, :delete_events,
|
|
8
|
+
:move_to_processing
|
|
8
9
|
|
|
9
10
|
attr_reader :adapter
|
|
10
11
|
|
|
@@ -3,11 +3,13 @@
|
|
|
3
3
|
module TransactionalOutbox
|
|
4
4
|
class Event
|
|
5
5
|
class Builder
|
|
6
|
-
def self.build(event_config, payload,
|
|
6
|
+
def self.build(event_config, payload, context)
|
|
7
7
|
{
|
|
8
8
|
id: SecureRandom.uuid,
|
|
9
|
-
|
|
9
|
+
queue: event_config.queue,
|
|
10
|
+
queue_extra_parameters: event_config.queue_extra_parameters,
|
|
10
11
|
aggregate_type: event_config.aggregate_type,
|
|
12
|
+
aggregate_id: context[event_config.aggregate_type.to_sym][:id],
|
|
11
13
|
event_type: event_config.event_type.to_s,
|
|
12
14
|
headers: {},
|
|
13
15
|
payload:
|
|
@@ -12,8 +12,9 @@ module TransactionalOutbox
|
|
|
12
12
|
setting :schema
|
|
13
13
|
setting :aggregate_type
|
|
14
14
|
setting :event_type
|
|
15
|
-
setting :topic
|
|
16
15
|
setting :event_builder
|
|
16
|
+
setting :queue
|
|
17
|
+
setting :queue_extra_parameters
|
|
17
18
|
|
|
18
19
|
config.values.keys.each do |attr|
|
|
19
20
|
define_singleton_method(attr) do |value|
|
|
@@ -45,7 +46,7 @@ module TransactionalOutbox
|
|
|
45
46
|
@event_builder ||= begin
|
|
46
47
|
klass = config.event_builder || TransactionalOutbox.config.default_event_builder
|
|
47
48
|
|
|
48
|
-
Object.const_get(klass)
|
|
49
|
+
klass.is_a?(String) ? Object.const_get(klass) : klass
|
|
49
50
|
end
|
|
50
51
|
end
|
|
51
52
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module TransactionalOutbox
|
|
4
|
+
class Producer
|
|
5
|
+
class Adapters
|
|
6
|
+
module InstanceMethods
|
|
7
|
+
def initialize(connection_config)
|
|
8
|
+
@connection_config = connection_config
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
attr_reader :connection_config
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -4,15 +4,8 @@ module TransactionalOutbox
|
|
|
4
4
|
class Producer
|
|
5
5
|
class Adapters
|
|
6
6
|
class Interface
|
|
7
|
-
def
|
|
8
|
-
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
def produce_batch(_topic, _batch) = raise NotImplementedError
|
|
12
|
-
|
|
13
|
-
private
|
|
14
|
-
|
|
15
|
-
attr_reader :client
|
|
7
|
+
def produce_batch(_queue, _batch) = raise NotImplementedError
|
|
8
|
+
def close = raise NotImplementedError
|
|
16
9
|
end
|
|
17
10
|
end
|
|
18
11
|
end
|
|
@@ -3,15 +3,18 @@
|
|
|
3
3
|
module TransactionalOutbox
|
|
4
4
|
class Producer
|
|
5
5
|
class Adapters
|
|
6
|
-
class Kafka <
|
|
6
|
+
class Kafka < Base
|
|
7
7
|
def produce_batch(topic, events)
|
|
8
8
|
buffer_events(topic, events)
|
|
9
9
|
|
|
10
10
|
producer.deliver_messages
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
+
def close = client.close
|
|
14
|
+
|
|
13
15
|
private
|
|
14
16
|
|
|
17
|
+
def client = @client ||= ::Kafka.new(**connection_config)
|
|
15
18
|
def producer = @producer ||= client.producer
|
|
16
19
|
|
|
17
20
|
def buffer_events(topic, events)
|
|
@@ -3,17 +3,19 @@
|
|
|
3
3
|
module TransactionalOutbox
|
|
4
4
|
class Producer
|
|
5
5
|
class Adapters
|
|
6
|
-
class Null <
|
|
6
|
+
class Null < Base
|
|
7
7
|
def messages = @messages ||= {}
|
|
8
8
|
def clear_store = @messages = {}
|
|
9
9
|
|
|
10
|
-
def produce_batch(
|
|
11
|
-
msg = messages[
|
|
10
|
+
def produce_batch(queue, events)
|
|
11
|
+
msg = messages[queue] ||= []
|
|
12
12
|
|
|
13
13
|
msg.concat(events)
|
|
14
14
|
|
|
15
|
-
TransactionalOutbox.config.logger
|
|
15
|
+
TransactionalOutbox.config.logger&.info("Batch produced")
|
|
16
16
|
end
|
|
17
|
+
|
|
18
|
+
def close = true
|
|
17
19
|
end
|
|
18
20
|
end
|
|
19
21
|
end
|
|
@@ -4,12 +4,12 @@ module TransactionalOutbox
|
|
|
4
4
|
class Producer
|
|
5
5
|
extend Forwardable
|
|
6
6
|
|
|
7
|
-
def_delegators :@adapter, :produce_batch
|
|
7
|
+
def_delegators :@adapter, :produce_batch, :close
|
|
8
8
|
|
|
9
9
|
attr_reader :adapter
|
|
10
10
|
|
|
11
11
|
def initialize
|
|
12
|
-
@adapter = TransactionalOutbox::Producer::Adapters.resolve(fetch_adapter).new(
|
|
12
|
+
@adapter = TransactionalOutbox::Producer::Adapters.resolve(fetch_adapter).new(config.producer.connection_config)
|
|
13
13
|
end
|
|
14
14
|
|
|
15
15
|
private
|
|
@@ -21,11 +21,5 @@ module TransactionalOutbox
|
|
|
21
21
|
|
|
22
22
|
config.producer.adapter.to_sym
|
|
23
23
|
end
|
|
24
|
-
|
|
25
|
-
def fetch_client
|
|
26
|
-
return if config.test_environment
|
|
27
|
-
|
|
28
|
-
config.producer.client
|
|
29
|
-
end
|
|
30
24
|
end
|
|
31
25
|
end
|
|
@@ -3,27 +3,23 @@
|
|
|
3
3
|
module TransactionalOutbox
|
|
4
4
|
class Relay
|
|
5
5
|
class EventProcessor
|
|
6
|
-
attr_reader :
|
|
6
|
+
attr_reader :queue, :db, :producer
|
|
7
7
|
|
|
8
|
-
def initialize(
|
|
9
|
-
@
|
|
8
|
+
def initialize(queue, producer)
|
|
9
|
+
@queue = queue
|
|
10
10
|
@db = TransactionalOutbox::Database.new
|
|
11
|
-
@producer =
|
|
11
|
+
@producer = producer
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
def call
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
return if events.empty?
|
|
18
|
-
|
|
19
|
-
process_events(events)
|
|
15
|
+
process
|
|
20
16
|
rescue StandardError => e
|
|
21
17
|
TransactionalOutbox::Relay.monitor.publish(
|
|
22
18
|
TransactionalOutbox::WORKER_EXCEPTIONS_TOTAL_MONITOR_EVENT,
|
|
23
|
-
{
|
|
19
|
+
{ queue:, exception: e.class.to_s }
|
|
24
20
|
)
|
|
25
21
|
|
|
26
|
-
resolve_failover.call(e, events)
|
|
22
|
+
resolve_failover.call(e, @events)
|
|
27
23
|
end
|
|
28
24
|
|
|
29
25
|
private
|
|
@@ -33,22 +29,34 @@ module TransactionalOutbox
|
|
|
33
29
|
def resolve_failover
|
|
34
30
|
configured = config.relay.failover
|
|
35
31
|
|
|
36
|
-
return Object.
|
|
32
|
+
return Object.const_get(configured) if configured.is_a?(String)
|
|
37
33
|
|
|
38
34
|
configured
|
|
39
35
|
end
|
|
40
36
|
|
|
41
|
-
def
|
|
42
|
-
|
|
37
|
+
def process # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
38
|
+
db.transaction do
|
|
39
|
+
@events = db.fetch_events(queue, config.relay.batch_size)
|
|
40
|
+
|
|
41
|
+
db.move_to_processing(events_ids) unless @events.empty?
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
return if @events.empty?
|
|
43
45
|
|
|
44
|
-
|
|
46
|
+
producer.produce_batch(queue, @events)
|
|
47
|
+
|
|
48
|
+
db.delete_events(events_ids)
|
|
49
|
+
|
|
50
|
+
events_count = @events.size
|
|
45
51
|
|
|
46
52
|
TransactionalOutbox::Relay.monitor.publish(
|
|
47
|
-
TransactionalOutbox::WORKER_EVENTS_PROCESSED_MONITOR_EVENT, {
|
|
53
|
+
TransactionalOutbox::WORKER_EVENTS_PROCESSED_MONITOR_EVENT, { queue:, count: events_count }
|
|
48
54
|
)
|
|
49
55
|
|
|
50
|
-
config.logger
|
|
56
|
+
config.logger&.info("Events have sent to queue #{queue}: #{events_count}")
|
|
51
57
|
end
|
|
58
|
+
|
|
59
|
+
def events_ids = @events_ids ||= @events.map { |e| e[:id] }
|
|
52
60
|
end
|
|
53
61
|
end
|
|
54
62
|
end
|
|
@@ -4,10 +4,10 @@ module TransactionalOutbox
|
|
|
4
4
|
class Relay
|
|
5
5
|
class GracefulShutdown
|
|
6
6
|
class << self
|
|
7
|
-
def call(worker_set)
|
|
7
|
+
def call(worker_set, exit_code = 0)
|
|
8
8
|
worker_set.stop_workers
|
|
9
9
|
|
|
10
|
-
wait_workers(worker_set)
|
|
10
|
+
call_exit(exit_code) if wait_workers(worker_set)
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
private
|
|
@@ -21,7 +21,7 @@ module TransactionalOutbox
|
|
|
21
21
|
if worker_set.all_stopped?
|
|
22
22
|
TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::RUNNER_STOPPED_MONITOR_EVENT)
|
|
23
23
|
|
|
24
|
-
config.logger
|
|
24
|
+
config.logger&.info("All workers have stopped.")
|
|
25
25
|
|
|
26
26
|
return true
|
|
27
27
|
end
|
|
@@ -31,6 +31,12 @@ module TransactionalOutbox
|
|
|
31
31
|
|
|
32
32
|
false
|
|
33
33
|
end
|
|
34
|
+
|
|
35
|
+
def call_exit(code)
|
|
36
|
+
return code if config.test_environment
|
|
37
|
+
|
|
38
|
+
exit(code)
|
|
39
|
+
end
|
|
34
40
|
end
|
|
35
41
|
end
|
|
36
42
|
end
|
|
@@ -25,24 +25,16 @@ module TransactionalOutbox
|
|
|
25
25
|
def start_relay(worker_set) = TransactionalOutbox::Relay::WorkerSet::Processor.new(worker_set).call
|
|
26
26
|
def config = @config ||= TransactionalOutbox.config
|
|
27
27
|
|
|
28
|
-
def start_graceful_shutdown(worker_set)
|
|
28
|
+
def start_graceful_shutdown(worker_set, exit_code)
|
|
29
29
|
return unless worker_set
|
|
30
30
|
|
|
31
|
-
TransactionalOutbox::Relay::GracefulShutdown.call(worker_set)
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
def call_exit(code)
|
|
35
|
-
return true if config.test_environment
|
|
36
|
-
|
|
37
|
-
exit(code)
|
|
31
|
+
TransactionalOutbox::Relay::GracefulShutdown.call(worker_set, exit_code)
|
|
38
32
|
end
|
|
39
33
|
|
|
40
34
|
def shutdown(exception, worker_set, exit_code)
|
|
41
|
-
config.logger
|
|
42
|
-
|
|
43
|
-
start_graceful_shutdown(worker_set)
|
|
35
|
+
config.logger&.info("Received exception: #{exception}. Shutting down...")
|
|
44
36
|
|
|
45
|
-
|
|
37
|
+
start_graceful_shutdown(worker_set, exit_code)
|
|
46
38
|
end
|
|
47
39
|
end
|
|
48
40
|
end
|
|
@@ -13,15 +13,15 @@ module TransactionalOutbox
|
|
|
13
13
|
@retry_counter = 0
|
|
14
14
|
|
|
15
15
|
loop do
|
|
16
|
-
|
|
16
|
+
queues = db.fetch_queues
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
process_queues(queues)
|
|
19
19
|
|
|
20
20
|
break if config.test_environment
|
|
21
21
|
|
|
22
22
|
@retry_counter = 0
|
|
23
23
|
rescue StandardError => e
|
|
24
|
-
config.logger
|
|
24
|
+
config.logger&.error("Exception: #{e}, trying to retry...")
|
|
25
25
|
|
|
26
26
|
raise e if @retry_counter >= config.relay.max_runner_retries_count
|
|
27
27
|
|
|
@@ -40,14 +40,14 @@ module TransactionalOutbox
|
|
|
40
40
|
def config = @config ||= TransactionalOutbox.config
|
|
41
41
|
def calculate_retry_delay = TransactionalOutbox::ExponentialBackoff.calculate_retry_delay(@retry_counter)
|
|
42
42
|
|
|
43
|
-
def
|
|
44
|
-
|
|
45
|
-
worker = worker_set.get_worker(
|
|
43
|
+
def process_queues(queues)
|
|
44
|
+
queues.each do |queue|
|
|
45
|
+
worker = worker_set.get_worker(queue)
|
|
46
46
|
|
|
47
|
-
worker ? worker_set.try_to_recover_worker(
|
|
47
|
+
worker ? worker_set.try_to_recover_worker(queue) : worker_set.add_worker(queue)
|
|
48
48
|
end
|
|
49
49
|
|
|
50
|
-
sleep(config.relay.
|
|
50
|
+
sleep(config.relay.delay_between_worker_set_processor_cycles_seconds)
|
|
51
51
|
end
|
|
52
52
|
end
|
|
53
53
|
end
|
|
@@ -6,10 +6,11 @@ module TransactionalOutbox
|
|
|
6
6
|
class Relay
|
|
7
7
|
class WorkerSet
|
|
8
8
|
class Worker
|
|
9
|
-
attr_reader :
|
|
9
|
+
attr_reader :queue, :db, :producer
|
|
10
10
|
|
|
11
|
-
def initialize(
|
|
12
|
-
@
|
|
11
|
+
def initialize(queue)
|
|
12
|
+
@queue = queue
|
|
13
|
+
@producer = TransactionalOutbox::Producer.new
|
|
13
14
|
end
|
|
14
15
|
|
|
15
16
|
def run
|
|
@@ -32,21 +33,21 @@ module TransactionalOutbox
|
|
|
32
33
|
def config = @config ||= TransactionalOutbox.config
|
|
33
34
|
|
|
34
35
|
def spawn_thread # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
|
35
|
-
TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::WORKER_RUN_MONITOR_EVENT, {
|
|
36
|
+
TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::WORKER_RUN_MONITOR_EVENT, { queue: })
|
|
36
37
|
|
|
37
38
|
Thread.new do
|
|
38
39
|
loop do
|
|
39
|
-
TransactionalOutbox::Relay::EventProcessor.new(
|
|
40
|
+
TransactionalOutbox::Relay::EventProcessor.new(queue, producer).call
|
|
40
41
|
|
|
41
42
|
if Thread.current[:shutdown]
|
|
42
|
-
config.logger
|
|
43
|
+
config.logger&.info("Thread for queue #{queue} successfully shutted down")
|
|
43
44
|
|
|
44
45
|
mark_thread_as_stopped
|
|
45
46
|
|
|
46
47
|
break
|
|
47
48
|
end
|
|
48
49
|
|
|
49
|
-
break if config.test_environment
|
|
50
|
+
break mark_thread_as_stopped if config.test_environment
|
|
50
51
|
|
|
51
52
|
sleep(config.relay.wait_between_batches_seconds)
|
|
52
53
|
end
|
|
@@ -54,13 +55,15 @@ module TransactionalOutbox
|
|
|
54
55
|
mark_thread_as_stopped
|
|
55
56
|
|
|
56
57
|
raise e
|
|
58
|
+
ensure
|
|
59
|
+
producer&.close
|
|
57
60
|
end
|
|
58
61
|
end
|
|
59
62
|
|
|
60
63
|
def mark_thread_as_stopped
|
|
61
64
|
Thread.current[:stopped] = true
|
|
62
65
|
|
|
63
|
-
TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::WORKER_STOPPED_MONITOR_EVENT, {
|
|
66
|
+
TransactionalOutbox::Relay.monitor.publish(TransactionalOutbox::WORKER_STOPPED_MONITOR_EVENT, { queue: })
|
|
64
67
|
end
|
|
65
68
|
end
|
|
66
69
|
end
|
|
@@ -9,20 +9,20 @@ module TransactionalOutbox
|
|
|
9
9
|
@workers = {}
|
|
10
10
|
end
|
|
11
11
|
|
|
12
|
-
def get_worker(
|
|
12
|
+
def get_worker(queue) = workers[queue]
|
|
13
13
|
|
|
14
|
-
def add_worker(
|
|
15
|
-
return if workers.key?(
|
|
14
|
+
def add_worker(queue)
|
|
15
|
+
return if workers.key?(queue)
|
|
16
16
|
|
|
17
|
-
create_worker(
|
|
17
|
+
create_worker(queue)
|
|
18
18
|
end
|
|
19
19
|
|
|
20
|
-
def try_to_recover_worker(
|
|
21
|
-
worker = get_worker(
|
|
20
|
+
def try_to_recover_worker(queue)
|
|
21
|
+
worker = get_worker(queue)
|
|
22
22
|
|
|
23
23
|
return if !worker.nil? && (worker.shutting_down? || !worker.stopped?)
|
|
24
24
|
|
|
25
|
-
create_worker(
|
|
25
|
+
create_worker(queue)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
def stop_workers = workers.each_value(&:shutdown)
|
|
@@ -32,7 +32,7 @@ module TransactionalOutbox
|
|
|
32
32
|
|
|
33
33
|
attr_reader :workers
|
|
34
34
|
|
|
35
|
-
def create_worker(
|
|
35
|
+
def create_worker(queue) = workers[queue] = Worker.new(queue).tap(&:run)
|
|
36
36
|
end
|
|
37
37
|
end
|
|
38
38
|
end
|
data/lib/transactional_outbox.rb
CHANGED
|
@@ -12,12 +12,12 @@ require "securerandom"
|
|
|
12
12
|
require_relative "transactional_outbox/version"
|
|
13
13
|
require_relative "transactional_outbox/adapters_container"
|
|
14
14
|
require_relative "transactional_outbox/database/adapters"
|
|
15
|
-
require_relative "transactional_outbox/database/adapters/
|
|
15
|
+
require_relative "transactional_outbox/database/adapters/base"
|
|
16
16
|
require_relative "transactional_outbox/database/adapters/null"
|
|
17
17
|
require_relative "transactional_outbox/database/adapters/active_record"
|
|
18
18
|
require_relative "transactional_outbox/database/adapters/sequel"
|
|
19
19
|
require_relative "transactional_outbox/producer/adapters"
|
|
20
|
-
require_relative "transactional_outbox/producer/adapters/
|
|
20
|
+
require_relative "transactional_outbox/producer/adapters/base"
|
|
21
21
|
require_relative "transactional_outbox/producer/adapters/null"
|
|
22
22
|
require_relative "transactional_outbox/producer/adapters/kafka"
|
|
23
23
|
require_relative "transactional_outbox/constants"
|
|
@@ -49,7 +49,7 @@ module TransactionalOutbox
|
|
|
49
49
|
|
|
50
50
|
setting :logger
|
|
51
51
|
setting :outbox_table_name, default: "outbox_events"
|
|
52
|
-
setting :default_event_builder, default: TransactionalOutbox::Event::Builder
|
|
52
|
+
setting :default_event_builder, default: "TransactionalOutbox::Event::Builder"
|
|
53
53
|
setting :shutdown_waiting_time_seconds, default: 10
|
|
54
54
|
setting :migrations_directory
|
|
55
55
|
setting :test_environment, default: false
|
|
@@ -57,9 +57,10 @@ module TransactionalOutbox
|
|
|
57
57
|
setting :relay do
|
|
58
58
|
setting :batch_size, default: 20
|
|
59
59
|
setting :wait_between_batches_seconds, default: 0.1
|
|
60
|
-
setting :failover, default: TransactionalOutbox::Relay::Failover
|
|
60
|
+
setting :failover, default: "TransactionalOutbox::Relay::Failover"
|
|
61
61
|
setting :max_runner_retries_count, default: 5
|
|
62
|
-
setting :
|
|
62
|
+
setting :delay_between_worker_set_processor_cycles_seconds, default: 1
|
|
63
|
+
setting :processing_events_claim_timeout_seconds, default: 300
|
|
63
64
|
end
|
|
64
65
|
|
|
65
66
|
setting :db do
|
|
@@ -69,14 +70,12 @@ module TransactionalOutbox
|
|
|
69
70
|
|
|
70
71
|
setting :producer do
|
|
71
72
|
setting :adapter
|
|
72
|
-
setting :
|
|
73
|
+
setting :connection_config
|
|
73
74
|
end
|
|
74
75
|
|
|
75
76
|
class << self
|
|
76
|
-
def
|
|
77
|
-
Database.new.transaction
|
|
78
|
-
yield(event.new)
|
|
79
|
-
end
|
|
77
|
+
def outboxable(&)
|
|
78
|
+
Database.new.transaction(&)
|
|
80
79
|
end
|
|
81
80
|
end
|
|
82
81
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: transactional_outbox
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.0beta2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Roman Kakorin
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-08-
|
|
11
|
+
date: 2026-08-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: dry-configurable
|
|
@@ -202,6 +202,8 @@ files:
|
|
|
202
202
|
- lib/transactional_outbox/database.rb
|
|
203
203
|
- lib/transactional_outbox/database/adapters.rb
|
|
204
204
|
- lib/transactional_outbox/database/adapters/active_record.rb
|
|
205
|
+
- lib/transactional_outbox/database/adapters/base.rb
|
|
206
|
+
- lib/transactional_outbox/database/adapters/instance_methods.rb
|
|
205
207
|
- lib/transactional_outbox/database/adapters/interface.rb
|
|
206
208
|
- lib/transactional_outbox/database/adapters/null.rb
|
|
207
209
|
- lib/transactional_outbox/database/adapters/sequel.rb
|
|
@@ -213,6 +215,8 @@ files:
|
|
|
213
215
|
- lib/transactional_outbox/exponential_backoff.rb
|
|
214
216
|
- lib/transactional_outbox/producer.rb
|
|
215
217
|
- lib/transactional_outbox/producer/adapters.rb
|
|
218
|
+
- lib/transactional_outbox/producer/adapters/base.rb
|
|
219
|
+
- lib/transactional_outbox/producer/adapters/instance_methods.rb
|
|
216
220
|
- lib/transactional_outbox/producer/adapters/interface.rb
|
|
217
221
|
- lib/transactional_outbox/producer/adapters/kafka.rb
|
|
218
222
|
- lib/transactional_outbox/producer/adapters/null.rb
|
|
@@ -226,7 +230,7 @@ files:
|
|
|
226
230
|
- lib/transactional_outbox/relay/worker_set.rb
|
|
227
231
|
- lib/transactional_outbox/relay/worker_set/processor.rb
|
|
228
232
|
- lib/transactional_outbox/relay/worker_set/worker.rb
|
|
229
|
-
- lib/transactional_outbox/tasks/
|
|
233
|
+
- lib/transactional_outbox/tasks/event_relay.rake
|
|
230
234
|
- lib/transactional_outbox/version.rb
|
|
231
235
|
homepage: https://github.com/sigmen/transactional_outbox
|
|
232
236
|
licenses:
|