outboxer 0.1.10 → 1.0.0.pre.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +86 -48
- data/db/migrate/create_outboxer_exceptions.rb +20 -0
- data/db/migrate/create_outboxer_frames.rb +16 -0
- data/db/migrate/create_outboxer_messages.rb +19 -0
- data/generators/message_publisher_generator.rb +11 -0
- data/generators/{outboxer/install_generator.rb → schema_generator.rb} +4 -9
- data/lib/outboxer/database.rb +44 -0
- data/lib/outboxer/logger.rb +17 -0
- data/lib/outboxer/message.rb +262 -13
- data/lib/outboxer/messages.rb +232 -0
- data/lib/outboxer/models/exception.rb +15 -0
- data/lib/outboxer/models/frame.rb +14 -0
- data/lib/outboxer/models/message.rb +46 -0
- data/lib/outboxer/models.rb +3 -0
- data/lib/outboxer/railtie.rb +2 -4
- data/lib/outboxer/version.rb +1 -1
- data/lib/outboxer/web/public/css/bootstrap-icons.css +2078 -0
- data/lib/outboxer/web/public/css/bootstrap-icons.min.css +5 -0
- data/lib/outboxer/web/public/css/bootstrap.css +12071 -0
- data/lib/outboxer/web/public/css/bootstrap.min.css +6 -0
- data/lib/outboxer/web/public/css/fonts/bootstrap-icons.woff +0 -0
- data/lib/outboxer/web/public/css/fonts/bootstrap-icons.woff2 +0 -0
- data/lib/outboxer/web/public/favicon.svg +3 -0
- data/lib/outboxer/web/public/js/bootstrap.bundle.js +6306 -0
- data/lib/outboxer/web/public/js/bootstrap.bundle.min.js +7 -0
- data/lib/outboxer/web/views/error.erb +63 -0
- data/lib/outboxer/web/views/home.erb +172 -0
- data/lib/outboxer/web/views/layout.erb +80 -0
- data/lib/outboxer/web/views/message.erb +81 -0
- data/lib/outboxer/web/views/messages/index.erb +60 -0
- data/lib/outboxer/web/views/messages/show.erb +31 -0
- data/lib/outboxer/web/views/messages.erb +262 -0
- data/lib/outboxer/web.rb +430 -0
- data/lib/outboxer.rb +9 -5
- metadata +279 -22
- data/.rspec +0 -3
- data/.rubocop.yml +0 -229
- data/CHANGELOG.md +0 -5
- data/CODE_OF_CONDUCT.md +0 -84
- data/LICENSE.txt +0 -21
- data/Rakefile +0 -14
- data/generators/outboxer/templates/bin/publisher.rb +0 -11
- data/generators/outboxer/templates/migrations/create_outboxer_exceptions.rb +0 -15
- data/generators/outboxer/templates/migrations/create_outboxer_messages.rb +0 -13
- data/lib/outboxer/exception.rb +0 -9
- data/lib/outboxer/outboxable.rb +0 -21
- data/lib/outboxer/publisher.rb +0 -132
- data/lib/tasks/gem.rake +0 -58
- data/outboxer.gemspec +0 -33
- data/sig/outboxer.rbs +0 -19
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 02b688da126c6e1e723b3f7df4f5b2bcf985c9d204051292e4b10f76ee9f1060
|
4
|
+
data.tar.gz: 512b45ceaa6f67f454c5fa9690684852624c76af9c804697ba79d3f5119a9d6e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f1aaaf3225fc64d1371ceaf295a1184296c691d355d8fa082ef201fe3efc82ed6767cfb60592855012853e1e8f1529a99d6d7f199d80a2e41a0b3dca3bfd2a8d
|
7
|
+
data.tar.gz: 2087f71e099d99f80b053bc61d4fee3e591205aa499946ced71c19ea0ad74dc17a0684029e52880f95de470ab93324827e6f29ca92f179a2e9ef2bc80b0c1fae
|
data/README.md
CHANGED
@@ -1,96 +1,134 @@
|
|
1
1
|
# Outboxer
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
Typically in event driven Ruby on Rails applications:
|
6
|
-
|
7
|
-
1. a domain event model is created in an SQL database table
|
8
|
-
2. a sidekiq worker is queued to handle the domain event asynchronously
|
9
|
-
|
10
|
-
## Problem
|
11
|
-
|
12
|
-
As these two operations span multiple database types (SQL and redis), they can not be combined into a single atomic operation using a transaction. If either step fails, inconsistencies can occur.
|
13
|
-
|
14
|
-
## Solution
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/outboxer.svg)](https://badge.fury.io/rb/outboxer)
|
4
|
+
![Ruby](https://github.com/fast-programmer/outboxer/actions/workflows/master.yml/badge.svg)
|
15
5
|
|
16
|
-
|
6
|
+
## Background
|
17
7
|
|
18
|
-
|
8
|
+
Outboxer is an ActiveRecord implementation of the [transactional outbox pattern](https://microservices.io/patterns/data/transactional-outbox.html).
|
19
9
|
|
20
|
-
|
10
|
+
## Installation
|
21
11
|
|
22
|
-
1.
|
12
|
+
### 1. add gem to your application's gemfile
|
23
13
|
|
24
|
-
```
|
14
|
+
```
|
25
15
|
gem 'outboxer'
|
26
16
|
```
|
27
17
|
|
28
|
-
2.
|
18
|
+
### 2. install gem
|
29
19
|
|
30
|
-
```
|
20
|
+
```
|
31
21
|
bundle install
|
32
22
|
```
|
33
23
|
|
34
|
-
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
### 1. generate schema
|
35
27
|
|
36
28
|
```bash
|
37
|
-
bin/rails
|
29
|
+
bin/rails g outboxer:schema
|
38
30
|
```
|
39
31
|
|
40
|
-
###
|
32
|
+
### 2. migrate schema
|
33
|
+
|
34
|
+
```bash
|
35
|
+
bin/rake db:migrate
|
36
|
+
```
|
41
37
|
|
42
|
-
|
38
|
+
### 3. after event created, backlog message
|
43
39
|
|
44
40
|
```ruby
|
45
|
-
class
|
46
|
-
|
41
|
+
class Event < ActiveRecord::Base
|
42
|
+
# ...
|
43
|
+
|
44
|
+
after_create do |event|
|
45
|
+
Outboxer::Message.backlog(
|
46
|
+
messageable_type: event.class.name,
|
47
|
+
messageable_id: event.id)
|
48
|
+
end
|
47
49
|
end
|
48
50
|
```
|
49
51
|
|
50
|
-
|
52
|
+
### 4. add event created job
|
51
53
|
|
52
54
|
```ruby
|
53
|
-
|
54
|
-
|
55
|
-
message = args.message
|
55
|
+
class EventCreatedJob
|
56
|
+
include Sidekiq::Job
|
56
57
|
|
57
|
-
|
58
|
-
|
59
|
-
|
58
|
+
def perform(args)
|
59
|
+
event = Event.find(args['id'])
|
60
|
+
|
61
|
+
# ...
|
62
|
+
end
|
60
63
|
end
|
61
64
|
```
|
62
65
|
|
63
|
-
|
66
|
+
### 5. generate message publisher
|
64
67
|
|
65
68
|
```bash
|
66
|
-
bin/
|
69
|
+
bin/rails g outboxer:message_publisher
|
70
|
+
```
|
71
|
+
|
72
|
+
### 6. update publish block to perform event created job async
|
73
|
+
|
74
|
+
```ruby
|
75
|
+
Outboxer::Publisher.publish do |message|
|
76
|
+
case message[:messageable_type]
|
77
|
+
when 'Event'
|
78
|
+
EventCreatedJob.perform_async({ 'id' => message[:messageable_id] })
|
79
|
+
end
|
80
|
+
end
|
67
81
|
```
|
68
82
|
|
69
|
-
|
83
|
+
### 6. run message publisher
|
70
84
|
|
71
85
|
```bash
|
72
|
-
bin/
|
86
|
+
bin/outboxer_message_publisher
|
73
87
|
```
|
74
88
|
|
75
|
-
|
89
|
+
### 7. manage messages
|
76
90
|
|
77
|
-
|
91
|
+
manage backlogged, queued, publishing and failed messages with the web ui
|
78
92
|
|
79
|
-
|
80
|
-
- removes it if the task completes successfully
|
81
|
-
- marks it as failed and records the error if there's a problem
|
93
|
+
<img width="1257" alt="Screenshot 2024-05-20 at 8 47 57 pm" src="https://github.com/fast-programmer/outboxer/assets/394074/0446bc7e-9d5f-4fe1-b210-ff394bdacdd6">
|
82
94
|
|
83
|
-
|
95
|
+
#### rails
|
84
96
|
|
97
|
+
##### config/routes.rb
|
85
98
|
|
86
|
-
|
99
|
+
```ruby
|
100
|
+
require 'outboxer/web'
|
101
|
+
|
102
|
+
Rails.application.routes.draw do
|
103
|
+
mount Outboxer::Web, at: '/outboxer'
|
104
|
+
end
|
105
|
+
```
|
106
|
+
|
107
|
+
#### rack
|
108
|
+
|
109
|
+
##### config.ru
|
110
|
+
|
111
|
+
```ruby
|
112
|
+
require 'outboxer/web'
|
87
113
|
|
88
|
-
|
114
|
+
map '/outboxer' do
|
115
|
+
run Outboxer::Web
|
116
|
+
end
|
117
|
+
```
|
118
|
+
|
119
|
+
### 8. monitor message publisher
|
120
|
+
|
121
|
+
understanding how much memory and cpu is required by the message publisher
|
122
|
+
|
123
|
+
<img width="310" alt="Screenshot 2024-05-20 at 10 41 57 pm" src="https://github.com/fast-programmer/outboxer/assets/394074/1222ad47-15e3-44d1-bb45-6abc6b3e4325">
|
124
|
+
|
125
|
+
```bash
|
126
|
+
run bin/outboxer_message_publishermon
|
127
|
+
```
|
128
|
+
|
129
|
+
## Motivation
|
89
130
|
|
90
|
-
|
91
|
-
2. comprehensive documentation that is easy to understand
|
92
|
-
3. high reliability in production environments (100% code coverage)
|
93
|
-
4. forever free to use in commerical applications (MIT licence)
|
131
|
+
Outboxer was created to help Rails teams migrate to eventually consistent event driven architecture quickly, using existing tools and infrastructure.
|
94
132
|
|
95
133
|
## Contributing
|
96
134
|
|
@@ -0,0 +1,20 @@
|
|
1
|
+
class CreateOutboxerExceptions < ActiveRecord::Migration[6.1]
|
2
|
+
def up
|
3
|
+
ActiveRecord::Base.transaction do
|
4
|
+
create_table :outboxer_exceptions do |t|
|
5
|
+
t.references :message, null: false, foreign_key: { to_table: :outboxer_messages }
|
6
|
+
|
7
|
+
t.text :class_name, null: false
|
8
|
+
t.text :message_text, null: false
|
9
|
+
|
10
|
+
t.timestamps
|
11
|
+
end
|
12
|
+
|
13
|
+
remove_column :outboxer_exceptions, :updated_at
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def down
|
18
|
+
drop_table :outboxer_exceptions if table_exists?(:outboxer_exceptions)
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
class CreateOutboxerFrames < ActiveRecord::Migration[6.1]
|
2
|
+
def up
|
3
|
+
create_table :outboxer_frames do |t|
|
4
|
+
t.references :exception, null: false, foreign_key: { to_table: :outboxer_exceptions }
|
5
|
+
|
6
|
+
t.integer :index, null: false
|
7
|
+
t.text :text, null: false
|
8
|
+
|
9
|
+
t.index [:exception_id, :index], unique: true
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def down
|
14
|
+
drop_table :outboxer_frames if table_exists?(:outboxer_frames)
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
class CreateOutboxerMessages < ActiveRecord::Migration[6.1]
|
2
|
+
def up
|
3
|
+
create_table :outboxer_messages do |t|
|
4
|
+
t.string :status, null: false, limit: 255
|
5
|
+
|
6
|
+
t.text :messageable_id, null: false
|
7
|
+
t.text :messageable_type, null: false
|
8
|
+
|
9
|
+
t.timestamps
|
10
|
+
end
|
11
|
+
|
12
|
+
add_index :outboxer_messages, :status
|
13
|
+
add_index :outboxer_messages, [:status, :updated_at]
|
14
|
+
end
|
15
|
+
|
16
|
+
def down
|
17
|
+
drop_table :outboxer_messages if table_exists?(:outboxer_messages)
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
module Outboxer
|
2
|
+
class MessagePublisherGenerator < Rails::Generators::Base
|
3
|
+
include Rails::Generators::Migration
|
4
|
+
|
5
|
+
source_root File.expand_path('../', __dir__)
|
6
|
+
def copy_bin_file
|
7
|
+
template "bin/outboxer_message_publisher", "bin/outboxer_message_publisher"
|
8
|
+
run "chmod +x bin/outboxer_message_publisher"
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -1,8 +1,8 @@
|
|
1
1
|
module Outboxer
|
2
|
-
class
|
2
|
+
class SchemaGenerator < Rails::Generators::Base
|
3
3
|
include Rails::Generators::Migration
|
4
4
|
|
5
|
-
source_root File.expand_path(
|
5
|
+
source_root File.expand_path('../', __dir__)
|
6
6
|
|
7
7
|
def self.next_migration_number(dirname)
|
8
8
|
next_number = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
|
@@ -12,18 +12,13 @@ module Outboxer
|
|
12
12
|
next_number.to_s
|
13
13
|
end
|
14
14
|
|
15
|
-
def copy_bin_file
|
16
|
-
template "bin/publisher.rb", "bin/publisher"
|
17
|
-
run "chmod +x bin/publisher"
|
18
|
-
end
|
19
|
-
|
20
15
|
def copy_migrations
|
21
16
|
migration_template(
|
22
|
-
"
|
17
|
+
"db/migrate/create_outboxer_messages.rb",
|
23
18
|
"db/migrate/create_outboxer_messages.rb")
|
24
19
|
|
25
20
|
migration_template(
|
26
|
-
"
|
21
|
+
"db/migrate/create_outboxer_exceptions.rb",
|
27
22
|
"db/migrate/create_outboxer_exceptions.rb")
|
28
23
|
end
|
29
24
|
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'erb'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Outboxer
|
5
|
+
module Database
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def config(environment: ENV['RAILS_ENV'] || 'development', path: 'config/database.yml')
|
9
|
+
db_config_content = File.read(path)
|
10
|
+
db_config_erb_result = ERB.new(db_config_content).result
|
11
|
+
YAML.safe_load(db_config_erb_result, aliases: true)[environment]
|
12
|
+
end
|
13
|
+
|
14
|
+
def connect(config:, logger: Logger.new($stdout, level: Logger::INFO))
|
15
|
+
logger&.info " _ _ "
|
16
|
+
logger&.info " | | | | "
|
17
|
+
logger&.info " ___ _ _| |_| |__ _____ _____ _ __ "
|
18
|
+
logger&.info " / _ \\| | | | __| '_ \\ / _ \\ \\/ / _ \\ '__|"
|
19
|
+
logger&.info " | (_) | |_| | |_| |_) | (_) > < __/ | "
|
20
|
+
logger&.info " \\___/ \\__,_|\\__|_.__/ \\___/_/\\_\\___|_| "
|
21
|
+
logger&.info " "
|
22
|
+
logger&.info " "
|
23
|
+
|
24
|
+
logger&.info "Running in ruby #{RUBY_VERSION} " \
|
25
|
+
"(#{RUBY_RELEASE_DATE} revision #{RUBY_REVISION[0, 10]}) [#{RUBY_PLATFORM}]"
|
26
|
+
|
27
|
+
logger&.info "Connecting to database"
|
28
|
+
ActiveRecord::Base.establish_connection(config)
|
29
|
+
ActiveRecord::Base.connection_pool.with_connection {}
|
30
|
+
logger&.info "Connected to database"
|
31
|
+
end
|
32
|
+
|
33
|
+
def connected?
|
34
|
+
ActiveRecord::Base.connected?
|
35
|
+
end
|
36
|
+
|
37
|
+
def disconnect(logger: Logger.new($stdout, level: Logger::INFO))
|
38
|
+
logger&.info "Disconnecting from database"
|
39
|
+
ActiveRecord::Base.connection_handler.clear_active_connections!
|
40
|
+
ActiveRecord::Base.connection_handler.connection_pool_list.each(&:disconnect!)
|
41
|
+
logger&.info "Disconnected from database"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Outboxer
|
4
|
+
class Logger < ::Logger
|
5
|
+
def initialize(*args, **kwargs)
|
6
|
+
super(*args, **kwargs)
|
7
|
+
|
8
|
+
self.formatter = proc do |severity, datetime, progname, msg|
|
9
|
+
formatted_time = datetime.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
|
10
|
+
pid = Process.pid
|
11
|
+
tid = Thread.current.object_id.to_s(36)
|
12
|
+
level = severity
|
13
|
+
"#{formatted_time} pid=#{pid} tid=#{tid} #{level}: #{msg}\n"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/outboxer/message.rb
CHANGED
@@ -1,19 +1,268 @@
|
|
1
|
-
require "active_record"
|
2
|
-
|
3
1
|
module Outboxer
|
4
|
-
|
5
|
-
self
|
2
|
+
module Message
|
3
|
+
extend self
|
4
|
+
|
5
|
+
Status = Models::Message::Status
|
6
|
+
|
7
|
+
def backlog(messageable_type:, messageable_id:)
|
8
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
9
|
+
ActiveRecord::Base.transaction do
|
10
|
+
current_time = Time.now.utc
|
11
|
+
|
12
|
+
message = Models::Message.create!(
|
13
|
+
messageable_id: messageable_id,
|
14
|
+
messageable_type: messageable_type,
|
15
|
+
status: Models::Message::Status::BACKLOGGED,
|
16
|
+
created_at: current_time,
|
17
|
+
updated_at: current_time)
|
18
|
+
|
19
|
+
{ id: message.id }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def find_by_id(id:)
|
25
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
26
|
+
ActiveRecord::Base.transaction do
|
27
|
+
message = Models::Message.includes(exceptions: :frames).find_by!(id: id)
|
28
|
+
|
29
|
+
{
|
30
|
+
id: message.id,
|
31
|
+
status: message.status,
|
32
|
+
messageable_type: message.messageable_type,
|
33
|
+
messageable_id: message.messageable_id,
|
34
|
+
created_at: message.created_at.utc.to_s,
|
35
|
+
updated_at: message.updated_at.utc.to_s,
|
36
|
+
exceptions: message.exceptions.map do |exception|
|
37
|
+
{
|
38
|
+
id: exception.id,
|
39
|
+
class_name: exception.class_name,
|
40
|
+
message_text: exception.message_text,
|
41
|
+
created_at: exception.created_at.utc.to_s,
|
42
|
+
frames: exception.frames.map do |frame|
|
43
|
+
{
|
44
|
+
id: frame.id,
|
45
|
+
index: frame.index,
|
46
|
+
text: frame.text
|
47
|
+
}
|
48
|
+
end
|
49
|
+
}
|
50
|
+
end
|
51
|
+
}
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def publishing(id:)
|
57
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
58
|
+
ActiveRecord::Base.transaction do
|
59
|
+
message = Models::Message.lock.find_by!(id: id)
|
60
|
+
|
61
|
+
if message.status != Models::Message::Status::QUEUED
|
62
|
+
raise ArgumentError,
|
63
|
+
"cannot transition outboxer message #{message.id} " \
|
64
|
+
"from #{message.status} to #{Models::Message::Status::PUBLISHING}"
|
65
|
+
end
|
66
|
+
|
67
|
+
message.update!(
|
68
|
+
status: Models::Message::Status::PUBLISHING,
|
69
|
+
updated_at: Time.now.utc)
|
70
|
+
|
71
|
+
{
|
72
|
+
id: id,
|
73
|
+
status: message.status,
|
74
|
+
messageable_type: message.messageable_type,
|
75
|
+
messageable_id: message.messageable_id
|
76
|
+
}
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def published(id:)
|
82
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
83
|
+
ActiveRecord::Base.transaction do
|
84
|
+
message = Models::Message.lock.find_by!(id: id)
|
85
|
+
|
86
|
+
if message.status != Models::Message::Status::PUBLISHING
|
87
|
+
raise ArgumentError,
|
88
|
+
"cannot transition outboxer message #{message.id} " \
|
89
|
+
"from #{message.status} to (deleted)"
|
90
|
+
end
|
91
|
+
|
92
|
+
message.exceptions.each { |exception| exception.frames.each(&:delete) }
|
93
|
+
message.exceptions.delete_all
|
94
|
+
message.delete
|
95
|
+
|
96
|
+
{ id: id }
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def failed(id:, exception:)
|
102
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
103
|
+
ActiveRecord::Base.transaction do
|
104
|
+
message = Models::Message.order(created_at: :asc).lock.find_by!(id: id)
|
105
|
+
|
106
|
+
if message.status != Models::Message::Status::PUBLISHING
|
107
|
+
raise ArgumentError,
|
108
|
+
"cannot transition outboxer message #{id} " \
|
109
|
+
"from #{message.status} to #{Models::Message::Status::FAILED}"
|
110
|
+
end
|
111
|
+
|
112
|
+
message.update!(
|
113
|
+
status: Models::Message::Status::FAILED,
|
114
|
+
updated_at: Time.now.utc)
|
115
|
+
|
116
|
+
outboxer_exception = message.exceptions.create!(
|
117
|
+
class_name: exception.class.name, message_text: exception.message)
|
118
|
+
|
119
|
+
exception.backtrace.each_with_index do |frame, index|
|
120
|
+
outboxer_exception.frames.create!(index: index, text: frame)
|
121
|
+
end
|
122
|
+
|
123
|
+
{ id: id }
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def delete(id:)
|
129
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
130
|
+
ActiveRecord::Base.transaction do
|
131
|
+
message = Models::Message.includes(exceptions: :frames).lock.find_by!(id: id)
|
132
|
+
|
133
|
+
message.exceptions.each { |exception| exception.frames.each(&:delete) }
|
134
|
+
message.exceptions.delete_all
|
135
|
+
message.delete
|
136
|
+
|
137
|
+
{ id: id }
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def republish(id:)
|
143
|
+
ActiveRecord::Base.connection_pool.with_connection do
|
144
|
+
ActiveRecord::Base.transaction do
|
145
|
+
message = Models::Message.lock.find_by!(id: id)
|
146
|
+
|
147
|
+
message.update!(
|
148
|
+
status: Models::Message::Status::BACKLOGGED,
|
149
|
+
updated_at: Time.now.utc)
|
150
|
+
|
151
|
+
{ id: id }
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def stop_publishing
|
157
|
+
@publishing = false
|
158
|
+
end
|
159
|
+
|
160
|
+
def publish(threads: 5, queue: 10, poll: 1,
|
161
|
+
logger: Logger.new($stdout, level: Logger::INFO),
|
162
|
+
kernel: Kernel, &block)
|
163
|
+
ruby_queue = Queue.new
|
164
|
+
|
165
|
+
@publishing = true
|
166
|
+
|
167
|
+
logger.info "Initializing #{threads} worker threads"
|
168
|
+
ruby_threads = threads.times.map do
|
169
|
+
Thread.new do
|
170
|
+
loop do
|
171
|
+
begin
|
172
|
+
queued_message = ruby_queue.pop
|
173
|
+
break if queued_message.nil?
|
174
|
+
|
175
|
+
queued_at = queued_message[:queued_at]
|
176
|
+
|
177
|
+
message = Message.publishing(id: queued_message[:id])
|
178
|
+
logger.info "Publishing message #{message[:id]} for " \
|
179
|
+
"#{message[:messageable_type]}::#{message[:messageable_id]} " \
|
180
|
+
"in #{(Time.now.utc - queued_at).round(3)}s"
|
181
|
+
|
182
|
+
begin
|
183
|
+
block.call(message)
|
184
|
+
rescue Exception => exception
|
185
|
+
Message.failed(id: message[:id], exception: exception)
|
186
|
+
logger.error "Failed to publish message #{message[:id]} for " \
|
187
|
+
"#{message[:messageable_type]}::#{message[:messageable_id]} " \
|
188
|
+
"in #{(Time.now.utc - queued_at).round(3)}s"
|
189
|
+
|
190
|
+
raise
|
191
|
+
end
|
192
|
+
|
193
|
+
Message.published(id: message[:id])
|
194
|
+
logger.info "Published message #{message[:id]} for " \
|
195
|
+
"#{message[:messageable_type]}::#{message[:messageable_id]} " \
|
196
|
+
"in #{(Time.now.utc - queued_at).round(3)}s"
|
197
|
+
rescue StandardError => exception
|
198
|
+
logger.error "#{exception.class}: #{exception.message} " \
|
199
|
+
"in #{(Time.now.utc - queued_at).round(3)}s"
|
200
|
+
|
201
|
+
exception.backtrace.each { |frame| logger.error frame }
|
202
|
+
rescue Exception => exception
|
203
|
+
logger.fatal "#{exception.class}: #{exception.message} " \
|
204
|
+
"in #{(Time.now.utc - queued_at).round(3)}s"
|
205
|
+
exception.backtrace.each { |frame| logger.error frame }
|
206
|
+
|
207
|
+
@publishing = false
|
208
|
+
|
209
|
+
break
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
213
|
+
end
|
214
|
+
logger.info "Initialized #{threads} worker threads"
|
215
|
+
|
216
|
+
logger.info "Queuing up to #{queue} messages every #{poll}s"
|
217
|
+
while @publishing
|
218
|
+
begin
|
219
|
+
messages = []
|
220
|
+
|
221
|
+
queue_available = queue - ruby_queue.length
|
222
|
+
queue_stats = { total: queue, available: queue_available, current: ruby_queue.length }
|
223
|
+
logger.debug "Queue: #{queue_stats}"
|
224
|
+
|
225
|
+
logger.debug "Connection pool: #{ActiveRecord::Base.connection_pool.stat}"
|
226
|
+
|
227
|
+
messages = (queue_available > 0) ? Messages.queue(limit: queue_available) : []
|
228
|
+
logger.debug "Updated #{messages.length} messages from backlogged to queued"
|
229
|
+
|
230
|
+
messages.each do |message|
|
231
|
+
logger.info "Queuing message #{message[:id]} for " \
|
232
|
+
"#{message[:messageable_type]}::#{message[:messageable_id]}"
|
233
|
+
|
234
|
+
ruby_queue.push({ id: message[:id], queued_at: Time.now.utc })
|
235
|
+
end
|
236
|
+
|
237
|
+
logger.debug "Pushed #{messages.length} messages to queue."
|
238
|
+
|
239
|
+
if messages.empty?
|
240
|
+
logger.debug "Sleeping for #{poll} seconds because no messages were queued..."
|
241
|
+
|
242
|
+
kernel.sleep(poll)
|
243
|
+
elsif ruby_queue.length >= queue
|
244
|
+
logger.debug "Sleeping for #{poll} seconds because queue was full..."
|
245
|
+
|
246
|
+
kernel.sleep(poll)
|
247
|
+
end
|
248
|
+
rescue StandardError => exception
|
249
|
+
logger.error "#{exception.class}: #{exception.message}"
|
250
|
+
exception.backtrace.each { |frame| logger.error frame }
|
6
251
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
}.freeze
|
252
|
+
kernel.sleep(poll)
|
253
|
+
rescue Exception => exception
|
254
|
+
logger.fatal "#{exception.class}: #{exception.message}"
|
255
|
+
exception.backtrace.each { |frame| logger.fatal frame }
|
12
256
|
|
13
|
-
|
257
|
+
@publishing = false
|
258
|
+
end
|
259
|
+
end
|
260
|
+
logger.info "Stopped queueing messages"
|
14
261
|
|
15
|
-
|
16
|
-
|
17
|
-
|
262
|
+
logger.info "Shutting down #{threads} worker threads"
|
263
|
+
ruby_threads.length.times { ruby_queue.push(nil) }
|
264
|
+
ruby_threads.each(&:join)
|
265
|
+
logger.info "Shut down #{threads} worker threads"
|
266
|
+
end
|
18
267
|
end
|
19
268
|
end
|