sidekiq-reliable_job 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.ruby-version +1 -0
- data/CHANGELOG.md +21 -0
- data/LICENSE +22 -0
- data/README.md +200 -0
- data/Rakefile +15 -0
- data/lib/generators/sidekiq/reliable_job/install_generator.rb +29 -0
- data/lib/generators/sidekiq_reliable_job/install/install_generator.rb +29 -0
- data/lib/generators/sidekiq_reliable_job/install/templates/migration.rb.tt +16 -0
- data/lib/sidekiq/reliable_job/client.rb +31 -0
- data/lib/sidekiq/reliable_job/client_middleware.rb +66 -0
- data/lib/sidekiq/reliable_job/configuration.rb +26 -0
- data/lib/sidekiq/reliable_job/enqueuer.rb +46 -0
- data/lib/sidekiq/reliable_job/outbox.rb +24 -0
- data/lib/sidekiq/reliable_job/outbox_processor.rb +84 -0
- data/lib/sidekiq/reliable_job/server_middleware.rb +28 -0
- data/lib/sidekiq/reliable_job/version.rb +7 -0
- data/lib/sidekiq/reliable_job.rb +58 -0
- metadata +104 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 02b897ad13f432b8eecbb0e67bc800fd7ed0ada1a982f0d79d1aee13f3ba66a2
|
|
4
|
+
data.tar.gz: 14746216146d4889d85662473e6c3ed118376375ff9c4a1bc95cc1d52a92a9ac
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 52c880d82c602f394faf6926beaea0838df1053d0c836f1e1ffeb0f9d6fa763a441b1a5067d3f3485bfe36b4cc6bfbca2b15c2d7128724d3181ef815749dac96
|
|
7
|
+
data.tar.gz: e1b1cc31d1cd4e430c75cc9ed910209eecfa148e0a47b2cf63a7f39fa8aab6e38556610a2531f90f85770090dde27b6871dbd7f3d0bd539446c52a5e27c58d04
|
data/.ruby-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.4.7
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] - 2024-12-15
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Initial release
|
|
15
|
+
- Transactional outbox pattern for reliable job enqueuing
|
|
16
|
+
- Client middleware for automatic outbox creation
|
|
17
|
+
- Server middleware for job completion tracking
|
|
18
|
+
- Support for Sidekiq 7+ and Rails 7.1+
|
|
19
|
+
- Configurable outbox model and table name
|
|
20
|
+
- Advisory lock-based outbox processor
|
|
21
|
+
- Support for ActiveJob and native Sidekiq jobs
|
data/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Wealthsimple
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
6
|
+
a copy of this software and associated documentation files (the
|
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
11
|
+
the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be
|
|
14
|
+
included in all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
# Sidekiq::ReliableJob
|
|
2
|
+
|
|
3
|
+
A Sidekiq extension that provides reliable job delivery by staging jobs to the database before pushing to Redis. This ensures jobs are only enqueued when database transactions commit, and provides durability during Redis outages.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- **Transaction Safety**: Jobs are staged to the database within your transaction. If the transaction rolls back, the job is never enqueued.
|
|
8
|
+
- **Redis Outage Resilience**: Jobs continue to be accepted during Redis outages and are pushed once Redis is available.
|
|
9
|
+
- **Reliable Delivery**: A background enqueuer process polls staged jobs and pushes them to Redis.
|
|
10
|
+
- **Automatic Cleanup**: Jobs are automatically deleted from the staging table after successful completion.
|
|
11
|
+
- **ActiveJob Support**: Works with both native Sidekiq jobs and ActiveJob.
|
|
12
|
+
|
|
13
|
+
## Installation
|
|
14
|
+
|
|
15
|
+
Add this line to your application's Gemfile:
|
|
16
|
+
|
|
17
|
+
```ruby
|
|
18
|
+
gem "sidekiq-reliable_job"
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Then execute:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
bundle install
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
## Setup
|
|
28
|
+
|
|
29
|
+
### 1. Run the generator to create the migration
|
|
30
|
+
|
|
31
|
+
```bash
|
|
32
|
+
rails generate sidekiq_reliable_job:install
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
This creates a migration for the `reliable_job_outbox` table.
|
|
36
|
+
|
|
37
|
+
### 2. Run the migration
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
rails db:migrate
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
### 3. Configure Sidekiq
|
|
44
|
+
|
|
45
|
+
In your Sidekiq initializer (`config/initializers/sidekiq.rb`):
|
|
46
|
+
|
|
47
|
+
```ruby
|
|
48
|
+
require "sidekiq/reliable_job"
|
|
49
|
+
|
|
50
|
+
Sidekiq::ReliableJob.configure do |config|
|
|
51
|
+
# The ActiveRecord base class for the Outbox model (default: "ActiveRecord::Base")
|
|
52
|
+
config.base_class = "ApplicationRecord"
|
|
53
|
+
# Enable reliable job for all jobs (default: false)
|
|
54
|
+
config.enable_for_all_jobs = false
|
|
55
|
+
# Preserve dead jobs in outbox with "dead" status instead of deleting (default: false)
|
|
56
|
+
config.preserve_dead_jobs = false
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
Sidekiq.configure_client do |config|
|
|
60
|
+
Sidekiq::ReliableJob.configure_client!(config)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
Sidekiq.configure_server do |config|
|
|
64
|
+
Sidekiq::ReliableJob.configure_server!(config)
|
|
65
|
+
end
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Usage
|
|
69
|
+
|
|
70
|
+
### Option 1: Enable for all jobs (recommended)
|
|
71
|
+
|
|
72
|
+
When `enable_for_all_jobs` is `true`, all Sidekiq jobs are automatically staged through the outbox. No changes to job classes required.
|
|
73
|
+
|
|
74
|
+
To opt-out a specific job from staged push:
|
|
75
|
+
|
|
76
|
+
```ruby
|
|
77
|
+
class DirectPushJob
|
|
78
|
+
include Sidekiq::Job
|
|
79
|
+
sidekiq_options reliable_job: false
|
|
80
|
+
|
|
81
|
+
def perform
|
|
82
|
+
# This job will push directly to Redis
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
### Option 2: Enable per job (opt-in)
|
|
88
|
+
|
|
89
|
+
If `enable_for_all_jobs` is `false` (default), use `sidekiq_options` to opt-in specific jobs:
|
|
90
|
+
|
|
91
|
+
```ruby
|
|
92
|
+
class MyJob
|
|
93
|
+
include Sidekiq::Job
|
|
94
|
+
sidekiq_options reliable_job: true
|
|
95
|
+
|
|
96
|
+
def perform(user_id)
|
|
97
|
+
# Your job logic here
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### ActiveJob Support
|
|
103
|
+
|
|
104
|
+
ReliableJob works with ActiveJob. Configure the job using `sidekiq_options`:
|
|
105
|
+
|
|
106
|
+
```ruby
|
|
107
|
+
class MyActiveJob < ApplicationJob
|
|
108
|
+
queue_as :default
|
|
109
|
+
sidekiq_options reliable_job: true
|
|
110
|
+
|
|
111
|
+
def perform(user_id)
|
|
112
|
+
# Your job logic here
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### Example
|
|
118
|
+
|
|
119
|
+
When you enqueue the job within a transaction, it will be staged to the database first:
|
|
120
|
+
|
|
121
|
+
```ruby
|
|
122
|
+
ActiveRecord::Base.transaction do
|
|
123
|
+
user = User.create!(name: "Alice")
|
|
124
|
+
MyJob.perform_async(user.id) # Staged to database, not Redis
|
|
125
|
+
|
|
126
|
+
# If an exception is raised here, the job is never enqueued
|
|
127
|
+
end
|
|
128
|
+
# Transaction committed - job is now pushed to Redis by the enqueuer
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## How It Works
|
|
132
|
+
|
|
133
|
+
1. **Client Middleware**: Intercepts `perform_async` and `perform_in` calls and stages jobs to the `reliable_job_outbox` table instead of pushing directly to Redis.
|
|
134
|
+
2. **Outbox Processor**: A background thread polls for pending jobs and pushes them to Redis:
|
|
135
|
+
3. **Server Middleware**: After successful job completion, deletes the staged job record from the outbox.
|
|
136
|
+
4. **Death Handler**: When a job exhausts all retries, removes (or optionally preserves) the record from the outbox.
|
|
137
|
+
|
|
138
|
+
## Deployment & Rollout
|
|
139
|
+
|
|
140
|
+
When enabling ReliableJob for the first time, use a **two-phase deployment** to avoid orphaned outbox records:
|
|
141
|
+
|
|
142
|
+
### Phase 1: Deploy with ReliableJob Disabled
|
|
143
|
+
|
|
144
|
+
First, deploy the gem with all jobs disabled. This installs the middleware on all containers without affecting any jobs:
|
|
145
|
+
|
|
146
|
+
```ruby
|
|
147
|
+
Sidekiq::ReliableJob.configure do |config|
|
|
148
|
+
config.enable_for_all_jobs = false # No jobs use reliable delivery yet
|
|
149
|
+
end
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
Wait for all containers to be running with the new code.
|
|
153
|
+
|
|
154
|
+
### Phase 2: Enable ReliableJob
|
|
155
|
+
|
|
156
|
+
Once all containers have the middleware installed, enable reliable delivery for your jobs:
|
|
157
|
+
|
|
158
|
+
```ruby
|
|
159
|
+
Sidekiq::ReliableJob.configure do |config|
|
|
160
|
+
config.enable_for_all_jobs = true # Or enable per-job with sidekiq_options
|
|
161
|
+
end
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Why This Matters
|
|
165
|
+
|
|
166
|
+
If containers are running different versions during deployment:
|
|
167
|
+
- New containers may stage jobs while old containers process them
|
|
168
|
+
- Old containers don't have the server middleware, so they won't delete completed jobs from the outbox
|
|
169
|
+
- This leaves orphaned "enqueued" records in the database
|
|
170
|
+
|
|
171
|
+
## Configuration Options
|
|
172
|
+
|
|
173
|
+
| Option | Default | Description |
|
|
174
|
+
|--------|---------|-------------|
|
|
175
|
+
| `enable_for_all_jobs` | `false` | When `true`, all jobs are staged through the outbox |
|
|
176
|
+
| `base_class` | `"ActiveRecord::Base"` | The ActiveRecord base class for the Outbox model |
|
|
177
|
+
| `preserve_dead_jobs` | `false` | When `true`, keeps dead jobs in outbox with "dead" status instead of deleting |
|
|
178
|
+
|
|
179
|
+
## Limitations
|
|
180
|
+
|
|
181
|
+
### Batch Jobs (Sidekiq Pro/Enterprise)
|
|
182
|
+
|
|
183
|
+
Jobs that are part of a batch (have a `bid` in their payload) are **automatically bypassed** and pushed directly to Redis. This ensures batch callbacks and completion tracking work correctly.
|
|
184
|
+
|
|
185
|
+
### Internal Sidekiq Jobs
|
|
186
|
+
|
|
187
|
+
All internal Sidekiq jobs (classes starting with `Sidekiq::`) are **automatically bypassed**. This includes:
|
|
188
|
+
|
|
189
|
+
- Batch callbacks (`Sidekiq::Batch::Callback`)
|
|
190
|
+
- Batch empty handlers (`Sidekiq::Batch::Empty`)
|
|
191
|
+
- Enterprise periodic jobs (`Sidekiq::Periodic::*`)
|
|
192
|
+
- Any other internal Sidekiq system jobs
|
|
193
|
+
|
|
194
|
+
## Development
|
|
195
|
+
|
|
196
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `bundle exec rspec` to run the tests.
|
|
197
|
+
|
|
198
|
+
## Contributing
|
|
199
|
+
|
|
200
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/wealthsimple/sidekiq-reliable_job.
|
data/Rakefile
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
|
|
5
|
+
APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
|
|
6
|
+
Rake.load_rakefile "spec/dummy/Rakefile"
|
|
7
|
+
|
|
8
|
+
require "bundler/gem_tasks"
|
|
9
|
+
require "rspec/core/rake_task"
|
|
10
|
+
require "rubocop/rake_task"
|
|
11
|
+
|
|
12
|
+
RSpec::Core::RakeTask.new(:spec)
|
|
13
|
+
RuboCop::RakeTask.new
|
|
14
|
+
|
|
15
|
+
task default: %i[spec rubocop]
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module SidekiqReliableJob
|
|
7
|
+
module Generators
|
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Creates the migration for the reliable_job_outbox table"
|
|
14
|
+
|
|
15
|
+
def create_migration_file
|
|
16
|
+
migration_template(
|
|
17
|
+
"migration.rb.tt",
|
|
18
|
+
"db/migrate/create_reliable_job_outbox.rb",
|
|
19
|
+
)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def migration_version
|
|
25
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module SidekiqReliableJob
|
|
7
|
+
module Generators
|
|
8
|
+
class InstallGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Creates the migration for the reliable_job_outbox table"
|
|
14
|
+
|
|
15
|
+
def create_migration_file
|
|
16
|
+
migration_template(
|
|
17
|
+
"migration.rb.tt",
|
|
18
|
+
"db/migrate/create_reliable_job_outbox.rb",
|
|
19
|
+
)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def migration_version
|
|
25
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
class CreateReliableJobOutbox < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :reliable_job_outbox do |t|
|
|
4
|
+
t.string :jid, null: false
|
|
5
|
+
t.string :job_class, null: false
|
|
6
|
+
t.json :payload, null: false
|
|
7
|
+
t.string :status, null: false, default: "pending"
|
|
8
|
+
t.datetime :enqueued_at
|
|
9
|
+
|
|
10
|
+
t.timestamps
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
add_index :reliable_job_outbox, :jid, unique: true
|
|
14
|
+
add_index :reliable_job_outbox, %i[status id]
|
|
15
|
+
end
|
|
16
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "securerandom"
|
|
4
|
+
|
|
5
|
+
module Sidekiq
|
|
6
|
+
module ReliableJob
|
|
7
|
+
# Stages jobs to the Outbox for later delivery to Redis.
|
|
8
|
+
class Client
|
|
9
|
+
class << self
|
|
10
|
+
def push(item)
|
|
11
|
+
item["jid"] ||= SecureRandom.hex(12)
|
|
12
|
+
|
|
13
|
+
Outbox.create!(
|
|
14
|
+
jid: item["jid"],
|
|
15
|
+
job_class: extract_job_class(item),
|
|
16
|
+
payload: item,
|
|
17
|
+
status: Outbox::PENDING,
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
item["jid"]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def extract_job_class(item)
|
|
26
|
+
(item["wrapped"] || item["class"]).to_s
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module ReliableJob
|
|
5
|
+
# Intercepts job pushes and stages them to the Outbox instead of Redis.
|
|
6
|
+
class ClientMiddleware
|
|
7
|
+
def call(_job_class, job, _queue, _redis_pool)
|
|
8
|
+
return yield if skip_staging?(job)
|
|
9
|
+
|
|
10
|
+
job["reliable_job"] = true
|
|
11
|
+
Client.push(job)
|
|
12
|
+
|
|
13
|
+
yield if testing_enabled?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
private
|
|
17
|
+
|
|
18
|
+
def skip_staging?(job)
|
|
19
|
+
retry?(job) || batch?(job) || sidekiq_internal?(job) || !enabled_for?(job)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def batch?(job)
|
|
23
|
+
job.key?("bid")
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Bypass internal Sidekiq jobs (batch callbacks, Enterprise features, etc.)
|
|
27
|
+
# but not ActiveJob wrapper which should be staged
|
|
28
|
+
def sidekiq_internal?(job)
|
|
29
|
+
klass = job["class"].to_s
|
|
30
|
+
klass.start_with?("Sidekiq::") && klass != "Sidekiq::ActiveJob::Wrapper"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def enabled_for?(job)
|
|
34
|
+
option = reliable_job_option(job)
|
|
35
|
+
|
|
36
|
+
case option
|
|
37
|
+
when true then true
|
|
38
|
+
when false then false
|
|
39
|
+
else ReliableJob.configuration.enable_for_all_jobs
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def reliable_job_option(job)
|
|
44
|
+
return job["reliable_job"] if job.key?("reliable_job")
|
|
45
|
+
|
|
46
|
+
wrapped_class_option(job)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def wrapped_class_option(job)
|
|
50
|
+
wrapped = job["wrapped"]
|
|
51
|
+
return unless wrapped
|
|
52
|
+
|
|
53
|
+
klass = wrapped.is_a?(Class) ? wrapped : wrapped.to_s.safe_constantize
|
|
54
|
+
klass&.sidekiq_options_hash&.dig("reliable_job")
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def retry?(job)
|
|
58
|
+
job.key?("retry_count")
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def testing_enabled?
|
|
62
|
+
defined?(Sidekiq::Testing) && Sidekiq::Testing.enabled?
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module ReliableJob
|
|
5
|
+
# Configuration options for ReliableJob.
|
|
6
|
+
class Configuration
|
|
7
|
+
attr_accessor :base_class, :enable_for_all_jobs, :preserve_dead_jobs
|
|
8
|
+
|
|
9
|
+
def initialize
|
|
10
|
+
@base_class = "ActiveRecord::Base"
|
|
11
|
+
@enable_for_all_jobs = false
|
|
12
|
+
@preserve_dead_jobs = false
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
class << self
|
|
17
|
+
def configuration
|
|
18
|
+
@configuration ||= Configuration.new
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def configure
|
|
22
|
+
yield(configuration)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq/component"
|
|
4
|
+
|
|
5
|
+
module Sidekiq
|
|
6
|
+
module ReliableJob
|
|
7
|
+
# Background thread that continuously processes the Outbox.
|
|
8
|
+
class Enqueuer
|
|
9
|
+
include Sidekiq::Component
|
|
10
|
+
|
|
11
|
+
POLL_INTERVAL = 0.1
|
|
12
|
+
ERROR_SLEEP = 1
|
|
13
|
+
LOCK_RETRY_SLEEP = 20
|
|
14
|
+
|
|
15
|
+
def initialize(config)
|
|
16
|
+
@config = config
|
|
17
|
+
@done = false
|
|
18
|
+
@processor = OutboxProcessor.new
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def start
|
|
22
|
+
@thread = Thread.new { run }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def stop
|
|
26
|
+
@done = true
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def run
|
|
32
|
+
process_loop until @done
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def process_loop
|
|
36
|
+
count = @processor.call
|
|
37
|
+
sleep POLL_INTERVAL if count.zero?
|
|
38
|
+
rescue WithAdvisoryLock::FailedToAcquireLock
|
|
39
|
+
sleep LOCK_RETRY_SLEEP
|
|
40
|
+
rescue StandardError => e
|
|
41
|
+
logger.error "ReliableJob::Enqueuer error: #{e.class} - #{e.message}"
|
|
42
|
+
sleep ERROR_SLEEP
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module ReliableJob
|
|
5
|
+
def self.base_class
|
|
6
|
+
@base_class ||= configuration.base_class.constantize
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
# ActiveRecord model for the job staging table.
|
|
10
|
+
class Outbox < base_class
|
|
11
|
+
self.table_name = "reliable_job_outbox"
|
|
12
|
+
|
|
13
|
+
PENDING = "pending"
|
|
14
|
+
ENQUEUED = "enqueued"
|
|
15
|
+
SCHEDULED = "scheduled"
|
|
16
|
+
DEAD = "dead"
|
|
17
|
+
|
|
18
|
+
scope :pending, -> { where(status: PENDING) }
|
|
19
|
+
scope :enqueued, -> { where(status: ENQUEUED) }
|
|
20
|
+
scope :scheduled, -> { where(status: SCHEDULED) }
|
|
21
|
+
scope :dead, -> { where(status: DEAD) }
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module ReliableJob
|
|
5
|
+
# Fetches pending jobs from the Outbox and pushes them to Redis.
|
|
6
|
+
class OutboxProcessor
|
|
7
|
+
BATCH_SIZE = 1000
|
|
8
|
+
|
|
9
|
+
def call
|
|
10
|
+
Outbox.transaction do
|
|
11
|
+
Outbox.with_advisory_lock!("sidekiq_reliable_job", transaction: true, timeout_seconds: 0) do
|
|
12
|
+
jobs = fetch_pending_jobs
|
|
13
|
+
return 0 if jobs.empty?
|
|
14
|
+
|
|
15
|
+
immediate, scheduled = partition_jobs(jobs)
|
|
16
|
+
|
|
17
|
+
process_immediate_jobs(immediate)
|
|
18
|
+
process_scheduled_jobs(scheduled)
|
|
19
|
+
|
|
20
|
+
jobs.size
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
private
|
|
26
|
+
|
|
27
|
+
def fetch_pending_jobs
|
|
28
|
+
Outbox.pending.order(:id).limit(BATCH_SIZE).pluck(:id, :payload)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def partition_jobs(jobs)
|
|
32
|
+
jobs.partition { |_, payload| payload["at"].blank? }
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def process_immediate_jobs(jobs)
|
|
36
|
+
return if jobs.empty?
|
|
37
|
+
|
|
38
|
+
mark_as_enqueued(jobs)
|
|
39
|
+
push_to_queues(jobs)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def process_scheduled_jobs(jobs)
|
|
43
|
+
return if jobs.empty?
|
|
44
|
+
|
|
45
|
+
mark_as_scheduled(jobs)
|
|
46
|
+
push_to_schedule(jobs)
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def mark_as_enqueued(jobs)
|
|
50
|
+
ids = jobs.map(&:first)
|
|
51
|
+
Outbox.where(id: ids).update_all(status: Outbox::ENQUEUED, enqueued_at: Time.current)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def mark_as_scheduled(jobs)
|
|
55
|
+
ids = jobs.map(&:first)
|
|
56
|
+
Outbox.where(id: ids).update_all(status: Outbox::SCHEDULED, enqueued_at: Time.current)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def push_to_queues(jobs)
|
|
60
|
+
jobs_by_queue = jobs.group_by { |_, payload| payload["queue"] || "default" }
|
|
61
|
+
|
|
62
|
+
Sidekiq.redis do |conn|
|
|
63
|
+
conn.pipelined do |pipeline|
|
|
64
|
+
jobs_by_queue.each do |queue, queue_jobs|
|
|
65
|
+
payloads = queue_jobs.map { |_, payload| Sidekiq.dump_json(payload) }
|
|
66
|
+
pipeline.lpush("queue:#{queue}", payloads)
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def push_to_schedule(jobs)
|
|
73
|
+
Sidekiq.redis do |conn|
|
|
74
|
+
conn.pipelined do |pipeline|
|
|
75
|
+
jobs.each do |(_id, payload)|
|
|
76
|
+
score = payload["at"].to_f
|
|
77
|
+
pipeline.zadd("schedule", score, Sidekiq.dump_json(payload))
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "English"
|
|
4
|
+
|
|
5
|
+
module Sidekiq
|
|
6
|
+
module ReliableJob
|
|
7
|
+
# Deletes staged jobs from the Outbox after successful completion.
|
|
8
|
+
class ServerMiddleware
|
|
9
|
+
include Sidekiq::ServerMiddleware
|
|
10
|
+
|
|
11
|
+
def call(_job_instance, job_payload, _queue)
|
|
12
|
+
yield
|
|
13
|
+
ensure
|
|
14
|
+
if $ERROR_INFO.nil? && job_payload["reliable_job"]
|
|
15
|
+
delete_staged_job(job_payload["jid"])
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
def delete_staged_job(jid)
|
|
22
|
+
Outbox.where(jid: jid).delete_all
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
Sidekiq.logger.error "Failed to delete reliable job #{jid}: #{e.message}"
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq"
|
|
4
|
+
require "sidekiq/job"
|
|
5
|
+
require "with_advisory_lock"
|
|
6
|
+
|
|
7
|
+
require_relative "reliable_job/version"
|
|
8
|
+
require_relative "reliable_job/configuration"
|
|
9
|
+
require_relative "reliable_job/outbox"
|
|
10
|
+
require_relative "reliable_job/client"
|
|
11
|
+
require_relative "reliable_job/client_middleware"
|
|
12
|
+
require_relative "reliable_job/server_middleware"
|
|
13
|
+
require_relative "reliable_job/outbox_processor"
|
|
14
|
+
require_relative "reliable_job/enqueuer"
|
|
15
|
+
|
|
16
|
+
module Sidekiq
|
|
17
|
+
module ReliableJob
|
|
18
|
+
class Error < StandardError; end
|
|
19
|
+
|
|
20
|
+
class << self
|
|
21
|
+
def configure_client!(config)
|
|
22
|
+
config.client_middleware do |chain|
|
|
23
|
+
chain.add Sidekiq::ReliableJob::ClientMiddleware
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def configure_server!(config)
|
|
28
|
+
configure_client!(config)
|
|
29
|
+
|
|
30
|
+
config.server_middleware do |chain|
|
|
31
|
+
chain.add Sidekiq::ReliableJob::ServerMiddleware
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
config.death_handlers << method(:on_death)
|
|
35
|
+
|
|
36
|
+
enqueuer = Enqueuer.new(config)
|
|
37
|
+
|
|
38
|
+
config.on(:startup) do
|
|
39
|
+
enqueuer.start
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
config.on(:quiet) do
|
|
43
|
+
enqueuer.stop
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def on_death(job, _exception)
|
|
48
|
+
return unless job["reliable_job"]
|
|
49
|
+
|
|
50
|
+
if configuration.preserve_dead_jobs
|
|
51
|
+
Outbox.where(jid: job["jid"]).update_all(status: Outbox::DEAD)
|
|
52
|
+
else
|
|
53
|
+
Outbox.where(jid: job["jid"]).delete_all
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: sidekiq-reliable_job
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Zulfiqar Ali
|
|
8
|
+
bindir: exe
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: activerecord
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - ">="
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '7.1'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - ">="
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '7.1'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: sidekiq
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '7.0'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - ">="
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '7.0'
|
|
40
|
+
- !ruby/object:Gem::Dependency
|
|
41
|
+
name: with_advisory_lock
|
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
|
43
|
+
requirements:
|
|
44
|
+
- - ">="
|
|
45
|
+
- !ruby/object:Gem::Version
|
|
46
|
+
version: '5.0'
|
|
47
|
+
type: :runtime
|
|
48
|
+
prerelease: false
|
|
49
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
50
|
+
requirements:
|
|
51
|
+
- - ">="
|
|
52
|
+
- !ruby/object:Gem::Version
|
|
53
|
+
version: '5.0'
|
|
54
|
+
description: A Sidekiq extension that ensures jobs are only enqueued when database
|
|
55
|
+
transactions commit, jobs are deleted when they are completed.
|
|
56
|
+
email:
|
|
57
|
+
- zulfiqar@wealthsimple.com
|
|
58
|
+
executables: []
|
|
59
|
+
extensions: []
|
|
60
|
+
extra_rdoc_files: []
|
|
61
|
+
files:
|
|
62
|
+
- ".ruby-version"
|
|
63
|
+
- CHANGELOG.md
|
|
64
|
+
- LICENSE
|
|
65
|
+
- README.md
|
|
66
|
+
- Rakefile
|
|
67
|
+
- lib/generators/sidekiq/reliable_job/install_generator.rb
|
|
68
|
+
- lib/generators/sidekiq_reliable_job/install/install_generator.rb
|
|
69
|
+
- lib/generators/sidekiq_reliable_job/install/templates/migration.rb.tt
|
|
70
|
+
- lib/sidekiq/reliable_job.rb
|
|
71
|
+
- lib/sidekiq/reliable_job/client.rb
|
|
72
|
+
- lib/sidekiq/reliable_job/client_middleware.rb
|
|
73
|
+
- lib/sidekiq/reliable_job/configuration.rb
|
|
74
|
+
- lib/sidekiq/reliable_job/enqueuer.rb
|
|
75
|
+
- lib/sidekiq/reliable_job/outbox.rb
|
|
76
|
+
- lib/sidekiq/reliable_job/outbox_processor.rb
|
|
77
|
+
- lib/sidekiq/reliable_job/server_middleware.rb
|
|
78
|
+
- lib/sidekiq/reliable_job/version.rb
|
|
79
|
+
homepage: https://github.com/wealthsimple/sidekiq-reliable_job
|
|
80
|
+
licenses: []
|
|
81
|
+
metadata:
|
|
82
|
+
allowed_push_host: https://rubygems.org
|
|
83
|
+
homepage_uri: https://github.com/wealthsimple/sidekiq-reliable_job
|
|
84
|
+
source_code_uri: https://github.com/wealthsimple/sidekiq-reliable_job
|
|
85
|
+
changelog_uri: https://github.com/wealthsimple/sidekiq-reliable_job/blob/main/CHANGELOG.md
|
|
86
|
+
rubygems_mfa_required: 'true'
|
|
87
|
+
rdoc_options: []
|
|
88
|
+
require_paths:
|
|
89
|
+
- lib
|
|
90
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
91
|
+
requirements:
|
|
92
|
+
- - ">="
|
|
93
|
+
- !ruby/object:Gem::Version
|
|
94
|
+
version: 3.3.0
|
|
95
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
96
|
+
requirements:
|
|
97
|
+
- - ">="
|
|
98
|
+
- !ruby/object:Gem::Version
|
|
99
|
+
version: '0'
|
|
100
|
+
requirements: []
|
|
101
|
+
rubygems_version: 3.6.9
|
|
102
|
+
specification_version: 4
|
|
103
|
+
summary: Reliable enqueuing and completion tracking for Sidekiq jobs
|
|
104
|
+
test_files: []
|