que 1.4.1 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/tests.yml +16 -8
- data/.ruby-version +1 -0
- data/CHANGELOG.md +90 -0
- data/Dockerfile +1 -1
- data/README.md +14 -10
- data/bin/command_line_interface.rb +1 -1
- data/docker-compose.yml +3 -2
- data/docs/README.md +150 -56
- data/lib/que/active_job/extensions.rb +46 -29
- data/lib/que/active_record/model.rb +2 -2
- data/lib/que/connection.rb +1 -1
- data/lib/que/job.rb +133 -41
- data/lib/que/job_methods.rb +6 -2
- data/lib/que/migrations/6/down.sql +8 -0
- data/lib/que/migrations/6/up.sql +8 -0
- data/lib/que/migrations/7/down.sql +5 -0
- data/lib/que/migrations/7/up.sql +13 -0
- data/lib/que/migrations.rb +1 -1
- data/lib/que/sequel/model.rb +5 -2
- data/lib/que/utils/ruby2_keywords.rb +19 -0
- data/lib/que/version.rb +2 -2
- data/lib/que/worker.rb +1 -1
- data/lib/que.rb +3 -1
- data/que.gemspec +1 -1
- data/scripts/test +1 -0
- metadata +11 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: '08d50e63c19dbe61966e4048eb1e30fce181e520b05f20f933220baede9cbc76'
|
|
4
|
+
data.tar.gz: 72abbb3390310c18897739e35a2fdb5e6a8187a9b5ae27e4ee651ce4b7c8d9bf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: aa542178d30c1fc031ae2508ff462d0b1ff23f2e5a4e6f9b3fb1cb08ec3ac3a3cc6c167e532654124b5d72c53a319c0b74c3f3a2eb0b5e66102242b43ad59a50
|
|
7
|
+
data.tar.gz: d3902890b359a759f9fe3c94f431afb512e312b6dc46d22223d0d3dc7154f21458a8c7592a72850318094af3c4bfe59950dc8f5bde68e314911695e39d19507a
|
data/.github/workflows/tests.yml
CHANGED
|
@@ -7,11 +7,19 @@ jobs:
|
|
|
7
7
|
runs-on: ubuntu-latest
|
|
8
8
|
strategy:
|
|
9
9
|
matrix:
|
|
10
|
-
ruby_version: [2.
|
|
11
|
-
|
|
12
|
-
postgres_version: [
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
ruby_version: ['2.7', '3.0', '3.1']
|
|
11
|
+
rails_gemfile: ['6.0', '6.1']
|
|
12
|
+
postgres_version: ['14']
|
|
13
|
+
include:
|
|
14
|
+
# Postgres versions
|
|
15
|
+
- { ruby_version: '3.1', rails_gemfile: '6.1', postgres_version: '9' }
|
|
16
|
+
- { ruby_version: '3.1', rails_gemfile: '6.1', postgres_version: '10' }
|
|
17
|
+
- { ruby_version: '3.1', rails_gemfile: '6.1', postgres_version: '11' }
|
|
18
|
+
- { ruby_version: '3.1', rails_gemfile: '6.1', postgres_version: '12' }
|
|
19
|
+
- { ruby_version: '3.1', rails_gemfile: '6.1', postgres_version: '13' }
|
|
20
|
+
- { ruby_version: '3.1', rails_gemfile: '6.1', postgres_version: '14' }
|
|
21
|
+
exclude: []
|
|
22
|
+
name: "Test: Ruby ${{ matrix.ruby_version }}, Rails ${{ matrix.rails_gemfile }}, PostgreSQL ${{ matrix.postgres_version }}"
|
|
15
23
|
services:
|
|
16
24
|
db:
|
|
17
25
|
image: postgres:${{ matrix.postgres_version }}
|
|
@@ -24,16 +32,16 @@ jobs:
|
|
|
24
32
|
--health-timeout 5s
|
|
25
33
|
--health-retries 5
|
|
26
34
|
steps:
|
|
27
|
-
- uses: actions/checkout@
|
|
35
|
+
- uses: actions/checkout@v2
|
|
28
36
|
- name: Set up Ruby
|
|
29
|
-
uses:
|
|
37
|
+
uses: ruby/setup-ruby@v1
|
|
30
38
|
with:
|
|
31
39
|
ruby-version: ${{ matrix.ruby_version }}
|
|
32
40
|
- name: Test with Rake
|
|
33
41
|
env:
|
|
34
42
|
PGHOST: 127.0.0.1
|
|
35
43
|
PGUSER: postgres
|
|
36
|
-
BUNDLE_GEMFILE: spec/gemfiles/Gemfile
|
|
44
|
+
BUNDLE_GEMFILE: spec/gemfiles/Gemfile-rails-${{ matrix.rails_gemfile }}
|
|
37
45
|
run: |
|
|
38
46
|
sudo apt-get -yqq install libpq-dev postgresql-client
|
|
39
47
|
createdb que-test
|
data/.ruby-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.1.1
|
data/CHANGELOG.md
CHANGED
|
@@ -2,7 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
<!-- MarkdownTOC autolink=true -->
|
|
4
4
|
|
|
5
|
+
- [2.1.0 \(2022-08-25\)](#210-2022-08-25)
|
|
6
|
+
- [2.0.0 \(2022-08-25\)](#200-2022-08-25)
|
|
5
7
|
- [1.4.1 \(2022-07-24\)](#141-2022-07-24)
|
|
8
|
+
- [2.0.0.beta1 \(2022-03-24\)](#200beta1-2022-03-24)
|
|
6
9
|
- [1.4.0 \(2022-03-23\)](#140-2022-03-23)
|
|
7
10
|
- [1.3.1 \(2022-02-25\)](#131-2022-02-25)
|
|
8
11
|
- [1.3.0 \(2022-02-25\)](#130-2022-02-25)
|
|
@@ -52,11 +55,98 @@
|
|
|
52
55
|
|
|
53
56
|
<!-- /MarkdownTOC -->
|
|
54
57
|
|
|
58
|
+
## 2.1.0 (2022-08-25)
|
|
59
|
+
|
|
60
|
+
- **Added**:
|
|
61
|
+
+ Added bulk enqueue interface for performance when enqueuing a large number of jobs at once - [docs](docs#enqueueing-jobs-in-bulk).
|
|
62
|
+
- **Deprecated**:
|
|
63
|
+
+ Deprecated `que_state_notify` trigger (`que_state` notification channel / `job_change` notification message). See [#372](https://github.com/que-rb/que/issues/372). We plan to remove this in a future release - let us know on the issue if you desire otherwise.
|
|
64
|
+
|
|
65
|
+
This release contains a database migration. You will need to migrate Que to the latest database schema version (7). For example, on ActiveRecord and Rails 6:
|
|
66
|
+
|
|
67
|
+
```ruby
|
|
68
|
+
class UpdateQueTablesToVersion6 < ActiveRecord::Migration[6.0]
|
|
69
|
+
def up
|
|
70
|
+
Que.migrate!(version: 7)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def down
|
|
74
|
+
Que.migrate!(version: 6)
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
## 2.0.0 (2022-08-25)
|
|
80
|
+
|
|
81
|
+
**Important: Do not upgrade straight to Que 2.** You will need to first update to the latest 1.x version, apply the Que database schema migration, and deploy, before you can safely begin the process of upgrading to Que 2. See the [2.0.0.beta1 changelog entry](#200beta1-2022-03-24) for details.
|
|
82
|
+
|
|
83
|
+
See beta 2.0.0.beta1, plus:
|
|
84
|
+
|
|
85
|
+
- **Fixed**:
|
|
86
|
+
+ Updated to use non-deprecated method from PG when params are passed (`#async_exec_params`). [#374](https://github.com/que-rb/que/pull/374)
|
|
87
|
+
|
|
88
|
+
Note that @dtcristo submitted a PR proposing an easier upgrade path to Que 2 and Ruby 3 - [#365](https://github.com/que-rb/que/pull/365). We are yet to properly consider this, but a later release which includes this feature would mean you don't need to simultaneously deploy Que 1.x and 2.x workers during the upgrade.
|
|
89
|
+
|
|
55
90
|
## 1.4.1 (2022-07-24)
|
|
56
91
|
|
|
57
92
|
- **Added**
|
|
58
93
|
+ Added Ruby version requirement of < 3. For Ruby 3 compatibility, upgrade to Que 2 - [upgrade process](https://github.com/que-rb/que/blob/master/CHANGELOG.md#200beta1-2022-03-24)
|
|
59
94
|
|
|
95
|
+
## 2.0.0.beta1 (2022-03-24)
|
|
96
|
+
|
|
97
|
+
**Preliminary release of Ruby 3 support**
|
|
98
|
+
|
|
99
|
+
**Notable changes**:
|
|
100
|
+
|
|
101
|
+
* Support for Ruby 3 introduced
|
|
102
|
+
* Database schema has changed to split the job arguments `args` column into `args` and `kwargs` columns, for reliable args and kwargs splitting for Ruby 3.
|
|
103
|
+
- The job schema version is now 2. Note that job schema version is distinct from database schema version and Que version. The `job_schema_version` column of the `que_jobs` table no longer defaults and has a not null constraint, so when manually inserting jobs into the table, this must be specified as `2`. If you have a gem that needs to support multiple Que versions, best not to blindly use the value of `Que.job_schema_version`; instead have different code paths depending on the value of `Que.job_schema_version`. You could also use this to know whether keyword arguments are in `args` or `kwargs`.
|
|
104
|
+
* Passing a hash literal as the last job argument to be splatted into job keyword arguments is no longer supported.
|
|
105
|
+
* Dropped support for providing job options as top-level keyword arguments to `Job.enqueue`, i.e. `queue`, `priority`, `run_at`, `job_class`, and `tags`. Job options now need to be nested under the `job_options` keyword argument instead. See [#336](https://github.com/que-rb/que/pull/336)
|
|
106
|
+
* Dropped support for Ruby < 2.7
|
|
107
|
+
* Dropped support for Rails < 6.0
|
|
108
|
+
* The `#by_args` method on the Job model (for both Sequel and ActiveRecord) now searches based on both args and kwargs, but it performs a subset match instead of an exact match. For instance, if your job was scheduled with `'a', 'b', 'c', foo: 'bar', baz: 1`, `by_args('a', 'b', baz: 1)` would find and return the job.
|
|
109
|
+
* This release contains a database migration. You will need to migrate Que to the latest database schema version (6). For example, on ActiveRecord and Rails 6:
|
|
110
|
+
|
|
111
|
+
```ruby
|
|
112
|
+
class UpdateQueTablesToVersion6 < ActiveRecord::Migration[6.0]
|
|
113
|
+
def up
|
|
114
|
+
Que.migrate!(version: 6)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def down
|
|
118
|
+
Que.migrate!(version: 5)
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
**Recommended upgrade process**:
|
|
124
|
+
|
|
125
|
+
When using Que 2.x, a job enqueued with Ruby 2.7 will run as expected on Ruby 3. We recommend:
|
|
126
|
+
|
|
127
|
+
1. Upgrade your project to the latest 1.x version of Que (1.3.1+)
|
|
128
|
+
- IMPORTANT: adds support for zero downtime upgrade to Que 2.x, see changelog below
|
|
129
|
+
2. Upgrade your project to Ruby 2.7 and Rails 6.x if it is not already
|
|
130
|
+
3. Upgrade your project to Que 2.x but stay on Ruby 2.7
|
|
131
|
+
- IMPORTANT: You will need to continue to run Que 1.x workers until all jobs enqueued using Que 1.x (i.e. with a `job_schema_version` of `1`) have been finished. See below
|
|
132
|
+
4. Upgrade your project to Ruby 3
|
|
133
|
+
|
|
134
|
+
*NOTES:*
|
|
135
|
+
|
|
136
|
+
* If you were already running Ruby 2.7 and were not passing a hash literal as the last job argument, you *may* be able to upgrade a running system without draining the queue, though this is not recommended.
|
|
137
|
+
* For all other cases, you will need to follow the recommended process above or first completely drain the queue (stop enqueuing new jobs and finish processing any jobs in the database, including cleaning out any expired jobs) before upgrading.
|
|
138
|
+
|
|
139
|
+
**Deploying Que 1.x and 2.x workers simultaneously**:
|
|
140
|
+
|
|
141
|
+
To run workers with two different versions of Que, you'll probably need to temporarily duplicate your gem bundle, with the Que version being the only difference. e.g.:
|
|
142
|
+
|
|
143
|
+
- Copy your `Gemfile` and `Gemfile.lock` into a directory called `que-1-gemfile`
|
|
144
|
+
- Set a suitable Que version in each `Gemfile`
|
|
145
|
+
- Update the bundle at `que-1-gemfile/Gemfile.lock` using `BUNDLE_GEMFILE=que-1-gemfile/Gemfile bundle`
|
|
146
|
+
- Create a second deployment of Que, but with your `que` command prefixed with `BUNDLE_GEMFILE=que-1-gemfile/Gemfile`
|
|
147
|
+
|
|
148
|
+
We'd appreciate feedback on your experience upgrading to and running Que 2. Feel free to post on our Discord, or if you run into trouble, open an issue on GitHub.
|
|
149
|
+
|
|
60
150
|
## 1.4.0 (2022-03-23)
|
|
61
151
|
|
|
62
152
|
- **Fixed**
|
data/Dockerfile
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
FROM ruby:
|
|
1
|
+
FROM ruby:3.1.1-slim-buster@sha256:2ada3e4fe7b1703c9333ad4eb9fc12c1d4d60bce0f981281b2151057e928d9ad AS base
|
|
2
2
|
|
|
3
3
|
# Install libpq-dev in our base layer, as it's needed in all environments
|
|
4
4
|
RUN apt-get update \
|
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Que 
|
|
2
2
|
|
|
3
|
-
**This README and the rest of the docs on the master branch all refer to Que
|
|
3
|
+
**This README and the rest of the docs on the master branch all refer to Que 2.x. For older versions, please refer to the docs on the respective branches: [1.x](https://github.com/que-rb/que/tree/1.x), or [0.x](https://github.com/que-rb/que/tree/0.x).**
|
|
4
4
|
|
|
5
5
|
*TL;DR: Que is a high-performance job queue that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.*
|
|
6
6
|
|
|
@@ -23,9 +23,9 @@ Que's secondary goal is performance. The worker process is multithreaded, so tha
|
|
|
23
23
|
|
|
24
24
|
Compatibility:
|
|
25
25
|
|
|
26
|
-
- MRI Ruby 2.
|
|
26
|
+
- MRI Ruby 2.7+ (for Ruby 3, Que 2+ is required)
|
|
27
27
|
- PostgreSQL 9.5+
|
|
28
|
-
- Rails
|
|
28
|
+
- Rails 6.0+ (optional)
|
|
29
29
|
|
|
30
30
|
**Please note** - Que's job table undergoes a lot of churn when it is under high load, and like any heavily-written table, is susceptible to bloat and slowness if Postgres isn't able to clean it up. The most common cause of this is long-running transactions, so it's recommended to try to keep all transactions against the database housing Que's job table as short as possible. This is good advice to remember for any high-activity database, but bears emphasizing when using tables that undergo a lot of writes.
|
|
31
31
|
|
|
@@ -54,12 +54,12 @@ gem install que
|
|
|
54
54
|
First, create the queue schema in a migration. For example:
|
|
55
55
|
|
|
56
56
|
```ruby
|
|
57
|
-
class CreateQueSchema < ActiveRecord::Migration[
|
|
57
|
+
class CreateQueSchema < ActiveRecord::Migration[6.0]
|
|
58
58
|
def up
|
|
59
59
|
# Whenever you use Que in a migration, always specify the version you're
|
|
60
60
|
# migrating to. If you're unsure what the current version is, check the
|
|
61
61
|
# changelog.
|
|
62
|
-
Que.migrate!(version:
|
|
62
|
+
Que.migrate!(version: 6)
|
|
63
63
|
end
|
|
64
64
|
|
|
65
65
|
def down
|
|
@@ -117,10 +117,14 @@ end
|
|
|
117
117
|
You can also add options to run the job after a specific time, or with a specific priority:
|
|
118
118
|
|
|
119
119
|
```ruby
|
|
120
|
-
ChargeCreditCard.enqueue
|
|
120
|
+
ChargeCreditCard.enqueue(card.id, user_id: current_user.id, job_options: { run_at: 1.day.from_now, priority: 5 })
|
|
121
121
|
```
|
|
122
|
+
|
|
123
|
+
[Learn more about job options](docs/README.md#job-options).
|
|
124
|
+
|
|
122
125
|
## Running the Que Worker
|
|
123
|
-
|
|
126
|
+
|
|
127
|
+
In order to process jobs, you must start a separate worker process outside of your main server.
|
|
124
128
|
|
|
125
129
|
```bash
|
|
126
130
|
bundle exec que
|
|
@@ -142,7 +146,7 @@ You may need to pass que a file path to require so that it can load your app. Qu
|
|
|
142
146
|
|
|
143
147
|
If you're using ActiveRecord to dump your database's schema, please [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly. This is a good idea regardless, as the `:ruby` schema format doesn't support many of PostgreSQL's advanced features.
|
|
144
148
|
|
|
145
|
-
Pre-1.0, the default queue name needed to be configured in order for Que to work out of the box with Rails.
|
|
149
|
+
Pre-1.0, the default queue name needed to be configured in order for Que to work out of the box with Rails. As of 1.0 the default queue name is now 'default', as Rails expects, but when Rails enqueues some types of jobs it may try to use another queue name that isn't worked by default. You can either:
|
|
146
150
|
|
|
147
151
|
- [Configure Rails](https://guides.rubyonrails.org/configuring.html) to send all internal job types to the 'default' queue by adding the following to `config/application.rb`:
|
|
148
152
|
|
|
@@ -192,11 +196,11 @@ If you have a project that uses or relates to Que, feel free to submit a PR addi
|
|
|
192
196
|
|
|
193
197
|
## Community and Contributing
|
|
194
198
|
|
|
195
|
-
- For bugs in the library, please feel free to [open an issue](https://github.com/que-rb/que/issues/new).
|
|
199
|
+
- For feature suggestions or bugs in the library, please feel free to [open an issue](https://github.com/que-rb/que/issues/new).
|
|
196
200
|
- For general discussion and questions/concerns that don't relate to obvious bugs, join our [Discord Server](https://discord.gg/B3EW32H).
|
|
197
201
|
- For contributions, pull requests submitted via Github are welcome.
|
|
198
202
|
|
|
199
|
-
Regarding contributions, one of the project's priorities is to keep Que as simple, lightweight and dependency-free as possible, and pull requests that change too much or wouldn't be useful to the majority of Que's users have a good chance of being rejected. If you're thinking of submitting a pull request that adds a new feature, consider starting a discussion in
|
|
203
|
+
Regarding contributions, one of the project's priorities is to keep Que as simple, lightweight and dependency-free as possible, and pull requests that change too much or wouldn't be useful to the majority of Que's users have a good chance of being rejected. If you're thinking of submitting a pull request that adds a new feature, consider starting a discussion in an issue first about what it would do and how it would be implemented. If it's a sufficiently large feature, or if most of Que's users wouldn't find it useful, it may be best implemented as a standalone gem, like some of the related projects above.
|
|
200
204
|
|
|
201
205
|
### Specs
|
|
202
206
|
|
data/docker-compose.yml
CHANGED
|
@@ -10,13 +10,14 @@ services:
|
|
|
10
10
|
- db
|
|
11
11
|
volumes:
|
|
12
12
|
- .:/work
|
|
13
|
-
- ruby-
|
|
13
|
+
- ruby-3.1.1-gem-cache:/usr/local/bundle
|
|
14
14
|
- ~/.docker-rc.d/:/.docker-rc.d/:ro
|
|
15
15
|
working_dir: /work
|
|
16
16
|
entrypoint: /work/scripts/docker-entrypoint
|
|
17
17
|
command: bash
|
|
18
18
|
environment:
|
|
19
19
|
DATABASE_URL: postgres://que:que@db/que-test
|
|
20
|
+
USE_RAILS: ~
|
|
20
21
|
|
|
21
22
|
db:
|
|
22
23
|
image: "postgres:${POSTGRES_VERSION-13}"
|
|
@@ -43,4 +44,4 @@ services:
|
|
|
43
44
|
|
|
44
45
|
volumes:
|
|
45
46
|
db-data: ~
|
|
46
|
-
ruby-
|
|
47
|
+
ruby-3.1.1-gem-cache: ~
|
data/docs/README.md
CHANGED
|
@@ -1,52 +1,63 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
# Que documentation
|
|
2
|
+
|
|
3
|
+
<!-- MarkdownTOC autolink=true -->
|
|
3
4
|
|
|
4
5
|
- [Command Line Interface](#command-line-interface)
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
6
|
+
- [`worker-priorities` and `worker-count`](#worker-priorities-and-worker-count)
|
|
7
|
+
- [`poll-interval`](#poll-interval)
|
|
8
|
+
- [`maximum-buffer-size`](#maximum-buffer-size)
|
|
9
|
+
- [`connection-url`](#connection-url)
|
|
10
|
+
- [`wait-period`](#wait-period)
|
|
11
|
+
- [`log-internals`](#log-internals)
|
|
11
12
|
- [Advanced Setup](#advanced-setup)
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
- [Using ActiveRecord Without Rails](#using-activerecord-without-rails)
|
|
14
|
+
- [Managing the Jobs Table](#managing-the-jobs-table)
|
|
15
|
+
- [Other Setup](#other-setup)
|
|
15
16
|
- [Error Handling](#error-handling)
|
|
16
|
-
|
|
17
|
-
|
|
17
|
+
- [Error Notifications](#error-notifications)
|
|
18
|
+
- [Error-Specific Handling](#error-specific-handling)
|
|
18
19
|
- [Inspecting the Queue](#inspecting-the-queue)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
20
|
+
- [Job Stats](#job-stats)
|
|
21
|
+
- [Custom Queries](#custom-queries)
|
|
22
|
+
- [ActiveRecord Example](#activerecord-example)
|
|
23
|
+
- [Sequel Example](#sequel-example)
|
|
23
24
|
- [Managing Workers](#managing-workers)
|
|
24
|
-
|
|
25
|
-
|
|
25
|
+
- [Working Jobs Via Executable](#working-jobs-via-executable)
|
|
26
|
+
- [Thread-Unsafe Application Code](#thread-unsafe-application-code)
|
|
26
27
|
- [Logging](#logging)
|
|
27
|
-
|
|
28
|
+
- [Logging Job Completion](#logging-job-completion)
|
|
28
29
|
- [Migrating](#migrating)
|
|
29
30
|
- [Multiple Queues](#multiple-queues)
|
|
30
31
|
- [Shutting Down Safely](#shutting-down-safely)
|
|
31
32
|
- [Using Plain Postgres Connections](#using-plain-postgres-connections)
|
|
32
|
-
|
|
33
|
-
|
|
33
|
+
- [Using ConnectionPool or Pond](#using-connectionpool-or-pond)
|
|
34
|
+
- [Using Any Other Connection Pool](#using-any-other-connection-pool)
|
|
34
35
|
- [Using Sequel](#using-sequel)
|
|
35
36
|
- [Using Que With ActiveJob](#using-que-with-activejob)
|
|
36
37
|
- [Job Helper Methods](#job-helper-methods)
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
- [`destroy`](#destroy)
|
|
39
|
+
- [`finish`](#finish)
|
|
40
|
+
- [`expire`](#expire)
|
|
41
|
+
- [`retry_in`](#retry_in)
|
|
42
|
+
- [`error_count`](#error_count)
|
|
43
|
+
- [`default_resolve_action`](#default_resolve_action)
|
|
43
44
|
- [Writing Reliable Jobs](#writing-reliable-jobs)
|
|
44
|
-
|
|
45
|
+
- [Timeouts](#timeouts)
|
|
46
|
+
- [Job Options](#job-options)
|
|
47
|
+
- [`queue`](#queue)
|
|
48
|
+
- [`priority`](#priority)
|
|
49
|
+
- [`run_at`](#run_at)
|
|
50
|
+
- [`job_class`](#job_class)
|
|
51
|
+
- [`tags`](#tags)
|
|
45
52
|
- [Middleware](#middleware)
|
|
46
|
-
|
|
47
|
-
|
|
53
|
+
- [Defining Middleware For Jobs](#defining-middleware-for-jobs)
|
|
54
|
+
- [Defining Middleware For SQL statements](#defining-middleware-for-sql-statements)
|
|
48
55
|
- [Vacuuming](#vacuuming)
|
|
56
|
+
- [Enqueueing jobs in bulk](#enqueueing-jobs-in-bulk)
|
|
57
|
+
- [Expired jobs](#expired-jobs)
|
|
58
|
+
- [Finished jobs](#finished-jobs)
|
|
49
59
|
|
|
60
|
+
<!-- /MarkdownTOC -->
|
|
50
61
|
|
|
51
62
|
## Command Line Interface
|
|
52
63
|
|
|
@@ -67,7 +78,7 @@ usage: que [options] [file/to/require] ...
|
|
|
67
78
|
|
|
68
79
|
Some explanation of the more unusual options:
|
|
69
80
|
|
|
70
|
-
### worker-priorities and worker-count
|
|
81
|
+
### `worker-priorities` and `worker-count`
|
|
71
82
|
|
|
72
83
|
These options dictate the size and priority distribution of the worker pool. The default worker-priorities is `10,30,50,any,any,any`. This means that the default worker pool will reserve one worker to only works jobs with priorities under 10, one for priorities under 30, and one for priorities under 50. Three more workers will work any job.
|
|
73
84
|
|
|
@@ -77,23 +88,23 @@ Instead of passing worker-priorities, you can pass a `worker-count` - this is a
|
|
|
77
88
|
|
|
78
89
|
If you pass both worker-count and worker-priorities, the count will trim or pad the priorities list with `any` workers. So, `--worker-priorities=20,30,40 --worker-count=6` would be the same as passing `--worker-priorities=20,30,40,any,any,any`.
|
|
79
90
|
|
|
80
|
-
### poll-interval
|
|
91
|
+
### `poll-interval`
|
|
81
92
|
|
|
82
93
|
This option sets the number of seconds the process will wait between polls of the job queue. Jobs that are ready to be worked immediately will be broadcast via the LISTEN/NOTIFY system, so polling is unnecessary for them - polling is only necessary for jobs that are scheduled in the future or which are being delayed due to errors. The default is 5 seconds.
|
|
83
94
|
|
|
84
|
-
### maximum-buffer-size
|
|
95
|
+
### `maximum-buffer-size`
|
|
85
96
|
|
|
86
97
|
This option sets the size of the internal buffer that Que uses to hold jobs until they're ready for workers. The default maximum is 8, meaning that the process won't buffer more than 8 jobs that aren't yet ready to be worked. If you don't want jobs to be buffered at all, you can set this value to zero.
|
|
87
98
|
|
|
88
|
-
### connection-url
|
|
99
|
+
### `connection-url`
|
|
89
100
|
|
|
90
101
|
This option sets the URL to be used to open a connection to the database for locking purposes. By default, Que will simply use a connection from the connection pool for locking - this option is only useful if your application connections can't use advisory locks - for example, if they're passed through an external connection pool like PgBouncer. In that case, you'll need to use this option to specify your actual database URL so that Que can establish a direct connection.
|
|
91
102
|
|
|
92
|
-
### wait-period
|
|
103
|
+
### `wait-period`
|
|
93
104
|
|
|
94
105
|
This option specifies (in milliseconds) how often the locking thread wakes up to check whether the workers have finished jobs, whether it's time to poll, etc. You shouldn't generally need to tweak this, but it may come in handy for some workloads. The default is 50 milliseconds.
|
|
95
106
|
|
|
96
|
-
### log-internals
|
|
107
|
+
### `log-internals`
|
|
97
108
|
|
|
98
109
|
This option instructs Que to output a lot of information about its internal state to the logger. It should only be used if it becomes necessary to debug issues.
|
|
99
110
|
|
|
@@ -128,8 +139,8 @@ There are other docs to read if you're using [Sequel](#using-sequel) or [plain P
|
|
|
128
139
|
After you've connected Que to the database, you can manage the jobs table. You'll want to migrate to a specific version in a migration file, to ensure that they work the same way even when you upgrade Que in the future:
|
|
129
140
|
|
|
130
141
|
```ruby
|
|
131
|
-
# Update the schema to version #
|
|
132
|
-
Que.migrate!(version:
|
|
142
|
+
# Update the schema to version #6.
|
|
143
|
+
Que.migrate!(version: 6)
|
|
133
144
|
|
|
134
145
|
# Remove Que's jobs table entirely.
|
|
135
146
|
Que.migrate!(version: 0)
|
|
@@ -147,7 +158,6 @@ Be sure to read the docs on [managing workers](#managing-workers) for more infor
|
|
|
147
158
|
|
|
148
159
|
You'll also want to set up [logging](#logging) and an [error handler](#error-handling) to track errors raised by jobs.
|
|
149
160
|
|
|
150
|
-
|
|
151
161
|
## Error Handling
|
|
152
162
|
|
|
153
163
|
If an error is raised and left uncaught by your job, Que will save the error message and backtrace to the database and schedule the job to be retried later.
|
|
@@ -425,7 +435,6 @@ Que.db_version #=> 3
|
|
|
425
435
|
|
|
426
436
|
Note that you can remove Que from your database completely by migrating to version 0.
|
|
427
437
|
|
|
428
|
-
|
|
429
438
|
## Multiple Queues
|
|
430
439
|
|
|
431
440
|
Que supports the use of multiple queues in a single job table. Please note that this feature is intended to support the case where multiple codebases are sharing the same job queue - if you want to support jobs of differing priorities, the numeric priority system offers better flexibility and performance.
|
|
@@ -441,7 +450,7 @@ que -q default -q credit_cards
|
|
|
441
450
|
Then you can set jobs to be enqueued in that queue specifically:
|
|
442
451
|
|
|
443
452
|
```ruby
|
|
444
|
-
ProcessCreditCard.enqueue
|
|
453
|
+
ProcessCreditCard.enqueue(current_user.id, job_options: { queue: 'credit_cards' })
|
|
445
454
|
|
|
446
455
|
# Or:
|
|
447
456
|
|
|
@@ -452,11 +461,7 @@ class ProcessCreditCard < Que::Job
|
|
|
452
461
|
end
|
|
453
462
|
```
|
|
454
463
|
|
|
455
|
-
In some cases, the ProcessCreditCard class may not be defined in the application that is enqueueing the job. In that case, you can specify the job class as a string
|
|
456
|
-
|
|
457
|
-
```ruby
|
|
458
|
-
Que.enqueue current_user.id, job_class: 'ProcessCreditCard', queue: 'credit_cards'
|
|
459
|
-
```
|
|
464
|
+
In some cases, the `ProcessCreditCard` class may not be defined in the application that is enqueueing the job. In that case, you can [specify the job class as a string](#job_class).
|
|
460
465
|
|
|
461
466
|
## Shutting Down Safely
|
|
462
467
|
|
|
@@ -466,7 +471,6 @@ To prevent this, Que will block the worker process from exiting until all jobs i
|
|
|
466
471
|
|
|
467
472
|
So, be prepared to use SIGKILL on your Ruby processes if they run for too long. For example, Heroku takes a good approach to this - when Heroku's platform is shutting down a process, it sends SIGTERM, waits ten seconds, then sends SIGKILL if the process still hasn't exited. This is a nice compromise - it will give each of your currently running jobs ten seconds to complete, and any jobs that haven't finished by then will be interrupted and retried later.
|
|
468
473
|
|
|
469
|
-
|
|
470
474
|
## Using Plain Postgres Connections
|
|
471
475
|
|
|
472
476
|
If you're not using an ORM like ActiveRecord or Sequel, you can use a distinct connection pool to manage your Postgres connections. Please be aware that if you **are** using ActiveRecord or Sequel, there's no reason for you to be using any of these methods - it's less efficient (unnecessary connections will waste memory on your database server) and you lose the reliability benefits of wrapping jobs in the same transactions as the rest of your data.
|
|
@@ -549,7 +553,7 @@ require 'que'
|
|
|
549
553
|
Sequel.migration do
|
|
550
554
|
up do
|
|
551
555
|
Que.connection = self
|
|
552
|
-
Que.migrate!(version:
|
|
556
|
+
Que.migrate!(version: 6)
|
|
553
557
|
end
|
|
554
558
|
down do
|
|
555
559
|
Que.connection = self
|
|
@@ -585,7 +589,7 @@ Sequel automatically wraps model persistance actions (create, update, destroy) i
|
|
|
585
589
|
|
|
586
590
|
## Using Que With ActiveJob
|
|
587
591
|
|
|
588
|
-
You can include `Que::ActiveJob::JobExtensions` into your `ApplicationJob` subclass to get support for all of Que's
|
|
592
|
+
You can include `Que::ActiveJob::JobExtensions` into your `ApplicationJob` subclass to get support for all of Que's
|
|
589
593
|
[helper methods](#job-helper-methods). These methods will become no-ops if you use a queue adapter that isn't Que, so if you like to use a different adapter in development they shouldn't interfere.
|
|
590
594
|
|
|
591
595
|
Additionally, including `Que::ActiveJob::JobExtensions` lets you define a run() method that supports keyword arguments.
|
|
@@ -594,27 +598,29 @@ Additionally, including `Que::ActiveJob::JobExtensions` lets you define a run()
|
|
|
594
598
|
|
|
595
599
|
There are a number of instance methods on Que::Job that you can use in your jobs, preferably in transactions. See [Writing Reliable Jobs](#writing-reliable-jobs) for more information on where to use these methods.
|
|
596
600
|
|
|
597
|
-
### destroy
|
|
601
|
+
### `destroy`
|
|
598
602
|
|
|
599
603
|
This method deletes the job from the queue table, ensuring that it won't be worked a second time.
|
|
600
604
|
|
|
601
|
-
### finish
|
|
605
|
+
### `finish`
|
|
602
606
|
|
|
603
607
|
This method marks the current job as finished, ensuring that it won't be worked a second time. This is like destroy, in that it finalizes a job, but this method leaves the job in the table, in case you want to query it later.
|
|
604
608
|
|
|
605
|
-
### expire
|
|
609
|
+
### `expire`
|
|
606
610
|
|
|
607
611
|
This method marks the current job as expired. It will be left in the table and won't be retried, but it will be easy to query for expired jobs. This method is called if the job exceeds its maximum_retry_count.
|
|
608
612
|
|
|
609
|
-
### retry_in
|
|
613
|
+
### `retry_in`
|
|
610
614
|
|
|
611
615
|
This method marks the current job to be retried later. You can pass a numeric to this method, in which case that is the number of seconds after which it can be retried (`retry_in(10)`, `retry_in(0.5)`), or, if you're using ActiveSupport, you can pass in a duration object (`retry_in(10.minutes)`). This automatically happens, with an exponentially-increasing interval, when the job encounters an error.
|
|
612
616
|
|
|
613
|
-
|
|
617
|
+
Note that `retry_in` increments the job's `error_count`.
|
|
618
|
+
|
|
619
|
+
### `error_count`
|
|
614
620
|
|
|
615
621
|
This method returns the total number of times the job has errored, in case you want to modify the job's behavior after it has failed a given number of times.
|
|
616
622
|
|
|
617
|
-
### default_resolve_action
|
|
623
|
+
### `default_resolve_action`
|
|
618
624
|
|
|
619
625
|
If you don't perform a resolve action (destroy, finish, expire, retry_in) while the job is worked, Que will call this method for you. By default it simply calls `destroy`, but you can override it in your Job subclasses if you wish - for example, to call `finish`, or to invoke some more complicated logic.
|
|
620
626
|
|
|
@@ -727,6 +733,51 @@ end
|
|
|
727
733
|
|
|
728
734
|
Now, if the request takes more than five seconds, an error will be raised (probably - check your library's documentation) and Que will just retry the job later.
|
|
729
735
|
|
|
736
|
+
## Job Options
|
|
737
|
+
|
|
738
|
+
When enqueueing a job, you can specify particular options for it in a `job_options` hash, e.g.:
|
|
739
|
+
|
|
740
|
+
```ruby
|
|
741
|
+
ChargeCreditCard.enqueue(card.id, user_id: current_user.id, job_options: { run_at: 1.day.from_now, priority: 5 })
|
|
742
|
+
```
|
|
743
|
+
|
|
744
|
+
### `queue`
|
|
745
|
+
|
|
746
|
+
See [Multiple Queues](#multiple-queues).
|
|
747
|
+
|
|
748
|
+
### `priority`
|
|
749
|
+
|
|
750
|
+
Provide an integer to customise the priority level of the job.
|
|
751
|
+
|
|
752
|
+
We use the Linux priority scale - a lower number is more important.
|
|
753
|
+
|
|
754
|
+
### `run_at`
|
|
755
|
+
|
|
756
|
+
Provide a `Time` as the `run_at` to make a job run at a later time (well, at some point after it, depending on how busy the workers are).
|
|
757
|
+
|
|
758
|
+
It's best not to use `Time.now` here, as the current time in the Ruby process and the database won't be perfectly aligned. When the database considers the `run_at` to be in the past, the job will not be broadcast via the LISTEN/NOTIFY system, and it will need to wait for a poll. This introduces an unnecessary delay of probably a few seconds (depending on your configured [poll interval](#poll-interval)). So if you want the job to run ASAP, just omit the `run_at` option.
|
|
759
|
+
|
|
760
|
+
### `job_class`
|
|
761
|
+
|
|
762
|
+
Specifying `job_class` allows you to enqueue a job using `Que.enqueue`:
|
|
763
|
+
|
|
764
|
+
```ruby
|
|
765
|
+
Que.enqueue(current_user.id, job_options: { job_class: 'ProcessCreditCard' })
|
|
766
|
+
```
|
|
767
|
+
|
|
768
|
+
Rather than needing to use the job class (nor even have it defined in the enqueueing process):
|
|
769
|
+
|
|
770
|
+
```ruby
|
|
771
|
+
ProcessCreditCard.enqueue(current_user.id)
|
|
772
|
+
```
|
|
773
|
+
|
|
774
|
+
### `tags`
|
|
775
|
+
|
|
776
|
+
You can provide an array of strings to give a job some tags. These are not used by Que and are completely custom.
|
|
777
|
+
|
|
778
|
+
A job can have up to five tags, each one up to 100 characters long.
|
|
779
|
+
|
|
780
|
+
Note that unlike the other job options, tags are stored within the `que_jobs.data` column, rather than a correspondingly-named column.
|
|
730
781
|
|
|
731
782
|
## Middleware
|
|
732
783
|
|
|
@@ -785,3 +836,46 @@ class ManualVacuumJob < CronJob
|
|
|
785
836
|
end
|
|
786
837
|
end
|
|
787
838
|
```
|
|
839
|
+
|
|
840
|
+
## Enqueueing jobs in bulk
|
|
841
|
+
|
|
842
|
+
If you need to enqueue a large number of jobs at once, enqueueing each one separately (and running the notify trigger for each) can become a performance bottleneck. To mitigate this, there is a bulk enqueue interface:
|
|
843
|
+
|
|
844
|
+
```ruby
|
|
845
|
+
Que.bulk_enqueue do
|
|
846
|
+
MyJob.enqueue(user_id: 1)
|
|
847
|
+
MyJob.enqueue(user_id: 2)
|
|
848
|
+
# ...
|
|
849
|
+
end
|
|
850
|
+
```
|
|
851
|
+
|
|
852
|
+
The jobs are only actually enqueued at the end of the block, at which point they are inserted into the database in one big query.
|
|
853
|
+
|
|
854
|
+
Limitations:
|
|
855
|
+
|
|
856
|
+
- ActiveJob is not supported
|
|
857
|
+
- All jobs must use the same job class
|
|
858
|
+
- All jobs must use the same `job_options` (`job_options` must be provided to `.bulk_enqueue` instead of `.enqueue`)
|
|
859
|
+
- The `que_attrs` of a job instance returned from `.enqueue` is empty (`{}`)
|
|
860
|
+
- The notify trigger is not run by default, so jobs will only be picked up by a worker upon its next poll
|
|
861
|
+
|
|
862
|
+
If you still want the notify trigger to run for each job, use `Que.bulk_enqueue(notify: true) { ... }`.
|
|
863
|
+
|
|
864
|
+
## Expired jobs
|
|
865
|
+
|
|
866
|
+
Expired jobs hang around in the `que_jobs` table. If necessary, you can get an expired job to run again by clearing the `error_count` and `expired_at` columns, e.g.:
|
|
867
|
+
|
|
868
|
+
```sql
|
|
869
|
+
UPDATE que_jobs SET error_count = 0, expired_at = NULL WHERE id = 172340879;
|
|
870
|
+
```
|
|
871
|
+
|
|
872
|
+
## Finished jobs
|
|
873
|
+
|
|
874
|
+
If you prefer to leave finished jobs in the database for a while, to performantly remove them periodically, you can use something like:
|
|
875
|
+
|
|
876
|
+
```sql
|
|
877
|
+
BEGIN;
|
|
878
|
+
SET LOCAL que.skip_notify TO true;
|
|
879
|
+
DELETE FROM que_jobs WHERE finished_at < (select now() - interval '7 days');
|
|
880
|
+
COMMIT;
|
|
881
|
+
```
|
|
@@ -12,8 +12,10 @@ module Que
|
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
def perform(*args)
|
|
15
|
+
args, kwargs = Que.split_out_ruby2_keywords(args)
|
|
16
|
+
|
|
15
17
|
Que.internal_log(:active_job_perform, self) do
|
|
16
|
-
{args: args}
|
|
18
|
+
{args: args, kwargs: kwargs}
|
|
17
19
|
end
|
|
18
20
|
|
|
19
21
|
_run(
|
|
@@ -21,7 +23,12 @@ module Que
|
|
|
21
23
|
que_filter_args(
|
|
22
24
|
args.map { |a| a.is_a?(Hash) ? a.deep_symbolize_keys : a }
|
|
23
25
|
)
|
|
24
|
-
)
|
|
26
|
+
),
|
|
27
|
+
kwargs: Que.recursively_freeze(
|
|
28
|
+
que_filter_args(
|
|
29
|
+
kwargs.deep_symbolize_keys,
|
|
30
|
+
)
|
|
31
|
+
),
|
|
25
32
|
)
|
|
26
33
|
end
|
|
27
34
|
|
|
@@ -53,37 +60,46 @@ module Que
|
|
|
53
60
|
# A module that we mix into ActiveJob's wrapper for Que::Job, to maintain
|
|
54
61
|
# backwards-compatibility with internal changes we make.
|
|
55
62
|
module WrapperExtensions
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
63
|
+
module ClassMethods
|
|
64
|
+
# We've dropped support for job options supplied as top-level keywords, but ActiveJob's QueAdapter still uses them. So we have to move them into the job_options hash ourselves.
|
|
65
|
+
def enqueue(args, priority:, queue:, run_at: nil)
|
|
66
|
+
super(args, job_options: { priority: priority, queue: queue, run_at: run_at })
|
|
67
|
+
end
|
|
61
68
|
end
|
|
62
69
|
|
|
63
|
-
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
#
|
|
67
|
-
|
|
68
|
-
|
|
70
|
+
module InstanceMethods
|
|
71
|
+
# The Rails adapter (built against a pre-1.0 version of this gem)
|
|
72
|
+
# assumes that it can access a job's id via job.attrs["job_id"]. So,
|
|
73
|
+
# oblige it.
|
|
74
|
+
def attrs
|
|
75
|
+
{"job_id" => que_attrs[:id]}
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def run(args)
|
|
79
|
+
# Our ActiveJob extensions expect to be able to operate on the actual
|
|
80
|
+
# job object, but there's no way to access it through ActiveJob. So,
|
|
81
|
+
# scope it to the current thread. It's a bit messy, but it's the best
|
|
82
|
+
# option under the circumstances (doesn't require hacking ActiveJob in
|
|
83
|
+
# any more extensive way).
|
|
69
84
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
85
|
+
# There's no reason this logic should ever nest, because it wouldn't
|
|
86
|
+
# make sense to run a worker inside of a job, but even so, assert that
|
|
87
|
+
# nothing absurd is going on.
|
|
88
|
+
Que.assert NilClass, Thread.current[:que_current_job]
|
|
74
89
|
|
|
75
|
-
|
|
76
|
-
|
|
90
|
+
begin
|
|
91
|
+
Thread.current[:que_current_job] = self
|
|
77
92
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
93
|
+
# We symbolize the args hash but ActiveJob doesn't like that :/
|
|
94
|
+
super(args.deep_stringify_keys)
|
|
95
|
+
ensure
|
|
96
|
+
# Also assert that the current job state was only removed now, but
|
|
97
|
+
# unset the job first so that an assertion failure doesn't mess up
|
|
98
|
+
# the state any more than it already has.
|
|
99
|
+
current = Thread.current[:que_current_job]
|
|
100
|
+
Thread.current[:que_current_job] = nil
|
|
101
|
+
Que.assert(self, current)
|
|
102
|
+
end
|
|
87
103
|
end
|
|
88
104
|
end
|
|
89
105
|
end
|
|
@@ -92,6 +108,7 @@ end
|
|
|
92
108
|
|
|
93
109
|
class ActiveJob::QueueAdapters::QueAdapter
|
|
94
110
|
class JobWrapper < Que::Job
|
|
95
|
-
|
|
111
|
+
extend Que::ActiveJob::WrapperExtensions::ClassMethods
|
|
112
|
+
prepend Que::ActiveJob::WrapperExtensions::InstanceMethods
|
|
96
113
|
end
|
|
97
114
|
end
|
|
@@ -39,8 +39,8 @@ module Que
|
|
|
39
39
|
where("que_jobs.data @> ?", JSON.dump(tags: [tag]))
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
def by_args(*args)
|
|
43
|
-
where("que_jobs.args @> ?", JSON.dump(args))
|
|
42
|
+
def by_args(*args, **kwargs)
|
|
43
|
+
where("que_jobs.args @> ? AND que_jobs.kwargs @> ?", JSON.dump(args), JSON.dump(kwargs))
|
|
44
44
|
end
|
|
45
45
|
end
|
|
46
46
|
end
|
data/lib/que/connection.rb
CHANGED
data/lib/que/job.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Que
|
|
|
12
12
|
SQL[:insert_job] =
|
|
13
13
|
%{
|
|
14
14
|
INSERT INTO public.que_jobs
|
|
15
|
-
(queue, priority, run_at, job_class, args, data, job_schema_version)
|
|
15
|
+
(queue, priority, run_at, job_class, args, kwargs, data, job_schema_version)
|
|
16
16
|
VALUES
|
|
17
17
|
(
|
|
18
18
|
coalesce($1, 'default')::text,
|
|
@@ -21,11 +21,32 @@ module Que
|
|
|
21
21
|
$4::text,
|
|
22
22
|
coalesce($5, '[]')::jsonb,
|
|
23
23
|
coalesce($6, '{}')::jsonb,
|
|
24
|
+
coalesce($7, '{}')::jsonb,
|
|
24
25
|
#{Que.job_schema_version}
|
|
25
26
|
)
|
|
26
27
|
RETURNING *
|
|
27
28
|
}
|
|
28
29
|
|
|
30
|
+
SQL[:bulk_insert_jobs] =
|
|
31
|
+
%{
|
|
32
|
+
WITH args_and_kwargs as (
|
|
33
|
+
SELECT * from json_to_recordset(coalesce($5, '[{args:{},kwargs:{}}]')::json) as x(args jsonb, kwargs jsonb)
|
|
34
|
+
)
|
|
35
|
+
INSERT INTO public.que_jobs
|
|
36
|
+
(queue, priority, run_at, job_class, args, kwargs, data, job_schema_version)
|
|
37
|
+
SELECT
|
|
38
|
+
coalesce($1, 'default')::text,
|
|
39
|
+
coalesce($2, 100)::smallint,
|
|
40
|
+
coalesce($3, now())::timestamptz,
|
|
41
|
+
$4::text,
|
|
42
|
+
args_and_kwargs.args,
|
|
43
|
+
args_and_kwargs.kwargs,
|
|
44
|
+
coalesce($6, '{}')::jsonb,
|
|
45
|
+
#{Que.job_schema_version}
|
|
46
|
+
FROM args_and_kwargs
|
|
47
|
+
RETURNING *
|
|
48
|
+
}
|
|
49
|
+
|
|
29
50
|
attr_reader :que_attrs
|
|
30
51
|
attr_accessor :que_error, :que_resolved
|
|
31
52
|
|
|
@@ -56,13 +77,10 @@ module Que
|
|
|
56
77
|
:priority,
|
|
57
78
|
:run_at
|
|
58
79
|
|
|
59
|
-
def enqueue(
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
)
|
|
64
|
-
arg_opts, job_options = _extract_job_options(arg_opts, job_options.dup)
|
|
65
|
-
args << arg_opts if arg_opts.any?
|
|
80
|
+
def enqueue(*args)
|
|
81
|
+
args, kwargs = Que.split_out_ruby2_keywords(args)
|
|
82
|
+
|
|
83
|
+
job_options = kwargs.delete(:job_options) || {}
|
|
66
84
|
|
|
67
85
|
if job_options[:tags]
|
|
68
86
|
if job_options[:tags].length > MAXIMUM_TAGS_COUNT
|
|
@@ -80,36 +98,131 @@ module Que
|
|
|
80
98
|
queue: job_options[:queue] || resolve_que_setting(:queue) || Que.default_queue,
|
|
81
99
|
priority: job_options[:priority] || resolve_que_setting(:priority),
|
|
82
100
|
run_at: job_options[:run_at] || resolve_que_setting(:run_at),
|
|
83
|
-
args:
|
|
84
|
-
|
|
101
|
+
args: args,
|
|
102
|
+
kwargs: kwargs,
|
|
103
|
+
data: job_options[:tags] ? { tags: job_options[:tags] } : {},
|
|
85
104
|
job_class: \
|
|
86
105
|
job_options[:job_class] || name ||
|
|
87
106
|
raise(Error, "Can't enqueue an anonymous subclass of Que::Job"),
|
|
88
107
|
}
|
|
89
108
|
|
|
90
|
-
if
|
|
91
|
-
|
|
92
|
-
|
|
109
|
+
if Thread.current[:que_jobs_to_bulk_insert]
|
|
110
|
+
if self.name == 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'
|
|
111
|
+
raise Que::Error, "Que.bulk_enqueue does not support ActiveJob."
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
raise Que::Error, "When using .bulk_enqueue, job_options must be passed to that method rather than .enqueue" unless job_options == {}
|
|
115
|
+
|
|
116
|
+
Thread.current[:que_jobs_to_bulk_insert][:jobs_attrs] << attrs
|
|
117
|
+
new({})
|
|
118
|
+
elsif attrs[:run_at].nil? && resolve_que_setting(:run_synchronously)
|
|
119
|
+
attrs.merge!(
|
|
120
|
+
args: Que.deserialize_json(Que.serialize_json(attrs[:args])),
|
|
121
|
+
kwargs: Que.deserialize_json(Que.serialize_json(attrs[:kwargs])),
|
|
122
|
+
data: Que.deserialize_json(Que.serialize_json(attrs[:data])),
|
|
123
|
+
)
|
|
93
124
|
_run_attrs(attrs)
|
|
94
125
|
else
|
|
95
|
-
|
|
96
|
-
Que.
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
126
|
+
attrs.merge!(
|
|
127
|
+
args: Que.serialize_json(attrs[:args]),
|
|
128
|
+
kwargs: Que.serialize_json(attrs[:kwargs]),
|
|
129
|
+
data: Que.serialize_json(attrs[:data]),
|
|
130
|
+
)
|
|
131
|
+
values = Que.execute(
|
|
132
|
+
:insert_job,
|
|
133
|
+
attrs.values_at(:queue, :priority, :run_at, :job_class, :args, :kwargs, :data),
|
|
134
|
+
).first
|
|
101
135
|
new(values)
|
|
102
136
|
end
|
|
103
137
|
end
|
|
138
|
+
ruby2_keywords(:enqueue) if respond_to?(:ruby2_keywords, true)
|
|
139
|
+
|
|
140
|
+
def bulk_enqueue(job_options: {}, notify: false)
|
|
141
|
+
raise Que::Error, "Can't nest .bulk_enqueue" unless Thread.current[:que_jobs_to_bulk_insert].nil?
|
|
142
|
+
Thread.current[:que_jobs_to_bulk_insert] = { jobs_attrs: [], job_options: job_options }
|
|
143
|
+
yield
|
|
144
|
+
jobs_attrs = Thread.current[:que_jobs_to_bulk_insert][:jobs_attrs]
|
|
145
|
+
job_options = Thread.current[:que_jobs_to_bulk_insert][:job_options]
|
|
146
|
+
return [] if jobs_attrs.empty?
|
|
147
|
+
raise Que::Error, "When using .bulk_enqueue, all jobs enqueued must be of the same job class" unless jobs_attrs.map { |attrs| attrs[:job_class] }.uniq.one?
|
|
148
|
+
args_and_kwargs_array = jobs_attrs.map { |attrs| attrs.slice(:args, :kwargs) }
|
|
149
|
+
klass = job_options[:job_class] ? Que::Job : Que.constantize(jobs_attrs.first[:job_class])
|
|
150
|
+
klass._bulk_enqueue_insert(args_and_kwargs_array, job_options: job_options, notify: notify)
|
|
151
|
+
ensure
|
|
152
|
+
Thread.current[:que_jobs_to_bulk_insert] = nil
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def _bulk_enqueue_insert(args_and_kwargs_array, job_options: {}, notify:)
|
|
156
|
+
raise 'Unexpected bulk args format' if !args_and_kwargs_array.is_a?(Array) || !args_and_kwargs_array.all? { |a| a.is_a?(Hash) }
|
|
157
|
+
|
|
158
|
+
if job_options[:tags]
|
|
159
|
+
if job_options[:tags].length > MAXIMUM_TAGS_COUNT
|
|
160
|
+
raise Que::Error, "Can't enqueue a job with more than #{MAXIMUM_TAGS_COUNT} tags! (passed #{job_options[:tags].length})"
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
job_options[:tags].each do |tag|
|
|
164
|
+
if tag.length > MAXIMUM_TAG_LENGTH
|
|
165
|
+
raise Que::Error, "Can't enqueue a job with a tag longer than 100 characters! (\"#{tag}\")"
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
args_and_kwargs_array = args_and_kwargs_array.map do |args_and_kwargs|
|
|
171
|
+
args_and_kwargs.merge(
|
|
172
|
+
args: args_and_kwargs.fetch(:args, []),
|
|
173
|
+
kwargs: args_and_kwargs.fetch(:kwargs, {}),
|
|
174
|
+
)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
attrs = {
|
|
178
|
+
queue: job_options[:queue] || resolve_que_setting(:queue) || Que.default_queue,
|
|
179
|
+
priority: job_options[:priority] || resolve_que_setting(:priority),
|
|
180
|
+
run_at: job_options[:run_at] || resolve_que_setting(:run_at),
|
|
181
|
+
args_and_kwargs_array: args_and_kwargs_array,
|
|
182
|
+
data: job_options[:tags] ? { tags: job_options[:tags] } : {},
|
|
183
|
+
job_class: \
|
|
184
|
+
job_options[:job_class] || name ||
|
|
185
|
+
raise(Error, "Can't enqueue an anonymous subclass of Que::Job"),
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if attrs[:run_at].nil? && resolve_que_setting(:run_synchronously)
|
|
189
|
+
args_and_kwargs_array = Que.deserialize_json(Que.serialize_json(attrs.delete(:args_and_kwargs_array)))
|
|
190
|
+
args_and_kwargs_array.map do |args_and_kwargs|
|
|
191
|
+
_run_attrs(
|
|
192
|
+
attrs.merge(
|
|
193
|
+
args: args_and_kwargs.fetch(:args),
|
|
194
|
+
kwargs: args_and_kwargs.fetch(:kwargs),
|
|
195
|
+
),
|
|
196
|
+
)
|
|
197
|
+
end
|
|
198
|
+
else
|
|
199
|
+
attrs.merge!(
|
|
200
|
+
args_and_kwargs_array: Que.serialize_json(attrs[:args_and_kwargs_array]),
|
|
201
|
+
data: Que.serialize_json(attrs[:data]),
|
|
202
|
+
)
|
|
203
|
+
values_array =
|
|
204
|
+
Que.transaction do
|
|
205
|
+
Que.execute('SET LOCAL que.skip_notify TO true') unless notify
|
|
206
|
+
Que.execute(
|
|
207
|
+
:bulk_insert_jobs,
|
|
208
|
+
attrs.values_at(:queue, :priority, :run_at, :job_class, :args_and_kwargs_array, :data),
|
|
209
|
+
)
|
|
210
|
+
end
|
|
211
|
+
values_array.map(&method(:new))
|
|
212
|
+
end
|
|
213
|
+
end
|
|
104
214
|
|
|
105
215
|
def run(*args)
|
|
106
216
|
# Make sure things behave the same as they would have with a round-trip
|
|
107
217
|
# to the DB.
|
|
218
|
+
args, kwargs = Que.split_out_ruby2_keywords(args)
|
|
108
219
|
args = Que.deserialize_json(Que.serialize_json(args))
|
|
220
|
+
kwargs = Que.deserialize_json(Que.serialize_json(kwargs))
|
|
109
221
|
|
|
110
222
|
# Should not fail if there's no DB connection.
|
|
111
|
-
_run_attrs(args: args)
|
|
223
|
+
_run_attrs(args: args, kwargs: kwargs)
|
|
112
224
|
end
|
|
225
|
+
ruby2_keywords(:run) if respond_to?(:ruby2_keywords, true)
|
|
113
226
|
|
|
114
227
|
def resolve_que_setting(setting, *args)
|
|
115
228
|
value = send(setting) if respond_to?(setting)
|
|
@@ -136,27 +249,6 @@ module Que
|
|
|
136
249
|
end
|
|
137
250
|
end
|
|
138
251
|
end
|
|
139
|
-
|
|
140
|
-
def _extract_job_options(arg_opts, job_options)
|
|
141
|
-
deprecated_job_option_names = []
|
|
142
|
-
|
|
143
|
-
%i[queue priority run_at job_class tags].each do |option_name|
|
|
144
|
-
next unless arg_opts.key?(option_name) && job_options[option_name].nil?
|
|
145
|
-
|
|
146
|
-
job_options[option_name] = arg_opts.delete(option_name)
|
|
147
|
-
deprecated_job_option_names << option_name
|
|
148
|
-
end
|
|
149
|
-
|
|
150
|
-
_log_job_options_deprecation(deprecated_job_option_names)
|
|
151
|
-
|
|
152
|
-
[arg_opts, job_options]
|
|
153
|
-
end
|
|
154
|
-
|
|
155
|
-
def _log_job_options_deprecation(deprecated_job_option_names)
|
|
156
|
-
return unless deprecated_job_option_names.any?
|
|
157
|
-
|
|
158
|
-
warn "Passing job options like (#{deprecated_job_option_names.join(', ')}) to `JobClass.enqueue` as top level keyword args has been deprecated and will be removed in version 2.0. Please wrap job options in an explicit `job_options` keyword arg instead."
|
|
159
|
-
end
|
|
160
252
|
end
|
|
161
253
|
|
|
162
254
|
# Set up some defaults.
|
data/lib/que/job_methods.rb
CHANGED
|
@@ -39,12 +39,16 @@ module Que
|
|
|
39
39
|
# Run the job with error handling and cleanup logic. Optionally support
|
|
40
40
|
# overriding the args, because it's necessary when jobs are invoked from
|
|
41
41
|
# ActiveJob.
|
|
42
|
-
def _run(args: nil, reraise_errors: false)
|
|
42
|
+
def _run(args: nil, kwargs: nil, reraise_errors: false)
|
|
43
43
|
if args.nil? && que_target
|
|
44
44
|
args = que_target.que_attrs.fetch(:args)
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
-
|
|
47
|
+
if kwargs.nil? && que_target
|
|
48
|
+
kwargs = que_target.que_attrs.fetch(:kwargs)
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
run(*args, **kwargs)
|
|
48
52
|
default_resolve_action if que_target && !que_target.que_resolved
|
|
49
53
|
rescue => error
|
|
50
54
|
raise error unless que_target
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
DROP INDEX que_jobs_kwargs_gin_idx;
|
|
2
|
+
ALTER TABLE que_jobs DROP COLUMN kwargs;
|
|
3
|
+
|
|
4
|
+
ALTER INDEX que_poll_idx RENAME TO que_poll_idx_with_job_schema_version;
|
|
5
|
+
CREATE INDEX que_poll_idx ON que_jobs (queue, priority, run_at, id) WHERE (finished_at IS NULL AND expired_at IS NULL);
|
|
6
|
+
|
|
7
|
+
ALTER TABLE que_jobs ALTER COLUMN job_schema_version SET DEFAULT 1;
|
|
8
|
+
ALTER TABLE que_jobs ALTER COLUMN job_schema_version DROP NOT NULL;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
ALTER TABLE que_jobs ADD COLUMN kwargs JSONB NOT NULL DEFAULT '{}';
|
|
2
|
+
CREATE INDEX que_jobs_kwargs_gin_idx ON que_jobs USING gin (kwargs jsonb_path_ops);
|
|
3
|
+
|
|
4
|
+
DROP INDEX que_poll_idx;
|
|
5
|
+
ALTER INDEX que_poll_idx_with_job_schema_version RENAME TO que_poll_idx;
|
|
6
|
+
|
|
7
|
+
ALTER TABLE que_jobs ALTER COLUMN job_schema_version DROP DEFAULT;
|
|
8
|
+
ALTER TABLE que_jobs ALTER COLUMN job_schema_version SET NOT NULL;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
DROP TRIGGER que_job_notify ON que_jobs;
|
|
2
|
+
CREATE TRIGGER que_job_notify
|
|
3
|
+
AFTER INSERT ON que_jobs
|
|
4
|
+
FOR EACH ROW
|
|
5
|
+
WHEN (NOT coalesce(current_setting('que.skip_notify', true), '') = 'true')
|
|
6
|
+
EXECUTE PROCEDURE public.que_job_notify();
|
|
7
|
+
|
|
8
|
+
DROP TRIGGER que_state_notify ON que_jobs;
|
|
9
|
+
CREATE TRIGGER que_state_notify
|
|
10
|
+
AFTER INSERT OR UPDATE OR DELETE ON que_jobs
|
|
11
|
+
FOR EACH ROW
|
|
12
|
+
WHEN (NOT coalesce(current_setting('que.skip_notify', true), '') = 'true')
|
|
13
|
+
EXECUTE PROCEDURE public.que_state_notify();
|
data/lib/que/migrations.rb
CHANGED
data/lib/que/sequel/model.rb
CHANGED
|
@@ -40,8 +40,11 @@ module Que
|
|
|
40
40
|
where(QUALIFIED_TABLE[:data].pg_jsonb.contains(JSON.dump(tags: [tag])))
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
def by_args(*args)
|
|
44
|
-
where(
|
|
43
|
+
def by_args(*args, **kwargs)
|
|
44
|
+
where(
|
|
45
|
+
QUALIFIED_TABLE[:args].pg_jsonb.contains(JSON.dump(args)) &
|
|
46
|
+
QUALIFIED_TABLE[:kwargs].pg_jsonb.contains(JSON.dump(kwargs))
|
|
47
|
+
)
|
|
45
48
|
end
|
|
46
49
|
end
|
|
47
50
|
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Temporary module allowing ruby2 keyword args to be extracted from an *args splat
|
|
4
|
+
# Allows us to ensure consistent behaviour when running on ruby 2 vs ruby 3
|
|
5
|
+
# We can remove this if/when we drop support for ruby 2
|
|
6
|
+
|
|
7
|
+
require 'json'
|
|
8
|
+
|
|
9
|
+
module Que
|
|
10
|
+
module Utils
|
|
11
|
+
module Ruby2Keywords
|
|
12
|
+
def split_out_ruby2_keywords(args)
|
|
13
|
+
return [args, {}] unless args.last&.is_a?(Hash) && Hash.ruby2_keywords_hash?(args.last)
|
|
14
|
+
|
|
15
|
+
[args[0..-2], args.last]
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
data/lib/que/version.rb
CHANGED
data/lib/que/worker.rb
CHANGED
data/lib/que.rb
CHANGED
|
@@ -29,6 +29,7 @@ module Que
|
|
|
29
29
|
require_relative 'que/utils/logging'
|
|
30
30
|
require_relative 'que/utils/middleware'
|
|
31
31
|
require_relative 'que/utils/queue_management'
|
|
32
|
+
require_relative 'que/utils/ruby2_keywords'
|
|
32
33
|
require_relative 'que/utils/transactions'
|
|
33
34
|
|
|
34
35
|
require_relative 'que/version'
|
|
@@ -61,13 +62,14 @@ module Que
|
|
|
61
62
|
include Utils::Logging
|
|
62
63
|
include Utils::Middleware
|
|
63
64
|
include Utils::QueueManagement
|
|
65
|
+
include Utils::Ruby2Keywords
|
|
64
66
|
include Utils::Transactions
|
|
65
67
|
|
|
66
68
|
extend Forwardable
|
|
67
69
|
|
|
68
70
|
# Copy some commonly-used methods here, for convenience.
|
|
69
71
|
def_delegators :pool, :execute, :checkout, :in_transaction?
|
|
70
|
-
def_delegators Job, :enqueue, :run_synchronously, :run_synchronously=
|
|
72
|
+
def_delegators Job, :enqueue, :bulk_enqueue, :run_synchronously, :run_synchronously=
|
|
71
73
|
def_delegators Migrations, :db_version, :migrate!
|
|
72
74
|
|
|
73
75
|
# Global configuration logic.
|
data/que.gemspec
CHANGED
data/scripts/test
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: que
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 2.1.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Chris Hanks
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2022-
|
|
11
|
+
date: 2022-08-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -34,6 +34,7 @@ extra_rdoc_files: []
|
|
|
34
34
|
files:
|
|
35
35
|
- ".github/workflows/tests.yml"
|
|
36
36
|
- ".gitignore"
|
|
37
|
+
- ".ruby-version"
|
|
37
38
|
- CHANGELOG.md
|
|
38
39
|
- Dockerfile
|
|
39
40
|
- LICENSE.txt
|
|
@@ -71,6 +72,10 @@ files:
|
|
|
71
72
|
- lib/que/migrations/4/up.sql
|
|
72
73
|
- lib/que/migrations/5/down.sql
|
|
73
74
|
- lib/que/migrations/5/up.sql
|
|
75
|
+
- lib/que/migrations/6/down.sql
|
|
76
|
+
- lib/que/migrations/6/up.sql
|
|
77
|
+
- lib/que/migrations/7/down.sql
|
|
78
|
+
- lib/que/migrations/7/up.sql
|
|
74
79
|
- lib/que/poller.rb
|
|
75
80
|
- lib/que/rails/railtie.rb
|
|
76
81
|
- lib/que/result_queue.rb
|
|
@@ -84,6 +89,7 @@ files:
|
|
|
84
89
|
- lib/que/utils/logging.rb
|
|
85
90
|
- lib/que/utils/middleware.rb
|
|
86
91
|
- lib/que/utils/queue_management.rb
|
|
92
|
+
- lib/que/utils/ruby2_keywords.rb
|
|
87
93
|
- lib/que/utils/transactions.rb
|
|
88
94
|
- lib/que/version.rb
|
|
89
95
|
- lib/que/worker.rb
|
|
@@ -100,16 +106,16 @@ require_paths:
|
|
|
100
106
|
- lib
|
|
101
107
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
102
108
|
requirements:
|
|
103
|
-
- - "
|
|
109
|
+
- - ">="
|
|
104
110
|
- !ruby/object:Gem::Version
|
|
105
|
-
version:
|
|
111
|
+
version: 2.7.0
|
|
106
112
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
107
113
|
requirements:
|
|
108
114
|
- - ">="
|
|
109
115
|
- !ruby/object:Gem::Version
|
|
110
116
|
version: '0'
|
|
111
117
|
requirements: []
|
|
112
|
-
rubygems_version: 3.
|
|
118
|
+
rubygems_version: 3.3.7
|
|
113
119
|
signing_key:
|
|
114
120
|
specification_version: 4
|
|
115
121
|
summary: A PostgreSQL-based Job Queue
|