que 1.0.0.beta → 1.0.0.beta5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/tests.yml +43 -0
- data/CHANGELOG.1.0.beta.md +137 -0
- data/CHANGELOG.md +34 -12
- data/README.md +67 -7
- data/bin/command_line_interface.rb +61 -49
- data/docs/README.md +785 -32
- data/lib/que/active_record/connection.rb +4 -4
- data/lib/que/active_record/model.rb +4 -4
- data/lib/que/connection.rb +35 -16
- data/lib/que/connection_pool.rb +2 -2
- data/lib/que/job.rb +1 -1
- data/lib/que/{job_cache.rb → job_buffer.rb} +96 -72
- data/lib/que/job_methods.rb +4 -0
- data/lib/que/locker.rb +170 -139
- data/lib/que/poller.rb +1 -1
- data/lib/que/rails/railtie.rb +2 -4
- data/lib/que/result_queue.rb +2 -2
- data/lib/que/sequel/model.rb +14 -16
- data/lib/que/utils/constantization.rb +1 -1
- data/lib/que/utils/logging.rb +2 -1
- data/lib/que/utils/middleware.rb +26 -13
- data/lib/que/version.rb +1 -1
- data/lib/que/worker.rb +43 -21
- data/lib/que.rb +9 -4
- data/que.gemspec +2 -2
- metadata +11 -25
- data/docs/active_job.md +0 -6
- data/docs/advanced_setup.md +0 -49
- data/docs/command_line_interface.md +0 -45
- data/docs/error_handling.md +0 -94
- data/docs/inspecting_the_queue.md +0 -64
- data/docs/job_helper_methods.md +0 -27
- data/docs/logging.md +0 -31
- data/docs/managing_workers.md +0 -25
- data/docs/middleware.md +0 -15
- data/docs/migrating.md +0 -27
- data/docs/multiple_queues.md +0 -31
- data/docs/shutting_down_safely.md +0 -7
- data/docs/using_plain_connections.md +0 -65
- data/docs/using_sequel.md +0 -33
- data/docs/writing_reliable_jobs.md +0 -108
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 90ae5c7c65d93f4ea3833d80351c2e0d3ed3f706ba00411937e8aba2137a41d7
|
4
|
+
data.tar.gz: 58aba934ee2735522e602dd5777de666c6acbee1cb531d9918ce6f2b75f3b693
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1cbfa5f3c6d13868897ac9f1fcf079aaeaa29c9eadfd231044a484878fbf8e99f4b159a4458690027b181b29c5da3abafcc1c9de2d33b3c487b1fac4a15c7e50
|
7
|
+
data.tar.gz: 1ed4b71f51cd72e3968de6ec745e2de592dc021e9d334526630b3011d3f5526388b8562da4bf513066cb68d4dee033043115027ddfc0e9bef5e816ed2c35ede5
|
@@ -0,0 +1,43 @@
|
|
1
|
+
name: tests
|
2
|
+
|
3
|
+
on: [push, pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
test:
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
strategy:
|
9
|
+
matrix:
|
10
|
+
ruby_version: [2.5.x, 2.6.x, 2.7.x]
|
11
|
+
gemfile: ["4.2", "5.2", "6.0"]
|
12
|
+
postgres_version: [9, 10, 11, 12]
|
13
|
+
exclude:
|
14
|
+
- { gemfile: "4.2", ruby_version: "2.7.x" }
|
15
|
+
services:
|
16
|
+
db:
|
17
|
+
image: postgres:${{ matrix.postgres_version }}
|
18
|
+
env:
|
19
|
+
POSTGRES_HOST_AUTH_METHOD: trust
|
20
|
+
ports: ['5432:5432']
|
21
|
+
options: >-
|
22
|
+
--health-cmd pg_isready
|
23
|
+
--health-interval 10s
|
24
|
+
--health-timeout 5s
|
25
|
+
--health-retries 5
|
26
|
+
steps:
|
27
|
+
- uses: actions/checkout@v1
|
28
|
+
- name: Set up Ruby
|
29
|
+
uses: actions/setup-ruby@v1
|
30
|
+
with:
|
31
|
+
ruby-version: ${{ matrix.ruby_version }}
|
32
|
+
- name: Test with Rake
|
33
|
+
env:
|
34
|
+
PGHOST: 127.0.0.1
|
35
|
+
PGUSER: postgres
|
36
|
+
BUNDLE_GEMFILE: spec/gemfiles/Gemfile.${{ matrix.gemfile }}
|
37
|
+
run: |
|
38
|
+
sudo apt-get -yqq install libpq-dev postgresql-client
|
39
|
+
createdb que-test
|
40
|
+
gem install bundler
|
41
|
+
bundle install --jobs 4 --retry 3
|
42
|
+
USE_RAILS=true bundle exec rake test
|
43
|
+
bundle exec rake test
|
@@ -0,0 +1,137 @@
|
|
1
|
+
### 1.0.0.beta3 (2018-05-18)
|
2
|
+
|
3
|
+
* Added support for customizing log levels for `job_worked` events (#217).
|
4
|
+
|
5
|
+
* Began logging all `job_errored` events at the `ERROR` log level.
|
6
|
+
|
7
|
+
* Fixed the Railtie when running in test mode (#214).
|
8
|
+
|
9
|
+
* Tweaked the meanings of worker-priorities and worker-count options in the CLI, to better support use cases with low worker counts (#216).
|
10
|
+
|
11
|
+
### 1.0.0.beta2 (2018-04-13)
|
12
|
+
|
13
|
+
* Fixed an incompatibility that caused the new locker to hang when using Rails in development mode (#213).
|
14
|
+
|
15
|
+
* Fixed a bug with setting the log level via the CLI when the configured logger was based on a callable (#210).
|
16
|
+
|
17
|
+
* Renamed Que.middleware to Que.job_middleware.
|
18
|
+
|
19
|
+
* Added Que.sql_middleware.
|
20
|
+
|
21
|
+
* Officially added support for Ruby 2.5.
|
22
|
+
|
23
|
+
* Internal cleanup and renamings.
|
24
|
+
|
25
|
+
### 1.0.0.beta (2017-10-25)
|
26
|
+
|
27
|
+
* **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/que-rb/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
|
28
|
+
|
29
|
+
* Please note that this migration requires a rewrite of the jobs table, which makes it O(n) with the size of the table. If you have a very large backlog of jobs you may want to schedule downtime for this migration.
|
30
|
+
|
31
|
+
* Que's implementation has been changed from one in which worker threads hold their own PG connections and lock their own jobs to one in which a single thread (and PG connection) locks jobs through LISTEN/NOTIFY and batch polling, and passes jobs along to worker threads. This has many benefits, including:
|
32
|
+
|
33
|
+
* Jobs queued for immediate processing can be actively distributed to workers with LISTEN/NOTIFY, which is more efficient than having workers repeatedly poll for new jobs.
|
34
|
+
|
35
|
+
* When polling is necessary (to pick up jobs that are scheduled for the future or that need to be retried due to errors), jobs can be locked and fetched in batches, rather than one at a time.
|
36
|
+
|
37
|
+
* Individual workers no longer need to monopolize their own (usually idle) connections while working jobs, so Ruby processes will require fewer Postgres connections.
|
38
|
+
|
39
|
+
* PgBouncer or another external connection pool can be used for workers' connections (though not for the connection used to lock and listen for jobs).
|
40
|
+
|
41
|
+
* Other features introduced in this version include:
|
42
|
+
|
43
|
+
* Much better support for all versions of ActiveJob.
|
44
|
+
|
45
|
+
* In particular, you may (optionally) include `Que::ActiveJob::JobExtensions` into `ApplicationJob` to get support for all of Que's job helper methods.
|
46
|
+
|
47
|
+
* Custom middleware that wrap running jobs are now supported.
|
48
|
+
|
49
|
+
* Support for categorizing jobs with tags.
|
50
|
+
|
51
|
+
* Support for configuring a `maximum_retry_count` on individual job classes.
|
52
|
+
|
53
|
+
* Job configuration options are now inheritable, so job class hierarchies are more useful.
|
54
|
+
|
55
|
+
* There are now built-in models for ActiveRecord and Sequel to allow inspecting the queue easily.
|
56
|
+
|
57
|
+
* Jobs that have finished working may optionally be retained in the database indefinitely.
|
58
|
+
|
59
|
+
* To keep a job record, replace the `destroy` calls in your jobs with `finish`. `destroy` will still delete records entirely, for jobs that you don't want to keep.
|
60
|
+
|
61
|
+
* If you don't resolve a job yourself one way or another, Que will still `destroy` the job for you by default.
|
62
|
+
|
63
|
+
* Finished jobs have a timestamp set in the finished_at column.
|
64
|
+
|
65
|
+
* Jobs that have errored too many times will now be marked as expired, and won't be retried again.
|
66
|
+
|
67
|
+
* You can configure a maximum_retry_count in your job classes, to set the threshold at which a job will be marked expired. The default is 15.
|
68
|
+
|
69
|
+
* To manually mark a job as expired (and keep it in the database but not try to run it again) you can call `expire` helper in your job.
|
70
|
+
|
71
|
+
* You can now set job priority thresholds for individual workers, to ensure that there will always be space available for high-priority jobs.
|
72
|
+
|
73
|
+
* `Que.job_states` returns a list of locked jobs and the hostname/pid of the Ruby processes that have locked them.
|
74
|
+
|
75
|
+
* `Que.connection_proc=` has been added, to allow for the easy integration of custom connection pools.
|
76
|
+
|
77
|
+
* In keeping with semantic versioning, the major version is being bumped since the new implementation requires some backwards-incompatible changes. These changes include:
|
78
|
+
|
79
|
+
* Support for MRI Rubies before 2.2 has been dropped.
|
80
|
+
|
81
|
+
* Support for Postgres versions before 9.5 has been dropped (JSONB and upsert support is required).
|
82
|
+
|
83
|
+
* JRuby support has been dropped. It will be reintroduced whenever the jruby-pg gem is production-ready.
|
84
|
+
|
85
|
+
* The `que:work` rake task has been removed. Use the `que` executable instead.
|
86
|
+
|
87
|
+
* Therefore, configuring workers using QUE_* environment variables is no longer supported. Please pass the appropriate options to the `que` executable instead.
|
88
|
+
|
89
|
+
* The `mode` setter has been removed.
|
90
|
+
|
91
|
+
* To run jobs synchronously when they are enqueued (the old `:sync` behavior) you can set `Que.run_synchronously = true`.
|
92
|
+
|
93
|
+
* To start up the worker pool (the old :async behavior) you should use the `que` executable to start up a worker process. There's no longer a supported API for running workers outside of the `que` executable.
|
94
|
+
|
95
|
+
* The following methods are not meaningful under the new implementation and have been removed:
|
96
|
+
|
97
|
+
* The `Que.wake_interval` getter and setter.
|
98
|
+
|
99
|
+
* The `Que.worker_count` getter and setter.
|
100
|
+
|
101
|
+
* `Que.wake!`
|
102
|
+
|
103
|
+
* `Que.wake_all!`
|
104
|
+
|
105
|
+
* Since Que needs a dedicated Postgres connection to manage job locks, running Que through a single PG connection is no longer supported.
|
106
|
+
|
107
|
+
* It's not clear that anyone ever actually did this.
|
108
|
+
|
109
|
+
* `Que.worker_states` has been removed, as the connection that locks a job is no longer the one that the job is using to run. Its functionality has been partially replaced with `Que.job_states`.
|
110
|
+
|
111
|
+
* When using Rails, for simplicity, job attributes and keys in argument hashes are now converted to symbols when retrieved from the database, rather than being converted to instances of HashWithIndifferentAccess.
|
112
|
+
|
113
|
+
* Arguments passed to jobs are now deep-frozen, to prevent unexpected behavior when the args are mutated and the job is reenqueued.
|
114
|
+
|
115
|
+
* Since JSONB is now used to store arguments, the order of argument hashes is no longer maintained.
|
116
|
+
|
117
|
+
* It wouldn't have been a good idea to rely on this anyway.
|
118
|
+
|
119
|
+
* Calling Que.log() directly is no longer supported/recommended.
|
120
|
+
|
121
|
+
* Features marked as deprecated in the final 0.x releases have been removed.
|
122
|
+
|
123
|
+
* Finally, if you've built up your own tooling and customizations around Que, you may need to be aware of some DB schema changes made in the migration to schema version #4.
|
124
|
+
|
125
|
+
* The `job_id` column has been renamed `id` and is now the primary key. This makes it easier to manage the queue using an ActiveRecord model.
|
126
|
+
|
127
|
+
* Finished jobs are now kept in the DB, unless you explicitly call `destroy`. If you want to query the DB for only jobs that haven't finished yet, add a `WHERE finished_at IS NULL` condition to your query, or use the not_finished scope on one of the provided ORM models.
|
128
|
+
|
129
|
+
* There is now an `expired_at` timestamp column, which is set when a job reaches its maximum number of retries and will not be attempted again.
|
130
|
+
|
131
|
+
* Due to popular demand, the default queue name is now "default" rather than an empty string. The migration will move pending jobs under the "" queue to the "default" queue.
|
132
|
+
|
133
|
+
* The `last_error` column has been split in two, to `last_error_message` and `last_error_backtrace`. These two columns are now limited to 500 and 10,000 characters, respectively. The migration will split old error data correctly, and truncate it if necessary.
|
134
|
+
|
135
|
+
* Names for queues and job classes are now limited to 500 characters, which is still far longer than either of these values should reasonably be.
|
136
|
+
|
137
|
+
* There is now a `data` JSONB column which is used to support various ways of organizing jobs (setting tags on them, etc).
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
### 1.0.0.
|
1
|
+
### 1.0.0.beta2 (2018-04-13)
|
2
2
|
|
3
|
-
* **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/
|
3
|
+
* **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/que-rb/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
|
4
4
|
|
5
5
|
* Please note that this migration requires a rewrite of the jobs table, which makes it O(n) with the size of the table. If you have a very large backlog of jobs you may want to schedule downtime for this migration.
|
6
6
|
|
@@ -20,7 +20,7 @@
|
|
20
20
|
|
21
21
|
* In particular, you may (optionally) include `Que::ActiveJob::JobExtensions` into `ApplicationJob` to get support for all of Que's job helper methods.
|
22
22
|
|
23
|
-
* Custom middleware that wrap running jobs are now supported.
|
23
|
+
* Custom middleware that wrap running jobs and executing SQL statements are now supported.
|
24
24
|
|
25
25
|
* Support for categorizing jobs with tags.
|
26
26
|
|
@@ -68,11 +68,7 @@
|
|
68
68
|
|
69
69
|
* To start up the worker pool (the old :async behavior) you should use the `que` executable to start up a worker process. There's no longer a supported API for running workers outside of the `que` executable.
|
70
70
|
|
71
|
-
* The
|
72
|
-
|
73
|
-
* Specifically, while Que previously used prepared statements for most of its built-in queries, now only the polling query uses it, due to its complexity. Since the polling query is only run through a dedicated connection, it's no longer possible for prepared statements to conflict with external connection pools, which was the reason that `disable_prepared_statements` was supported in the first place.
|
74
|
-
|
75
|
-
* In addition to `Que.disable_prepared_statements=`, the following methods are not meaningful under the new implementation and have been removed:
|
71
|
+
* The following methods are not meaningful under the new implementation and have been removed:
|
76
72
|
|
77
73
|
* The `Que.wake_interval` getter and setter.
|
78
74
|
|
@@ -116,6 +112,32 @@
|
|
116
112
|
|
117
113
|
* There is now a `data` JSONB column which is used to support various ways of organizing jobs (setting tags on them, etc).
|
118
114
|
|
115
|
+
For a detailed list of the changes between each beta release of 1.0.0, see [the beta Changelog](CHANGELOG.1.0.beta.md).
|
116
|
+
|
117
|
+
### 0.14.3 (2018-03-02)
|
118
|
+
|
119
|
+
* Recorded errors now always include the error class, so that empty error messages can still be helpful. ( joehorsnell)
|
120
|
+
|
121
|
+
* Recorded error messages are now truncated to the first 500 characters.
|
122
|
+
|
123
|
+
### 0.14.2 (2018-01-05)
|
124
|
+
|
125
|
+
* Deprecate the Que.disable_prepared_statements= accessors.
|
126
|
+
|
127
|
+
* Add Que.use_prepared_statements= configuration accessors.
|
128
|
+
|
129
|
+
* Update the generated Rails migration to declare a version. (NARKOZ)
|
130
|
+
|
131
|
+
### 0.14.1 (2017-12-14)
|
132
|
+
|
133
|
+
* Fix a bug with typecasting boolean values on Rails 5+.
|
134
|
+
|
135
|
+
### 0.14.0 (2017-08-11)
|
136
|
+
|
137
|
+
* Fix incompatibility with Rails 5.1.
|
138
|
+
|
139
|
+
* Drop support for waking an in-process worker when an ActiveRecord transaction commits.
|
140
|
+
|
119
141
|
### 0.13.1 (2017-07-05)
|
120
142
|
|
121
143
|
* Fix issue that caused error stacktraces to not be persisted in most cases.
|
@@ -250,7 +272,7 @@
|
|
250
272
|
|
251
273
|
### 0.6.0 (2014-02-04)
|
252
274
|
|
253
|
-
* **A schema upgrade to version 3 is required for this release.** See [the migration doc](https://github.com/
|
275
|
+
* **A schema upgrade to version 3 is required for this release.** See [the migration doc](https://github.com/que-rb/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
|
254
276
|
|
255
277
|
* You can now run a job's logic directly (without enqueueing it) like `MyJob.run(arg1, arg2, other_arg: arg3)`. This is useful when a job class encapsulates logic that you want to invoke without involving the entire queue.
|
256
278
|
|
@@ -262,11 +284,11 @@
|
|
262
284
|
|
263
285
|
* Log lines now include the machine's hostname, since a pid alone may not uniquely identify a process.
|
264
286
|
|
265
|
-
* Multiple queues are now supported. See [the docs](https://github.com/
|
287
|
+
* Multiple queues are now supported. See [the docs](https://github.com/que-rb/que/blob/master/docs/multiple_queues.md) for details. (chanks, joevandyk)
|
266
288
|
|
267
289
|
* Rubinius 2.2 is now supported. (brixen)
|
268
290
|
|
269
|
-
* Job classes may now define their own logic for determining the retry interval when a job raises an error. See [error handling](https://github.com/
|
291
|
+
* Job classes may now define their own logic for determining the retry interval when a job raises an error. See [error handling](https://github.com/que-rb/que/blob/master/docs/error_handling.md) for more information.
|
270
292
|
|
271
293
|
### 0.5.0 (2014-01-14)
|
272
294
|
|
@@ -280,7 +302,7 @@
|
|
280
302
|
|
281
303
|
* Added a migration system to make it easier to change the schema when updating Que. You can now write, for example, `Que.migrate!(version: 2)` in your migrations. Migrations are run transactionally.
|
282
304
|
|
283
|
-
* The logging format has changed to be more easily machine-readable. You can also now customize the logging format by assigning a callable to Que.log_formatter=. See the new doc on [logging](https://github.com/
|
305
|
+
* The logging format has changed to be more easily machine-readable. You can also now customize the logging format by assigning a callable to Que.log_formatter=. See the new doc on [logging](https://github.com/que-rb/que/blob/master/docs/logging.md)) for details. The default logger level is INFO - for less critical information (such as when no jobs were found to be available or when a job-lock race condition has been detected and avoided) you can set the QUE_LOG_LEVEL environment variable to DEBUG.
|
284
306
|
|
285
307
|
* MultiJson is now a soft dependency. Que will use it if it is available, but it is not required.
|
286
308
|
|
data/README.md
CHANGED
@@ -1,6 +1,8 @@
|
|
1
|
-
# Que
|
1
|
+
# Que ![tests](https://github.com/que-rb/que/workflows/tests/badge.svg)
|
2
2
|
|
3
|
-
**
|
3
|
+
**This README and the rest of the docs on the master branch all refer to Que 1.0, which is currently in beta. If you're using version 0.x, please refer to the docs on [the 0.x branch](https://github.com/que-rb/que/tree/0.x).**
|
4
|
+
|
5
|
+
*TL;DR: Que is a high-performance job queue that improves the reliability of your application by protecting your jobs with the same [ACID guarantees](https://en.wikipedia.org/wiki/ACID) as the rest of your data.*
|
4
6
|
|
5
7
|
Que ("keɪ", or "kay") is a queue for Ruby and PostgreSQL that manages jobs using [advisory locks](http://www.postgresql.org/docs/current/static/explicit-locking.html#ADVISORY-LOCKS), which gives it several advantages over other RDBMS-backed queues:
|
6
8
|
|
@@ -15,7 +17,7 @@ Additionally, there are the general benefits of storing jobs in Postgres, alongs
|
|
15
17
|
* **Fewer Dependencies** - If you're already using Postgres (and you probably should be), a separate queue is another moving part that can break.
|
16
18
|
* **Security** - Postgres' support for SSL connections keeps your data safe in transport, for added protection when you're running workers on cloud platforms that you can't completely control.
|
17
19
|
|
18
|
-
Que's primary goal is reliability. You should be able to leave your application running indefinitely without worrying about jobs being lost due to a lack of transactional support, or left in limbo due to a crashing process. Que does everything it can to ensure that jobs you queue are performed exactly once (though the occasional repetition of a job can be impossible to avoid - see the docs on [how to write a reliable job](
|
20
|
+
Que's primary goal is reliability. You should be able to leave your application running indefinitely without worrying about jobs being lost due to a lack of transactional support, or left in limbo due to a crashing process. Que does everything it can to ensure that jobs you queue are performed exactly once (though the occasional repetition of a job can be impossible to avoid - see the docs on [how to write a reliable job](/docs/README.md#writing-reliable-jobs)).
|
19
21
|
|
20
22
|
Que's secondary goal is performance. The worker process is multithreaded, so that a single process can run many jobs simultaneously.
|
21
23
|
|
@@ -112,23 +114,81 @@ You can also add options to run the job after a specific time, or with a specifi
|
|
112
114
|
``` ruby
|
113
115
|
ChargeCreditCard.enqueue card.id, user_id: current_user.id, run_at: 1.day.from_now, priority: 5
|
114
116
|
```
|
117
|
+
## Running the Que Worker
|
118
|
+
In order to process jobs, you must start a separate worker process outside of your main server.
|
119
|
+
|
120
|
+
```
|
121
|
+
bundle exec que
|
122
|
+
```
|
123
|
+
|
124
|
+
Try running `que -h` to get a list of runtime options:
|
125
|
+
```
|
126
|
+
$ que -h
|
127
|
+
usage: que [options] [file/to/require] ...
|
128
|
+
-h, --help Show this help text.
|
129
|
+
-i, --poll-interval [INTERVAL] Set maximum interval between polls for available jobs, in seconds (default: 5)
|
130
|
+
...
|
131
|
+
```
|
132
|
+
|
133
|
+
You may need to pass que a file path to require so that it can load your app. Que will automatically load `config/environment.rb` if it exists, so you shouldn't need an argument if you're using Rails.
|
134
|
+
|
135
|
+
## Additional Rails-specific Setup
|
136
|
+
|
137
|
+
If you're using ActiveRecord to dump your database's schema, please [set your schema_format to :sql](http://guides.rubyonrails.org/migrations.html#types-of-schema-dumps) so that Que's table structure is managed correctly. This is a good idea regardless, as the `:ruby` schema format doesn't support many of PostgreSQL's advanced features.
|
138
|
+
|
139
|
+
Pre-1.0, the default queue name needed to be configured in order for Que to work out of the box with Rails. In 1.0 the default queue name is now 'default', as Rails expects, but when Rails enqueues some types of jobs it may try to use another queue name that isn't worked by default. You can either:
|
140
|
+
|
141
|
+
* [Configure Rails](https://guides.rubyonrails.org/configuring.html) to send all internal job types to the 'default' queue by adding the following to `config/application.rb`:
|
142
|
+
```ruby
|
143
|
+
config.action_mailer.deliver_later_queue_name = :default
|
144
|
+
config.action_mailbox.queues.incineration = :default
|
145
|
+
config.action_mailbox.queues.routing = :default
|
146
|
+
config.active_storage.queues.analysis = :default
|
147
|
+
config.active_storage.queues.purge = :default
|
148
|
+
```
|
149
|
+
|
150
|
+
* [Tell que](/docs#multiple-queues) to work all of these queues (less efficient because it requires polling all of them):
|
151
|
+
```
|
152
|
+
que -q default -q mailers -q action_mailbox_incineration -q action_mailbox_routing -q active_storage_analysis -q active_storage_purge
|
153
|
+
```
|
154
|
+
|
155
|
+
Also, if you would like to integrate Que with Active Job, you can do it by setting the adapter in `config/application.rb` or in a specific environment by setting it in `config/environments/production.rb`, for example:
|
156
|
+
```ruby
|
157
|
+
config.active_job.queue_adapter = :que
|
158
|
+
```
|
159
|
+
|
160
|
+
Que will automatically use the database configuration of your rails application, so there is no need to configure anything else.
|
161
|
+
|
162
|
+
You can then write your jobs as usual following the [Active Job documentation](https://guides.rubyonrails.org/active_job_basics.html). However, be aware that you'll lose the ability to finish the job in the same transaction as other database operations. That happens because Active Job is a generic background job framework that doesn't benefit from the database integration Que provides.
|
163
|
+
|
164
|
+
If you later decide to switch a job from Active Job to Que to have transactional integrity you can easily change the corresponding job class to inherit from `Que::Job` and follow the usage guidelines in the previous section.
|
115
165
|
|
116
166
|
## Testing
|
117
167
|
|
118
168
|
There are a couple ways to do testing. You may want to set `Que::Job.run_synchronously = true`, which will cause JobClass.enqueue to simply execute the job's logic synchronously, as if you'd run JobClass.run(*your_args). Or, you may want to leave it disabled so you can assert on the job state once they are stored in the database.
|
119
169
|
|
120
|
-
|
170
|
+
## Documentation
|
171
|
+
|
172
|
+
**For full documentation, see [here](docs/README.md)**.
|
173
|
+
|
174
|
+
## Related Projects
|
121
175
|
|
176
|
+
These projects are tested to be compatible with Que 1.x:
|
177
|
+
|
178
|
+
- [que-web](https://github.com/statianzo/que-web) is a Sinatra-based UI for inspecting your job queue.
|
179
|
+
- [que-scheduler](https://github.com/hlascelles/que-scheduler) lets you schedule tasks using a cron style config file.
|
180
|
+
- [que-locks](https://github.com/airhorns/que-locks) lets you lock around job execution for so only one job runs at once for a set of arguments.
|
181
|
+
|
182
|
+
If you have a project that uses or relates to Que, feel free to submit a PR adding it to the list!
|
122
183
|
|
123
184
|
## Community and Contributing
|
124
185
|
|
125
|
-
* For bugs in the library, please feel free to [open an issue](https://github.com/
|
126
|
-
* For general discussion and questions/concerns that don't relate to obvious bugs,
|
186
|
+
* For bugs in the library, please feel free to [open an issue](https://github.com/que-rb/que/issues/new).
|
187
|
+
* For general discussion and questions/concerns that don't relate to obvious bugs, join our [Discord Server](https://discord.gg/B3EW32H).
|
127
188
|
* For contributions, pull requests submitted via Github are welcome.
|
128
189
|
|
129
190
|
Regarding contributions, one of the project's priorities is to keep Que as simple, lightweight and dependency-free as possible, and pull requests that change too much or wouldn't be useful to the majority of Que's users have a good chance of being rejected. If you're thinking of submitting a pull request that adds a new feature, consider starting a discussion in [que-talk](https://groups.google.com/forum/#!forum/que-talk) first about what it would do and how it would be implemented. If it's a sufficiently large feature, or if most of Que's users wouldn't find it useful, it may be best implemented as a standalone gem, like some of the related projects above.
|
130
191
|
|
131
|
-
|
132
192
|
### Specs
|
133
193
|
|
134
194
|
A note on running specs - Que's worker system is multithreaded and therefore prone to race conditions. As such, if you've touched that code, a single spec run passing isn't a guarantee that any changes you've made haven't introduced bugs. One thing I like to do before pushing changes is rerun the specs many times and watching for hangs. You can do this from the command line with something like:
|
@@ -18,12 +18,14 @@ module Que
|
|
18
18
|
default_require_file: RAILS_ENVIRONMENT_FILE
|
19
19
|
)
|
20
20
|
|
21
|
-
options
|
22
|
-
queues
|
23
|
-
log_level
|
24
|
-
log_internals
|
25
|
-
poll_interval
|
26
|
-
connection_url
|
21
|
+
options = {}
|
22
|
+
queues = []
|
23
|
+
log_level = 'info'
|
24
|
+
log_internals = false
|
25
|
+
poll_interval = 5
|
26
|
+
connection_url = nil
|
27
|
+
worker_count = nil
|
28
|
+
worker_priorities = nil
|
27
29
|
|
28
30
|
parser =
|
29
31
|
OptionParser.new do |opts|
|
@@ -58,6 +60,26 @@ module Que
|
|
58
60
|
log_level = l
|
59
61
|
end
|
60
62
|
|
63
|
+
opts.on(
|
64
|
+
'-p',
|
65
|
+
'--worker-priorities [LIST]',
|
66
|
+
Array,
|
67
|
+
"List of priorities to assign to workers (default: 10,30,50,any,any,any)",
|
68
|
+
) do |priority_array|
|
69
|
+
worker_priorities =
|
70
|
+
priority_array.map do |p|
|
71
|
+
case p
|
72
|
+
when /\Aany\z/i
|
73
|
+
nil
|
74
|
+
when /\A\d+\z/
|
75
|
+
Integer(p)
|
76
|
+
else
|
77
|
+
output.puts "Invalid priority option: '#{p}'. Please use an integer or the word 'any'."
|
78
|
+
return 1
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
61
83
|
opts.on(
|
62
84
|
'-q',
|
63
85
|
'--queue-name [NAME]',
|
@@ -69,6 +91,15 @@ module Que
|
|
69
91
|
queues << queue_name
|
70
92
|
end
|
71
93
|
|
94
|
+
opts.on(
|
95
|
+
'-w',
|
96
|
+
'--worker-count [COUNT]',
|
97
|
+
Integer,
|
98
|
+
"Set number of workers in process (default: 6)",
|
99
|
+
) do |w|
|
100
|
+
worker_count = w
|
101
|
+
end
|
102
|
+
|
72
103
|
opts.on(
|
73
104
|
'-v',
|
74
105
|
'--version',
|
@@ -79,21 +110,12 @@ module Que
|
|
79
110
|
return 0
|
80
111
|
end
|
81
112
|
|
82
|
-
opts.on(
|
83
|
-
'-w',
|
84
|
-
'--worker-count [COUNT]',
|
85
|
-
Integer,
|
86
|
-
"Set number of workers in process (default: 6)",
|
87
|
-
) do |w|
|
88
|
-
options[:worker_count] = w
|
89
|
-
end
|
90
|
-
|
91
113
|
opts.on(
|
92
114
|
'--connection-url [URL]',
|
93
115
|
String,
|
94
116
|
"Set a custom database url to connect to for locking purposes.",
|
95
117
|
) do |url|
|
96
|
-
connection_url = url
|
118
|
+
options[:connection_url] = url
|
97
119
|
end
|
98
120
|
|
99
121
|
opts.on(
|
@@ -107,19 +129,19 @@ module Que
|
|
107
129
|
opts.on(
|
108
130
|
'--maximum-buffer-size [SIZE]',
|
109
131
|
Integer,
|
110
|
-
"Set maximum number of jobs to be
|
111
|
-
"awaiting a worker (default: 8)",
|
132
|
+
"Set maximum number of jobs to be locked and held in this " \
|
133
|
+
"process awaiting a worker (default: 8)",
|
112
134
|
) do |s|
|
113
|
-
options[:
|
135
|
+
options[:maximum_buffer_size] = s
|
114
136
|
end
|
115
137
|
|
116
138
|
opts.on(
|
117
139
|
'--minimum-buffer-size [SIZE]',
|
118
140
|
Integer,
|
119
|
-
"Set minimum number of jobs to be
|
120
|
-
"awaiting a worker (default: 2)",
|
141
|
+
"Set minimum number of jobs to be locked and held in this " \
|
142
|
+
"process awaiting a worker (default: 2)",
|
121
143
|
) do |s|
|
122
|
-
options[:
|
144
|
+
options[:minimum_buffer_size] = s
|
123
145
|
end
|
124
146
|
|
125
147
|
opts.on(
|
@@ -130,19 +152,21 @@ module Que
|
|
130
152
|
) do |p|
|
131
153
|
options[:wait_period] = p
|
132
154
|
end
|
133
|
-
|
134
|
-
opts.on(
|
135
|
-
'--worker-priorities [LIST]',
|
136
|
-
Array,
|
137
|
-
"List of priorities to assign to workers, " \
|
138
|
-
"unspecified workers take jobs of any priority (default: 10,30,50)",
|
139
|
-
) do |p|
|
140
|
-
options[:worker_priorities] = p.map(&:to_i)
|
141
|
-
end
|
142
155
|
end
|
143
156
|
|
144
157
|
parser.parse!(args)
|
145
158
|
|
159
|
+
options[:worker_priorities] =
|
160
|
+
if worker_count && worker_priorities
|
161
|
+
worker_priorities.values_at(0...worker_count)
|
162
|
+
elsif worker_priorities
|
163
|
+
worker_priorities
|
164
|
+
elsif worker_count
|
165
|
+
Array.new(worker_count) { nil }
|
166
|
+
else
|
167
|
+
[10, 30, 50, nil, nil, nil]
|
168
|
+
end
|
169
|
+
|
146
170
|
if args.length.zero?
|
147
171
|
if File.exist?(default_require_file)
|
148
172
|
args << default_require_file
|
@@ -159,8 +183,8 @@ OUTPUT
|
|
159
183
|
args.each do |file|
|
160
184
|
begin
|
161
185
|
require file
|
162
|
-
rescue LoadError
|
163
|
-
output.puts "Could not load file '#{file}'"
|
186
|
+
rescue LoadError => e
|
187
|
+
output.puts "Could not load file '#{file}': #{e}"
|
164
188
|
return 1
|
165
189
|
end
|
166
190
|
end
|
@@ -172,7 +196,7 @@ OUTPUT
|
|
172
196
|
end
|
173
197
|
|
174
198
|
begin
|
175
|
-
Que.
|
199
|
+
Que.get_logger.level = Logger.const_get(log_level.upcase)
|
176
200
|
rescue NameError
|
177
201
|
output.puts "Unsupported logging level: #{log_level} (try debug, info, warn, error, or fatal)"
|
178
202
|
return 1
|
@@ -195,19 +219,6 @@ OUTPUT
|
|
195
219
|
|
196
220
|
options[:poll_interval] = poll_interval
|
197
221
|
|
198
|
-
if connection_url
|
199
|
-
uri = URI.parse(connection_url)
|
200
|
-
|
201
|
-
options[:connection] =
|
202
|
-
PG::Connection.open(
|
203
|
-
host: uri.host,
|
204
|
-
user: uri.user,
|
205
|
-
password: uri.password,
|
206
|
-
port: uri.port || 5432,
|
207
|
-
dbname: uri.path[1..-1],
|
208
|
-
)
|
209
|
-
end
|
210
|
-
|
211
222
|
locker =
|
212
223
|
begin
|
213
224
|
Que::Locker.new(options)
|
@@ -221,13 +232,14 @@ OUTPUT
|
|
221
232
|
$stop_que_executable = false
|
222
233
|
%w[INT TERM].each { |signal| trap(signal) { $stop_que_executable = true } }
|
223
234
|
|
235
|
+
output.puts "Que waiting for jobs..."
|
236
|
+
|
224
237
|
loop do
|
225
238
|
sleep 0.01
|
226
239
|
break if $stop_que_executable || locker.stopping?
|
227
240
|
end
|
228
241
|
|
229
|
-
output.puts '
|
230
|
-
output.puts "Finishing Que's current jobs before exiting..."
|
242
|
+
output.puts "\nFinishing Que's current jobs before exiting..."
|
231
243
|
|
232
244
|
locker.stop!
|
233
245
|
|