que 1.2.0 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +392 -525
- data/README.md +27 -1
- data/auto/pre-push-hook +30 -0
- data/bin/command_line_interface.rb +12 -4
- data/docs/README.md +11 -12
- data/lib/que/job.rb +3 -2
- data/lib/que/job_buffer.rb +1 -10
- data/lib/que/locker.rb +21 -11
- data/lib/que/migrations/4/up.sql +3 -1
- data/lib/que/migrations/5/down.sql +73 -0
- data/lib/que/migrations/5/up.sql +76 -0
- data/lib/que/migrations.rb +1 -2
- data/lib/que/poller.rb +10 -3
- data/lib/que/version.rb +5 -1
- data/lib/que.rb +2 -1
- metadata +6 -3
data/README.md
CHANGED
@@ -59,7 +59,7 @@ class CreateQueSchema < ActiveRecord::Migration[5.0]
|
|
59
59
|
# Whenever you use Que in a migration, always specify the version you're
|
60
60
|
# migrating to. If you're unsure what the current version is, check the
|
61
61
|
# changelog.
|
62
|
-
Que.migrate!(version:
|
62
|
+
Que.migrate!(version: 5)
|
63
63
|
end
|
64
64
|
|
65
65
|
def down
|
@@ -253,3 +253,29 @@ If you want to try a different version of Postgres, e.g. 12:
|
|
253
253
|
```bash
|
254
254
|
export POSTGRES_VERSION=12
|
255
255
|
```
|
256
|
+
|
257
|
+
### Git pre-push hook
|
258
|
+
|
259
|
+
So we can avoid breaking the build, we've created Git pre-push hooks to verify everything is ok before pushing.
|
260
|
+
|
261
|
+
To set up the pre-push hook locally, run:
|
262
|
+
|
263
|
+
```bash
|
264
|
+
echo -e "#\!/bin/bash\n\$(dirname \$0)/../../auto/pre-push-hook" > .git/hooks/pre-push
|
265
|
+
chmod +x .git/hooks/pre-push
|
266
|
+
```
|
267
|
+
|
268
|
+
### Release process
|
269
|
+
|
270
|
+
The process for releasing a new version of the gem is:
|
271
|
+
|
272
|
+
- Merge PR(s)
|
273
|
+
- Git pull locally
|
274
|
+
- Update the version number, bundle install, and commit
|
275
|
+
- Update `CHANGELOG.md`, and commit
|
276
|
+
- Tag the commit with the version number, prefixed by `v`
|
277
|
+
- Git push to master
|
278
|
+
- Git push the tag
|
279
|
+
- Publish the new version of the gem to RubyGems: `gem build -o que.gem && gem push que.gem`
|
280
|
+
- Create a GitHub release - rather than describe anything there, link to the heading for the release in `CHANGELOG.md`
|
281
|
+
- Post on the Que Discord in `#announcements`
|
data/auto/pre-push-hook
ADDED
@@ -0,0 +1,30 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
set -Eeuo pipefail
|
4
|
+
|
5
|
+
cd "$(dirname "$0")/.."
|
6
|
+
|
7
|
+
green='\e[32m'; blue='\e[36m'; red='\e[31m'; bold='\e[1m'; reset='\e[0m'
|
8
|
+
coloured-arrow() { printf "$bold$1==> $2$reset\n"; }
|
9
|
+
success() { coloured-arrow "$green" "$1"; }
|
10
|
+
info() { coloured-arrow "$blue" "$1"; }
|
11
|
+
err() { coloured-arrow "$red" "$1"; exit 1; }
|
12
|
+
|
13
|
+
info 'Running pre-push hook...'
|
14
|
+
|
15
|
+
on-exit() {
|
16
|
+
[[ -n "${succeeded-}" ]] || err 'Pre-push checks failed'
|
17
|
+
}
|
18
|
+
trap on-exit EXIT
|
19
|
+
|
20
|
+
info 'Checking for uncommitted changes...'
|
21
|
+
[[ -z $(git status -s) ]] || err 'ERROR: You have uncommited changes'
|
22
|
+
|
23
|
+
info 'Checking bundle...'
|
24
|
+
bundle check --dry-run || bundle install
|
25
|
+
|
26
|
+
info 'Specs...'
|
27
|
+
auto/test
|
28
|
+
|
29
|
+
succeeded=true
|
30
|
+
success 'All pre-push checks passed! =)'
|
@@ -138,10 +138,9 @@ module Que
|
|
138
138
|
opts.on(
|
139
139
|
'--minimum-buffer-size [SIZE]',
|
140
140
|
Integer,
|
141
|
-
"
|
142
|
-
"process awaiting a worker (default: 2)",
|
141
|
+
"Unused (deprecated)",
|
143
142
|
) do |s|
|
144
|
-
|
143
|
+
warn "The --minimum-buffer-size SIZE option has been deprecated and will be removed in v2.0 (it's actually been unused since v1.0.0.beta4)"
|
145
144
|
end
|
146
145
|
|
147
146
|
opts.on(
|
@@ -232,7 +231,16 @@ OUTPUT
|
|
232
231
|
$stop_que_executable = false
|
233
232
|
%w[INT TERM].each { |signal| trap(signal) { $stop_que_executable = true } }
|
234
233
|
|
235
|
-
output.puts
|
234
|
+
output.puts(
|
235
|
+
<<~STARTUP
|
236
|
+
Que #{Que::VERSION} started worker process with:
|
237
|
+
Worker threads: #{locker.workers.length} (priorities: #{locker.workers.map { |w| w.priority || 'any' }.join(', ')})
|
238
|
+
Buffer size: #{locker.job_buffer.maximum_size}
|
239
|
+
Queues:
|
240
|
+
#{locker.queues.map { |queue, interval| " - #{queue} (poll interval: #{interval}s)" }.join("\n")}
|
241
|
+
Que waiting for jobs...
|
242
|
+
STARTUP
|
243
|
+
)
|
236
244
|
|
237
245
|
loop do
|
238
246
|
sleep 0.01
|
data/docs/README.md
CHANGED
@@ -4,7 +4,7 @@ Docs Index
|
|
4
4
|
- [Command Line Interface](#command-line-interface)
|
5
5
|
* [worker-priorities and worker-count](#worker-priorities-and-worker-count)
|
6
6
|
* [poll-interval](#poll-interval)
|
7
|
-
* [
|
7
|
+
* [maximum-buffer-size](#maximum-buffer-size)
|
8
8
|
* [connection-url](#connection-url)
|
9
9
|
* [wait-period](#wait-period)
|
10
10
|
* [log-internals](#log-internals)
|
@@ -62,7 +62,6 @@ usage: que [options] [file/to/require] ...
|
|
62
62
|
--connection-url [URL] Set a custom database url to connect to for locking purposes.
|
63
63
|
--log-internals Log verbosely about Que's internal state. Only recommended for debugging issues
|
64
64
|
--maximum-buffer-size [SIZE] Set maximum number of jobs to be locked and held in this process awaiting a worker (default: 8)
|
65
|
-
--minimum-buffer-size [SIZE] Set minimum number of jobs to be locked and held in this process awaiting a worker (default: 2)
|
66
65
|
--wait-period [PERIOD] Set maximum interval between checks of the in-memory job queue, in milliseconds (default: 50)
|
67
66
|
```
|
68
67
|
|
@@ -82,9 +81,9 @@ If you pass both worker-count and worker-priorities, the count will trim or pad
|
|
82
81
|
|
83
82
|
This option sets the number of seconds the process will wait between polls of the job queue. Jobs that are ready to be worked immediately will be broadcast via the LISTEN/NOTIFY system, so polling is unnecessary for them - polling is only necessary for jobs that are scheduled in the future or which are being delayed due to errors. The default is 5 seconds.
|
84
83
|
|
85
|
-
###
|
84
|
+
### maximum-buffer-size
|
86
85
|
|
87
|
-
|
86
|
+
This option sets the size of the internal buffer that Que uses to hold jobs until they're ready for workers. The default maximum is 8, meaning that the process won't buffer more than 8 jobs that aren't yet ready to be worked. If you don't want jobs to be buffered at all, you can set this value to zero.
|
88
87
|
|
89
88
|
### connection-url
|
90
89
|
|
@@ -129,11 +128,11 @@ There are other docs to read if you're using [Sequel](#using-sequel) or [plain P
|
|
129
128
|
After you've connected Que to the database, you can manage the jobs table. You'll want to migrate to a specific version in a migration file, to ensure that they work the same way even when you upgrade Que in the future:
|
130
129
|
|
131
130
|
```ruby
|
132
|
-
# Update the schema to version #
|
133
|
-
Que.migrate!
|
131
|
+
# Update the schema to version #5.
|
132
|
+
Que.migrate!(version: 5)
|
134
133
|
|
135
134
|
# Remove Que's jobs table entirely.
|
136
|
-
Que.migrate!
|
135
|
+
Que.migrate!(version: 0)
|
137
136
|
```
|
138
137
|
|
139
138
|
There's also a helper method to clear all jobs from the jobs table:
|
@@ -405,11 +404,11 @@ Some new releases of Que may require updates to the database schema. It's recomm
|
|
405
404
|
```ruby
|
406
405
|
class UpdateQue < ActiveRecord::Migration[5.0]
|
407
406
|
def self.up
|
408
|
-
Que.migrate!
|
407
|
+
Que.migrate!(version: 3)
|
409
408
|
end
|
410
409
|
|
411
410
|
def self.down
|
412
|
-
Que.migrate!
|
411
|
+
Que.migrate!(version: 2)
|
413
412
|
end
|
414
413
|
end
|
415
414
|
```
|
@@ -418,7 +417,7 @@ This will make sure that your database schema stays consistent with your codebas
|
|
418
417
|
|
419
418
|
```ruby
|
420
419
|
# Change schema to version 3.
|
421
|
-
Que.migrate!
|
420
|
+
Que.migrate!(version: 3)
|
422
421
|
|
423
422
|
# Check your current schema version.
|
424
423
|
Que.db_version #=> 3
|
@@ -550,11 +549,11 @@ require 'que'
|
|
550
549
|
Sequel.migration do
|
551
550
|
up do
|
552
551
|
Que.connection = self
|
553
|
-
Que.migrate!
|
552
|
+
Que.migrate!(version: 5)
|
554
553
|
end
|
555
554
|
down do
|
556
555
|
Que.connection = self
|
557
|
-
Que.migrate!
|
556
|
+
Que.migrate!(version: 0)
|
558
557
|
end
|
559
558
|
end
|
560
559
|
```
|
data/lib/que/job.rb
CHANGED
@@ -12,7 +12,7 @@ module Que
|
|
12
12
|
SQL[:insert_job] =
|
13
13
|
%{
|
14
14
|
INSERT INTO public.que_jobs
|
15
|
-
(queue, priority, run_at, job_class, args, data)
|
15
|
+
(queue, priority, run_at, job_class, args, data, job_schema_version)
|
16
16
|
VALUES
|
17
17
|
(
|
18
18
|
coalesce($1, 'default')::text,
|
@@ -20,7 +20,8 @@ module Que
|
|
20
20
|
coalesce($3, now())::timestamptz,
|
21
21
|
$4::text,
|
22
22
|
coalesce($5, '[]')::jsonb,
|
23
|
-
coalesce($6, '{}')::jsonb
|
23
|
+
coalesce($6, '{}')::jsonb,
|
24
|
+
#{Que.job_schema_version}
|
24
25
|
)
|
25
26
|
RETURNING *
|
26
27
|
}
|
data/lib/que/job_buffer.rb
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
|
7
7
|
module Que
|
8
8
|
class JobBuffer
|
9
|
-
attr_reader :maximum_size, :
|
9
|
+
attr_reader :maximum_size, :priority_queues
|
10
10
|
|
11
11
|
# Since we use a mutex, which is not reentrant, we have to be a little
|
12
12
|
# careful to not call a method that locks the mutex when we've already
|
@@ -17,20 +17,11 @@ module Que
|
|
17
17
|
|
18
18
|
def initialize(
|
19
19
|
maximum_size:,
|
20
|
-
minimum_size:,
|
21
20
|
priorities:
|
22
21
|
)
|
23
22
|
@maximum_size = Que.assert(Integer, maximum_size)
|
24
23
|
Que.assert(maximum_size >= 0) { "maximum_size for a JobBuffer must be at least zero!" }
|
25
24
|
|
26
|
-
@minimum_size = Que.assert(Integer, minimum_size)
|
27
|
-
Que.assert(minimum_size >= 0) { "minimum_size for a JobBuffer must be at least zero!" }
|
28
|
-
|
29
|
-
Que.assert(minimum_size <= maximum_size) do
|
30
|
-
"minimum buffer size (#{minimum_size}) is " \
|
31
|
-
"greater than the maximum buffer size (#{maximum_size})!"
|
32
|
-
end
|
33
|
-
|
34
25
|
@stop = false
|
35
26
|
@array = []
|
36
27
|
@mutex = Mutex.new
|
data/lib/que/locker.rb
CHANGED
@@ -24,12 +24,12 @@ module Que
|
|
24
24
|
|
25
25
|
SQL[:register_locker] =
|
26
26
|
%{
|
27
|
-
INSERT INTO public.que_lockers (pid, worker_count, worker_priorities, ruby_pid, ruby_hostname, listening, queues)
|
28
|
-
VALUES (pg_backend_pid(), $1::integer, $2::integer[], $3::integer, $4::text, $5::boolean, $6::text[])
|
27
|
+
INSERT INTO public.que_lockers (pid, worker_count, worker_priorities, ruby_pid, ruby_hostname, listening, queues, job_schema_version)
|
28
|
+
VALUES (pg_backend_pid(), $1::integer, $2::integer[], $3::integer, $4::text, $5::boolean, $6::text[], $7::integer)
|
29
29
|
}
|
30
30
|
|
31
31
|
class Locker
|
32
|
-
attr_reader :thread, :workers, :job_buffer, :locks
|
32
|
+
attr_reader :thread, :workers, :job_buffer, :locks, :queues, :poll_interval
|
33
33
|
|
34
34
|
MESSAGE_RESOLVERS = {}
|
35
35
|
RESULT_RESOLVERS = {}
|
@@ -45,7 +45,6 @@ module Que
|
|
45
45
|
|
46
46
|
DEFAULT_POLL_INTERVAL = 5.0
|
47
47
|
DEFAULT_WAIT_PERIOD = 50
|
48
|
-
DEFAULT_MINIMUM_BUFFER_SIZE = 2
|
49
48
|
DEFAULT_MAXIMUM_BUFFER_SIZE = 8
|
50
49
|
DEFAULT_WORKER_PRIORITIES = [10, 30, 50, nil, nil, nil].freeze
|
51
50
|
|
@@ -57,7 +56,6 @@ module Que
|
|
57
56
|
poll_interval: DEFAULT_POLL_INTERVAL,
|
58
57
|
wait_period: DEFAULT_WAIT_PERIOD,
|
59
58
|
maximum_buffer_size: DEFAULT_MAXIMUM_BUFFER_SIZE,
|
60
|
-
minimum_buffer_size: DEFAULT_MINIMUM_BUFFER_SIZE,
|
61
59
|
worker_priorities: DEFAULT_WORKER_PRIORITIES,
|
62
60
|
on_worker_start: nil
|
63
61
|
)
|
@@ -77,7 +75,6 @@ module Que
|
|
77
75
|
# ResultQueue to receive messages from workers.
|
78
76
|
@job_buffer = JobBuffer.new(
|
79
77
|
maximum_size: maximum_buffer_size,
|
80
|
-
minimum_size: minimum_buffer_size,
|
81
78
|
priorities: worker_priorities.uniq,
|
82
79
|
)
|
83
80
|
|
@@ -93,7 +90,6 @@ module Que
|
|
93
90
|
poll_interval: poll_interval,
|
94
91
|
wait_period: wait_period,
|
95
92
|
maximum_buffer_size: maximum_buffer_size,
|
96
|
-
minimum_buffer_size: minimum_buffer_size,
|
97
93
|
worker_priorities: worker_priorities,
|
98
94
|
}
|
99
95
|
end
|
@@ -101,7 +97,20 @@ module Que
|
|
101
97
|
# Local cache of which advisory locks are held by this connection.
|
102
98
|
@locks = Set.new
|
103
99
|
|
104
|
-
@
|
100
|
+
@poll_interval = poll_interval
|
101
|
+
|
102
|
+
if queues.is_a?(Hash)
|
103
|
+
@queue_names = queues.keys
|
104
|
+
@queues = queues.transform_values do |interval|
|
105
|
+
interval || poll_interval
|
106
|
+
end
|
107
|
+
else
|
108
|
+
@queue_names = queues
|
109
|
+
@queues = queues.map do |queue_name|
|
110
|
+
[queue_name, poll_interval]
|
111
|
+
end.to_h
|
112
|
+
end
|
113
|
+
|
105
114
|
@wait_period = wait_period.to_f / 1000 # Milliseconds to seconds.
|
106
115
|
|
107
116
|
@workers =
|
@@ -183,11 +192,11 @@ module Que
|
|
183
192
|
|
184
193
|
@pollers =
|
185
194
|
if poll
|
186
|
-
queues.map do |
|
195
|
+
@queues.map do |queue_name, interval|
|
187
196
|
Poller.new(
|
188
197
|
connection: @connection,
|
189
|
-
queue:
|
190
|
-
poll_interval: interval
|
198
|
+
queue: queue_name,
|
199
|
+
poll_interval: interval,
|
191
200
|
)
|
192
201
|
end
|
193
202
|
end
|
@@ -266,6 +275,7 @@ module Que
|
|
266
275
|
CURRENT_HOSTNAME,
|
267
276
|
!!@listener,
|
268
277
|
"{\"#{@queue_names.join('","')}\"}",
|
278
|
+
Que.job_schema_version,
|
269
279
|
]
|
270
280
|
end
|
271
281
|
|
data/lib/que/migrations/4/up.sql
CHANGED
@@ -146,7 +146,9 @@ CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
|
|
146
146
|
FROM (
|
147
147
|
SELECT *
|
148
148
|
FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
|
149
|
-
WHERE
|
149
|
+
WHERE
|
150
|
+
listening AND
|
151
|
+
queues @> ARRAY[NEW.queue]
|
150
152
|
ORDER BY md5(pid::text || id::text)
|
151
153
|
) t1
|
152
154
|
) t2
|
@@ -0,0 +1,73 @@
|
|
1
|
+
DROP TRIGGER que_job_notify ON que_jobs;
|
2
|
+
DROP FUNCTION que_job_notify();
|
3
|
+
|
4
|
+
DROP INDEX que_poll_idx_with_job_schema_version;
|
5
|
+
|
6
|
+
ALTER TABLE que_jobs
|
7
|
+
DROP COLUMN job_schema_version;
|
8
|
+
|
9
|
+
ALTER TABLE que_lockers
|
10
|
+
DROP COLUMN job_schema_version;
|
11
|
+
|
12
|
+
CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
|
13
|
+
DECLARE
|
14
|
+
locker_pid integer;
|
15
|
+
sort_key json;
|
16
|
+
BEGIN
|
17
|
+
-- Don't do anything if the job is scheduled for a future time.
|
18
|
+
IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN
|
19
|
+
RETURN null;
|
20
|
+
END IF;
|
21
|
+
|
22
|
+
-- Pick a locker to notify of the job's insertion, weighted by their number
|
23
|
+
-- of workers. Should bounce pseudorandomly between lockers on each
|
24
|
+
-- invocation, hence the md5-ordering, but still touch each one equally,
|
25
|
+
-- hence the modulo using the job_id.
|
26
|
+
SELECT pid
|
27
|
+
INTO locker_pid
|
28
|
+
FROM (
|
29
|
+
SELECT *, last_value(row_number) OVER () + 1 AS count
|
30
|
+
FROM (
|
31
|
+
SELECT *, row_number() OVER () - 1 AS row_number
|
32
|
+
FROM (
|
33
|
+
SELECT *
|
34
|
+
FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
|
35
|
+
WHERE
|
36
|
+
listening AND
|
37
|
+
queues @> ARRAY[NEW.queue]
|
38
|
+
ORDER BY md5(pid::text || id::text)
|
39
|
+
) t1
|
40
|
+
) t2
|
41
|
+
) t3
|
42
|
+
WHERE NEW.id % count = row_number;
|
43
|
+
|
44
|
+
IF locker_pid IS NOT NULL THEN
|
45
|
+
-- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so
|
46
|
+
-- rather than throw errors when someone enqueues a big job, just
|
47
|
+
-- broadcast the most pertinent information, and let the locker query for
|
48
|
+
-- the record after it's taken the lock. The worker will have to hit the
|
49
|
+
-- DB in order to make sure the job is still visible anyway.
|
50
|
+
SELECT row_to_json(t)
|
51
|
+
INTO sort_key
|
52
|
+
FROM (
|
53
|
+
SELECT
|
54
|
+
'job_available' AS message_type,
|
55
|
+
NEW.queue AS queue,
|
56
|
+
NEW.priority AS priority,
|
57
|
+
NEW.id AS id,
|
58
|
+
-- Make sure we output timestamps as UTC ISO 8601
|
59
|
+
to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at
|
60
|
+
) t;
|
61
|
+
|
62
|
+
PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text);
|
63
|
+
END IF;
|
64
|
+
|
65
|
+
RETURN null;
|
66
|
+
END
|
67
|
+
$$
|
68
|
+
LANGUAGE plpgsql;
|
69
|
+
|
70
|
+
CREATE TRIGGER que_job_notify
|
71
|
+
AFTER INSERT ON que_jobs
|
72
|
+
FOR EACH ROW
|
73
|
+
EXECUTE PROCEDURE public.que_job_notify();
|
@@ -0,0 +1,76 @@
|
|
1
|
+
DROP TRIGGER que_job_notify ON que_jobs;
|
2
|
+
DROP FUNCTION que_job_notify();
|
3
|
+
|
4
|
+
ALTER TABLE que_jobs
|
5
|
+
ADD COLUMN job_schema_version INTEGER DEFAULT 1;
|
6
|
+
|
7
|
+
ALTER TABLE que_lockers
|
8
|
+
ADD COLUMN job_schema_version INTEGER DEFAULT 1;
|
9
|
+
|
10
|
+
CREATE INDEX que_poll_idx_with_job_schema_version
|
11
|
+
ON que_jobs (job_schema_version, queue, priority, run_at, id)
|
12
|
+
WHERE (finished_at IS NULL AND expired_at IS NULL);
|
13
|
+
|
14
|
+
CREATE FUNCTION que_job_notify() RETURNS trigger AS $$
|
15
|
+
DECLARE
|
16
|
+
locker_pid integer;
|
17
|
+
sort_key json;
|
18
|
+
BEGIN
|
19
|
+
-- Don't do anything if the job is scheduled for a future time.
|
20
|
+
IF NEW.run_at IS NOT NULL AND NEW.run_at > now() THEN
|
21
|
+
RETURN null;
|
22
|
+
END IF;
|
23
|
+
|
24
|
+
-- Pick a locker to notify of the job's insertion, weighted by their number
|
25
|
+
-- of workers. Should bounce pseudorandomly between lockers on each
|
26
|
+
-- invocation, hence the md5-ordering, but still touch each one equally,
|
27
|
+
-- hence the modulo using the job_id.
|
28
|
+
SELECT pid
|
29
|
+
INTO locker_pid
|
30
|
+
FROM (
|
31
|
+
SELECT *, last_value(row_number) OVER () + 1 AS count
|
32
|
+
FROM (
|
33
|
+
SELECT *, row_number() OVER () - 1 AS row_number
|
34
|
+
FROM (
|
35
|
+
SELECT *
|
36
|
+
FROM public.que_lockers ql, generate_series(1, ql.worker_count) AS id
|
37
|
+
WHERE
|
38
|
+
listening AND
|
39
|
+
queues @> ARRAY[NEW.queue] AND
|
40
|
+
ql.job_schema_version = NEW.job_schema_version
|
41
|
+
ORDER BY md5(pid::text || id::text)
|
42
|
+
) t1
|
43
|
+
) t2
|
44
|
+
) t3
|
45
|
+
WHERE NEW.id % count = row_number;
|
46
|
+
|
47
|
+
IF locker_pid IS NOT NULL THEN
|
48
|
+
-- There's a size limit to what can be broadcast via LISTEN/NOTIFY, so
|
49
|
+
-- rather than throw errors when someone enqueues a big job, just
|
50
|
+
-- broadcast the most pertinent information, and let the locker query for
|
51
|
+
-- the record after it's taken the lock. The worker will have to hit the
|
52
|
+
-- DB in order to make sure the job is still visible anyway.
|
53
|
+
SELECT row_to_json(t)
|
54
|
+
INTO sort_key
|
55
|
+
FROM (
|
56
|
+
SELECT
|
57
|
+
'job_available' AS message_type,
|
58
|
+
NEW.queue AS queue,
|
59
|
+
NEW.priority AS priority,
|
60
|
+
NEW.id AS id,
|
61
|
+
-- Make sure we output timestamps as UTC ISO 8601
|
62
|
+
to_char(NEW.run_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS run_at
|
63
|
+
) t;
|
64
|
+
|
65
|
+
PERFORM pg_notify('que_listener_' || locker_pid::text, sort_key::text);
|
66
|
+
END IF;
|
67
|
+
|
68
|
+
RETURN null;
|
69
|
+
END
|
70
|
+
$$
|
71
|
+
LANGUAGE plpgsql;
|
72
|
+
|
73
|
+
CREATE TRIGGER que_job_notify
|
74
|
+
AFTER INSERT ON que_jobs
|
75
|
+
FOR EACH ROW
|
76
|
+
EXECUTE PROCEDURE public.que_job_notify();
|
data/lib/que/migrations.rb
CHANGED
@@ -4,7 +4,7 @@ module Que
|
|
4
4
|
module Migrations
|
5
5
|
# In order to ship a schema change, add the relevant up and down sql files
|
6
6
|
# to the migrations directory, and bump the version here.
|
7
|
-
CURRENT_VERSION =
|
7
|
+
CURRENT_VERSION = 5
|
8
8
|
|
9
9
|
class << self
|
10
10
|
def migrate!(version:)
|
@@ -28,7 +28,6 @@ module Que
|
|
28
28
|
step,
|
29
29
|
direction,
|
30
30
|
].join('/') << '.sql'
|
31
|
-
|
32
31
|
Que.execute(File.read(filename))
|
33
32
|
end
|
34
33
|
|
data/lib/que/poller.rb
CHANGED
@@ -68,6 +68,7 @@ module Que
|
|
68
68
|
SELECT j
|
69
69
|
FROM public.que_jobs AS j
|
70
70
|
WHERE queue = $1::text
|
71
|
+
AND job_schema_version = #{Que.job_schema_version}
|
71
72
|
AND NOT id = ANY($2::bigint[])
|
72
73
|
AND priority <= pg_temp.que_highest_remaining_priority($3::jsonb)
|
73
74
|
AND run_at <= now()
|
@@ -88,6 +89,7 @@ module Que
|
|
88
89
|
SELECT j
|
89
90
|
FROM public.que_jobs AS j
|
90
91
|
WHERE queue = $1::text
|
92
|
+
AND job_schema_version = #{Que.job_schema_version}
|
91
93
|
AND NOT id = ANY($2::bigint[])
|
92
94
|
AND priority <= pg_temp.que_highest_remaining_priority(jobs.remaining_priorities)
|
93
95
|
AND run_at <= now()
|
@@ -144,8 +146,6 @@ module Que
|
|
144
146
|
|
145
147
|
return unless should_poll?
|
146
148
|
|
147
|
-
expected_count = priorities.inject(0){|s,(_,c)| s + c}
|
148
|
-
|
149
149
|
jobs =
|
150
150
|
connection.execute_prepared(
|
151
151
|
:poll_jobs,
|
@@ -157,7 +157,7 @@ module Que
|
|
157
157
|
)
|
158
158
|
|
159
159
|
@last_polled_at = Time.now
|
160
|
-
@last_poll_satisfied =
|
160
|
+
@last_poll_satisfied = poll_satisfied?(priorities, jobs)
|
161
161
|
|
162
162
|
Que.internal_log :poller_polled, self do
|
163
163
|
{
|
@@ -263,5 +263,12 @@ module Que
|
|
263
263
|
SQL
|
264
264
|
end
|
265
265
|
end
|
266
|
+
|
267
|
+
private
|
268
|
+
|
269
|
+
def poll_satisfied?(priorities, jobs)
|
270
|
+
lowest_priority = priorities.keys.max
|
271
|
+
jobs.count >= priorities[lowest_priority]
|
272
|
+
end
|
266
273
|
end
|
267
274
|
end
|
data/lib/que/version.rb
CHANGED
data/lib/que.rb
CHANGED
@@ -31,6 +31,8 @@ module Que
|
|
31
31
|
require_relative 'que/utils/queue_management'
|
32
32
|
require_relative 'que/utils/transactions'
|
33
33
|
|
34
|
+
require_relative 'que/version'
|
35
|
+
|
34
36
|
require_relative 'que/connection'
|
35
37
|
require_relative 'que/connection_pool'
|
36
38
|
require_relative 'que/job_methods'
|
@@ -41,7 +43,6 @@ module Que
|
|
41
43
|
require_relative 'que/migrations'
|
42
44
|
require_relative 'que/poller'
|
43
45
|
require_relative 'que/result_queue'
|
44
|
-
require_relative 'que/version'
|
45
46
|
require_relative 'que/worker'
|
46
47
|
|
47
48
|
class << self
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Hanks
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-03-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -40,6 +40,7 @@ files:
|
|
40
40
|
- README.md
|
41
41
|
- Rakefile
|
42
42
|
- auto/dev
|
43
|
+
- auto/pre-push-hook
|
43
44
|
- auto/psql
|
44
45
|
- auto/test
|
45
46
|
- auto/test-postgres-14
|
@@ -68,6 +69,8 @@ files:
|
|
68
69
|
- lib/que/migrations/3/up.sql
|
69
70
|
- lib/que/migrations/4/down.sql
|
70
71
|
- lib/que/migrations/4/up.sql
|
72
|
+
- lib/que/migrations/5/down.sql
|
73
|
+
- lib/que/migrations/5/up.sql
|
71
74
|
- lib/que/poller.rb
|
72
75
|
- lib/que/rails/railtie.rb
|
73
76
|
- lib/que/result_queue.rb
|
@@ -106,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
106
109
|
- !ruby/object:Gem::Version
|
107
110
|
version: '0'
|
108
111
|
requirements: []
|
109
|
-
rubygems_version: 3.
|
112
|
+
rubygems_version: 3.3.7
|
110
113
|
signing_key:
|
111
114
|
specification_version: 4
|
112
115
|
summary: A PostgreSQL-based Job Queue
|