que 2.0.0 → 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/docs/README.md +26 -2
- data/lib/que/job.rb +122 -12
- data/lib/que/migrations/7/down.sql +5 -0
- data/lib/que/migrations/7/up.sql +13 -0
- data/lib/que/migrations.rb +1 -1
- data/lib/que/version.rb +1 -1
- data/lib/que.rb +1 -1
- data/scripts/test +1 -0
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: '08d50e63c19dbe61966e4048eb1e30fce181e520b05f20f933220baede9cbc76'
|
4
|
+
data.tar.gz: 72abbb3390310c18897739e35a2fdb5e6a8187a9b5ae27e4ee651ce4b7c8d9bf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aa542178d30c1fc031ae2508ff462d0b1ff23f2e5a4e6f9b3fb1cb08ec3ac3a3cc6c167e532654124b5d72c53a319c0b74c3f3a2eb0b5e66102242b43ad59a50
|
7
|
+
data.tar.gz: d3902890b359a759f9fe3c94f431afb512e312b6dc46d22223d0d3dc7154f21458a8c7592a72850318094af3c4bfe59950dc8f5bde68e314911695e39d19507a
|
data/CHANGELOG.md
CHANGED
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
<!-- MarkdownTOC autolink=true -->
|
4
4
|
|
5
|
+
- [2.1.0 \(2022-08-25\)](#210-2022-08-25)
|
5
6
|
- [2.0.0 \(2022-08-25\)](#200-2022-08-25)
|
6
7
|
- [1.4.1 \(2022-07-24\)](#141-2022-07-24)
|
7
8
|
- [2.0.0.beta1 \(2022-03-24\)](#200beta1-2022-03-24)
|
@@ -54,8 +55,31 @@
|
|
54
55
|
|
55
56
|
<!-- /MarkdownTOC -->
|
56
57
|
|
58
|
+
## 2.1.0 (2022-08-25)
|
59
|
+
|
60
|
+
- **Added**:
|
61
|
+
+ Added bulk enqueue interface for performance when enqueuing a large number of jobs at once - [docs](docs#enqueueing-jobs-in-bulk).
|
62
|
+
- **Deprecated**:
|
63
|
+
+ Deprecated `que_state_notify` trigger (`que_state` notification channel / `job_change` notification message). See [#372](https://github.com/que-rb/que/issues/372). We plan to remove this in a future release - let us know on the issue if you desire otherwise.
|
64
|
+
|
65
|
+
This release contains a database migration. You will need to migrate Que to the latest database schema version (7). For example, on ActiveRecord and Rails 6:
|
66
|
+
|
67
|
+
```ruby
|
68
|
+
class UpdateQueTablesToVersion6 < ActiveRecord::Migration[6.0]
|
69
|
+
def up
|
70
|
+
Que.migrate!(version: 7)
|
71
|
+
end
|
72
|
+
|
73
|
+
def down
|
74
|
+
Que.migrate!(version: 6)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
```
|
78
|
+
|
57
79
|
## 2.0.0 (2022-08-25)
|
58
80
|
|
81
|
+
**Important: Do not upgrade straight to Que 2.** You will need to first update to the latest 1.x version, apply the Que database schema migration, and deploy, before you can safely begin the process of upgrading to Que 2. See the [2.0.0.beta1 changelog entry](#200beta1-2022-03-24) for details.
|
82
|
+
|
59
83
|
See beta 2.0.0.beta1, plus:
|
60
84
|
|
61
85
|
- **Fixed**:
|
data/docs/README.md
CHANGED
@@ -53,6 +53,7 @@
|
|
53
53
|
- [Defining Middleware For Jobs](#defining-middleware-for-jobs)
|
54
54
|
- [Defining Middleware For SQL statements](#defining-middleware-for-sql-statements)
|
55
55
|
- [Vacuuming](#vacuuming)
|
56
|
+
- [Enqueueing jobs in bulk](#enqueueing-jobs-in-bulk)
|
56
57
|
- [Expired jobs](#expired-jobs)
|
57
58
|
- [Finished jobs](#finished-jobs)
|
58
59
|
|
@@ -836,6 +837,30 @@ class ManualVacuumJob < CronJob
|
|
836
837
|
end
|
837
838
|
```
|
838
839
|
|
840
|
+
## Enqueueing jobs in bulk
|
841
|
+
|
842
|
+
If you need to enqueue a large number of jobs at once, enqueueing each one separately (and running the notify trigger for each) can become a performance bottleneck. To mitigate this, there is a bulk enqueue interface:
|
843
|
+
|
844
|
+
```ruby
|
845
|
+
Que.bulk_enqueue do
|
846
|
+
MyJob.enqueue(user_id: 1)
|
847
|
+
MyJob.enqueue(user_id: 2)
|
848
|
+
# ...
|
849
|
+
end
|
850
|
+
```
|
851
|
+
|
852
|
+
The jobs are only actually enqueued at the end of the block, at which point they are inserted into the database in one big query.
|
853
|
+
|
854
|
+
Limitations:
|
855
|
+
|
856
|
+
- ActiveJob is not supported
|
857
|
+
- All jobs must use the same job class
|
858
|
+
- All jobs must use the same `job_options` (`job_options` must be provided to `.bulk_enqueue` instead of `.enqueue`)
|
859
|
+
- The `que_attrs` of a job instance returned from `.enqueue` is empty (`{}`)
|
860
|
+
- The notify trigger is not run by default, so jobs will only be picked up by a worker upon its next poll
|
861
|
+
|
862
|
+
If you still want the notify trigger to run for each job, use `Que.bulk_enqueue(notify: true) { ... }`.
|
863
|
+
|
839
864
|
## Expired jobs
|
840
865
|
|
841
866
|
Expired jobs hang around in the `que_jobs` table. If necessary, you can get an expired job to run again by clearing the `error_count` and `expired_at` columns, e.g.:
|
@@ -850,8 +875,7 @@ If you prefer to leave finished jobs in the database for a while, to performantl
|
|
850
875
|
|
851
876
|
```sql
|
852
877
|
BEGIN;
|
853
|
-
|
878
|
+
SET LOCAL que.skip_notify TO true;
|
854
879
|
DELETE FROM que_jobs WHERE finished_at < (select now() - interval '7 days');
|
855
|
-
ALTER TABLE que_jobs ENABLE TRIGGER que_state_notify;
|
856
880
|
COMMIT;
|
857
881
|
```
|
data/lib/que/job.rb
CHANGED
@@ -27,6 +27,26 @@ module Que
|
|
27
27
|
RETURNING *
|
28
28
|
}
|
29
29
|
|
30
|
+
SQL[:bulk_insert_jobs] =
|
31
|
+
%{
|
32
|
+
WITH args_and_kwargs as (
|
33
|
+
SELECT * from json_to_recordset(coalesce($5, '[{args:{},kwargs:{}}]')::json) as x(args jsonb, kwargs jsonb)
|
34
|
+
)
|
35
|
+
INSERT INTO public.que_jobs
|
36
|
+
(queue, priority, run_at, job_class, args, kwargs, data, job_schema_version)
|
37
|
+
SELECT
|
38
|
+
coalesce($1, 'default')::text,
|
39
|
+
coalesce($2, 100)::smallint,
|
40
|
+
coalesce($3, now())::timestamptz,
|
41
|
+
$4::text,
|
42
|
+
args_and_kwargs.args,
|
43
|
+
args_and_kwargs.kwargs,
|
44
|
+
coalesce($6, '{}')::jsonb,
|
45
|
+
#{Que.job_schema_version}
|
46
|
+
FROM args_and_kwargs
|
47
|
+
RETURNING *
|
48
|
+
}
|
49
|
+
|
30
50
|
attr_reader :que_attrs
|
31
51
|
attr_accessor :que_error, :que_resolved
|
32
52
|
|
@@ -78,30 +98,120 @@ module Que
|
|
78
98
|
queue: job_options[:queue] || resolve_que_setting(:queue) || Que.default_queue,
|
79
99
|
priority: job_options[:priority] || resolve_que_setting(:priority),
|
80
100
|
run_at: job_options[:run_at] || resolve_que_setting(:run_at),
|
81
|
-
args:
|
82
|
-
kwargs:
|
83
|
-
data: job_options[:tags] ?
|
101
|
+
args: args,
|
102
|
+
kwargs: kwargs,
|
103
|
+
data: job_options[:tags] ? { tags: job_options[:tags] } : {},
|
84
104
|
job_class: \
|
85
105
|
job_options[:job_class] || name ||
|
86
106
|
raise(Error, "Can't enqueue an anonymous subclass of Que::Job"),
|
87
107
|
}
|
88
108
|
|
89
|
-
if
|
90
|
-
|
91
|
-
|
92
|
-
|
109
|
+
if Thread.current[:que_jobs_to_bulk_insert]
|
110
|
+
if self.name == 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'
|
111
|
+
raise Que::Error, "Que.bulk_enqueue does not support ActiveJob."
|
112
|
+
end
|
113
|
+
|
114
|
+
raise Que::Error, "When using .bulk_enqueue, job_options must be passed to that method rather than .enqueue" unless job_options == {}
|
115
|
+
|
116
|
+
Thread.current[:que_jobs_to_bulk_insert][:jobs_attrs] << attrs
|
117
|
+
new({})
|
118
|
+
elsif attrs[:run_at].nil? && resolve_que_setting(:run_synchronously)
|
119
|
+
attrs.merge!(
|
120
|
+
args: Que.deserialize_json(Que.serialize_json(attrs[:args])),
|
121
|
+
kwargs: Que.deserialize_json(Que.serialize_json(attrs[:kwargs])),
|
122
|
+
data: Que.deserialize_json(Que.serialize_json(attrs[:data])),
|
123
|
+
)
|
93
124
|
_run_attrs(attrs)
|
94
125
|
else
|
95
|
-
|
96
|
-
Que.
|
97
|
-
|
98
|
-
|
99
|
-
|
126
|
+
attrs.merge!(
|
127
|
+
args: Que.serialize_json(attrs[:args]),
|
128
|
+
kwargs: Que.serialize_json(attrs[:kwargs]),
|
129
|
+
data: Que.serialize_json(attrs[:data]),
|
130
|
+
)
|
131
|
+
values = Que.execute(
|
132
|
+
:insert_job,
|
133
|
+
attrs.values_at(:queue, :priority, :run_at, :job_class, :args, :kwargs, :data),
|
134
|
+
).first
|
100
135
|
new(values)
|
101
136
|
end
|
102
137
|
end
|
103
138
|
ruby2_keywords(:enqueue) if respond_to?(:ruby2_keywords, true)
|
104
139
|
|
140
|
+
def bulk_enqueue(job_options: {}, notify: false)
|
141
|
+
raise Que::Error, "Can't nest .bulk_enqueue" unless Thread.current[:que_jobs_to_bulk_insert].nil?
|
142
|
+
Thread.current[:que_jobs_to_bulk_insert] = { jobs_attrs: [], job_options: job_options }
|
143
|
+
yield
|
144
|
+
jobs_attrs = Thread.current[:que_jobs_to_bulk_insert][:jobs_attrs]
|
145
|
+
job_options = Thread.current[:que_jobs_to_bulk_insert][:job_options]
|
146
|
+
return [] if jobs_attrs.empty?
|
147
|
+
raise Que::Error, "When using .bulk_enqueue, all jobs enqueued must be of the same job class" unless jobs_attrs.map { |attrs| attrs[:job_class] }.uniq.one?
|
148
|
+
args_and_kwargs_array = jobs_attrs.map { |attrs| attrs.slice(:args, :kwargs) }
|
149
|
+
klass = job_options[:job_class] ? Que::Job : Que.constantize(jobs_attrs.first[:job_class])
|
150
|
+
klass._bulk_enqueue_insert(args_and_kwargs_array, job_options: job_options, notify: notify)
|
151
|
+
ensure
|
152
|
+
Thread.current[:que_jobs_to_bulk_insert] = nil
|
153
|
+
end
|
154
|
+
|
155
|
+
def _bulk_enqueue_insert(args_and_kwargs_array, job_options: {}, notify:)
|
156
|
+
raise 'Unexpected bulk args format' if !args_and_kwargs_array.is_a?(Array) || !args_and_kwargs_array.all? { |a| a.is_a?(Hash) }
|
157
|
+
|
158
|
+
if job_options[:tags]
|
159
|
+
if job_options[:tags].length > MAXIMUM_TAGS_COUNT
|
160
|
+
raise Que::Error, "Can't enqueue a job with more than #{MAXIMUM_TAGS_COUNT} tags! (passed #{job_options[:tags].length})"
|
161
|
+
end
|
162
|
+
|
163
|
+
job_options[:tags].each do |tag|
|
164
|
+
if tag.length > MAXIMUM_TAG_LENGTH
|
165
|
+
raise Que::Error, "Can't enqueue a job with a tag longer than 100 characters! (\"#{tag}\")"
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
args_and_kwargs_array = args_and_kwargs_array.map do |args_and_kwargs|
|
171
|
+
args_and_kwargs.merge(
|
172
|
+
args: args_and_kwargs.fetch(:args, []),
|
173
|
+
kwargs: args_and_kwargs.fetch(:kwargs, {}),
|
174
|
+
)
|
175
|
+
end
|
176
|
+
|
177
|
+
attrs = {
|
178
|
+
queue: job_options[:queue] || resolve_que_setting(:queue) || Que.default_queue,
|
179
|
+
priority: job_options[:priority] || resolve_que_setting(:priority),
|
180
|
+
run_at: job_options[:run_at] || resolve_que_setting(:run_at),
|
181
|
+
args_and_kwargs_array: args_and_kwargs_array,
|
182
|
+
data: job_options[:tags] ? { tags: job_options[:tags] } : {},
|
183
|
+
job_class: \
|
184
|
+
job_options[:job_class] || name ||
|
185
|
+
raise(Error, "Can't enqueue an anonymous subclass of Que::Job"),
|
186
|
+
}
|
187
|
+
|
188
|
+
if attrs[:run_at].nil? && resolve_que_setting(:run_synchronously)
|
189
|
+
args_and_kwargs_array = Que.deserialize_json(Que.serialize_json(attrs.delete(:args_and_kwargs_array)))
|
190
|
+
args_and_kwargs_array.map do |args_and_kwargs|
|
191
|
+
_run_attrs(
|
192
|
+
attrs.merge(
|
193
|
+
args: args_and_kwargs.fetch(:args),
|
194
|
+
kwargs: args_and_kwargs.fetch(:kwargs),
|
195
|
+
),
|
196
|
+
)
|
197
|
+
end
|
198
|
+
else
|
199
|
+
attrs.merge!(
|
200
|
+
args_and_kwargs_array: Que.serialize_json(attrs[:args_and_kwargs_array]),
|
201
|
+
data: Que.serialize_json(attrs[:data]),
|
202
|
+
)
|
203
|
+
values_array =
|
204
|
+
Que.transaction do
|
205
|
+
Que.execute('SET LOCAL que.skip_notify TO true') unless notify
|
206
|
+
Que.execute(
|
207
|
+
:bulk_insert_jobs,
|
208
|
+
attrs.values_at(:queue, :priority, :run_at, :job_class, :args_and_kwargs_array, :data),
|
209
|
+
)
|
210
|
+
end
|
211
|
+
values_array.map(&method(:new))
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
105
215
|
def run(*args)
|
106
216
|
# Make sure things behave the same as they would have with a round-trip
|
107
217
|
# to the DB.
|
@@ -0,0 +1,13 @@
|
|
1
|
+
DROP TRIGGER que_job_notify ON que_jobs;
|
2
|
+
CREATE TRIGGER que_job_notify
|
3
|
+
AFTER INSERT ON que_jobs
|
4
|
+
FOR EACH ROW
|
5
|
+
WHEN (NOT coalesce(current_setting('que.skip_notify', true), '') = 'true')
|
6
|
+
EXECUTE PROCEDURE public.que_job_notify();
|
7
|
+
|
8
|
+
DROP TRIGGER que_state_notify ON que_jobs;
|
9
|
+
CREATE TRIGGER que_state_notify
|
10
|
+
AFTER INSERT OR UPDATE OR DELETE ON que_jobs
|
11
|
+
FOR EACH ROW
|
12
|
+
WHEN (NOT coalesce(current_setting('que.skip_notify', true), '') = 'true')
|
13
|
+
EXECUTE PROCEDURE public.que_state_notify();
|
data/lib/que/migrations.rb
CHANGED
data/lib/que/version.rb
CHANGED
data/lib/que.rb
CHANGED
@@ -69,7 +69,7 @@ module Que
|
|
69
69
|
|
70
70
|
# Copy some commonly-used methods here, for convenience.
|
71
71
|
def_delegators :pool, :execute, :checkout, :in_transaction?
|
72
|
-
def_delegators Job, :enqueue, :run_synchronously, :run_synchronously=
|
72
|
+
def_delegators Job, :enqueue, :bulk_enqueue, :run_synchronously, :run_synchronously=
|
73
73
|
def_delegators Migrations, :db_version, :migrate!
|
74
74
|
|
75
75
|
# Global configuration logic.
|
data/scripts/test
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: que
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chris Hanks
|
@@ -74,6 +74,8 @@ files:
|
|
74
74
|
- lib/que/migrations/5/up.sql
|
75
75
|
- lib/que/migrations/6/down.sql
|
76
76
|
- lib/que/migrations/6/up.sql
|
77
|
+
- lib/que/migrations/7/down.sql
|
78
|
+
- lib/que/migrations/7/up.sql
|
77
79
|
- lib/que/poller.rb
|
78
80
|
- lib/que/rails/railtie.rb
|
79
81
|
- lib/que/result_queue.rb
|