que 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -84,7 +84,7 @@ describe Que::Worker do
84
84
  @worker = Que::Worker.new
85
85
 
86
86
  $q1.pop
87
- @worker.stop!
87
+ @worker.stop
88
88
  $q2.push nil
89
89
 
90
90
  @worker.wait_until_stopped
@@ -105,7 +105,7 @@ describe Que::Worker do
105
105
  @worker = Que::Worker.new
106
106
  sleep_until { @worker.sleeping? }
107
107
 
108
- @worker.stop!
108
+ @worker.stop
109
109
  @worker.wait_until_stopped
110
110
  ensure
111
111
  if @worker
@@ -114,4 +114,23 @@ describe Que::Worker do
114
114
  end
115
115
  end
116
116
  end
117
+
118
+ it "should receive and respect a notification to kill its job and stop running immediately" do
119
+ begin
120
+ # Worker#stop! can leave the database connection in an unpredictable
121
+ # state, which would impact the rest of the tests, so we need a special
122
+ # connection for it.
123
+ pg = NEW_PG_CONNECTION.call
124
+ Que.connection = pg
125
+
126
+ BlockJob.queue
127
+
128
+ @worker = Que::Worker.new
129
+ $q1.pop
130
+ @worker.stop!
131
+ @worker.wait_until_stopped
132
+ ensure
133
+ pg.close if pg
134
+ end
135
+ end
117
136
  end
@@ -1,93 +1 @@
1
- task :benchmark do
2
- # The following benchmark is meant to test Que's scalability by having it
3
- # bombard Postgres from many, many processes. Note that this benchmark tests
4
- # the entire Que stack, not just the locking queries.
5
-
6
- JOB_COUNT = (ENV['JOB_COUNT'] || 1000).to_i
7
- PROCESS_COUNT = (ENV['PROCESS_COUNT'] || 1).to_i
8
- WORKER_COUNT = (ENV['WORKER_COUNT'] || 4).to_i
9
- SYNCHRONOUS_COMMIT = ENV['SYNCHRONOUS_COMMIT'] || 'on'
10
-
11
- require 'que'
12
- require 'uri'
13
- require 'pg'
14
- require 'connection_pool'
15
-
16
- uri = URI.parse ENV["DATABASE_URL"] || "postgres://postgres:@localhost/que-test"
17
-
18
- new_connection = proc do
19
- conn = PG::Connection.open :host => uri.host,
20
- :user => uri.user,
21
- :password => uri.password,
22
- :port => uri.port || 5432,
23
- :dbname => uri.path[1..-1]
24
-
25
- conn.async_exec "SET SESSION synchronous_commit = #{SYNCHRONOUS_COMMIT}"
26
- conn
27
- end
28
-
29
- Que.connection = pg = new_connection.call
30
- Que.drop! rescue nil
31
- Que.create!
32
-
33
- # Stock table with jobs and analyze.
34
- pg.async_exec <<-SQL
35
- INSERT INTO que_jobs (job_class, args, priority)
36
- SELECT 'Que::Job', ('[' || i || ',{}]')::json, 1
37
- FROM generate_Series(1,#{JOB_COUNT}) AS i;
38
- ANALYZE;
39
- SQL
40
-
41
- # Fork!
42
- $parent_pid = Process.pid
43
- def parent?
44
- Process.pid == $parent_pid
45
- end
46
-
47
- # Synchronize all workers to start at the same time using, what else?
48
- # Advisory locks. I am such a one-trick pony.
49
- pg.async_exec("SELECT pg_advisory_lock(0)")
50
-
51
- PROCESS_COUNT.times { Process.fork if parent? }
52
-
53
- if parent?
54
- # This is the main process, get ready to start monitoring the queues.
55
-
56
- # First hold until all the children are ready.
57
- sleep 0.1 until pg.async_exec("select count(*) from pg_locks where locktype = 'advisory' and objid = 0").first['count'].to_i == PROCESS_COUNT + 1
58
-
59
- puts "Benchmarking: #{JOB_COUNT} jobs, #{PROCESS_COUNT} processes with #{WORKER_COUNT} workers each, synchronous_commit = #{SYNCHRONOUS_COMMIT}..."
60
- pg.async_exec("select pg_advisory_unlock_all()") # Go!
61
- start = Time.now
62
-
63
- loop do
64
- sleep 0.01
65
- break if pg.async_exec("SELECT 1 AS one FROM que_jobs LIMIT 1").none? # There must be a better way to do this?
66
- end
67
- time = Time.now - start
68
-
69
- locks = pg.async_exec("SELECT * FROM pg_locks WHERE locktype = 'advisory'").to_a
70
- puts "Advisory locks left over! #{locks.inspect}" if locks.any?
71
- puts "#{JOB_COUNT} jobs in #{time} seconds = #{(JOB_COUNT / time).round} jobs per second"
72
- else
73
- # This is a child, get ready to start hitting the queues.
74
- pool = ConnectionPool.new :size => WORKER_COUNT, &new_connection
75
-
76
- Que.connection = pool
77
-
78
- pool.with do |conn|
79
- # Block here until the advisory lock is released, which is our start pistol.
80
- conn.async_exec "SELECT pg_advisory_lock(0); SELECT pg_advisory_unlock_all();"
81
- end
82
-
83
- Que.mode = :async
84
- Que.worker_count = WORKER_COUNT
85
-
86
- loop do
87
- sleep 1
88
- break if pool.with { |pg| pg.async_exec("SELECT 1 AS one FROM que_jobs LIMIT 1").none? }
89
- end
90
- end
91
-
92
- Process.waitall
93
- end
1
+ # The benchmarking tasks have been merged together and now live at github.com/chanks/queue-shootout.
@@ -0,0 +1,63 @@
1
+ # This task is used to test Que's behavior when its process is shut down.
2
+
3
+ # The situation we're trying to avoid occurs when the process dies while a job
4
+ # is in the middle of a transaction - ideally, the transaction would be rolled
5
+ # back and the job could just be reattempted later, but if we're not careful,
6
+ # the transaction could be committed too early. For specifics, see this post:
7
+
8
+ # http://coderrr.wordpress.com/2011/05/03/beware-of-threadkill-or-your-activerecord-transactions-are-in-danger-of-being-partially-committed/
9
+
10
+ # So, this task opens a transaction within a job, makes a write, then prompts
11
+ # you to kill it with one of a few signals. It also checks to see whether the
12
+ # previous shutdown was clean or not.
13
+
14
+ task :safe_shutdown do
15
+ require 'sequel'
16
+ require 'que'
17
+
18
+ url = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
19
+ DB = Sequel.connect(url)
20
+
21
+ if DB.table_exists?(:que_jobs)
22
+ puts "Uh-oh! Previous shutdown wasn't clean!" if DB[:que_jobs].where(:job_id => 0).count > 0
23
+ DB.drop_table :que_jobs
24
+ end
25
+
26
+ Que.connection = DB
27
+ Que.create!
28
+
29
+ $queue = Queue.new
30
+
31
+ class SafeJob < Que::Job
32
+ def run
33
+ DB.transaction do
34
+ DB[:que_jobs].insert(:job_id => 0, :job_class => 'Que::Job')
35
+ $queue.push nil
36
+ sleep
37
+ end
38
+ end
39
+ end
40
+
41
+ SafeJob.queue
42
+ Que.mode = :async
43
+ $queue.pop
44
+
45
+ puts "From a different terminal window, run one of the following:"
46
+ %w(SIGINT SIGTERM SIGKILL).each do |signal|
47
+ puts "kill -#{signal} #{Process.pid}"
48
+ end
49
+
50
+ # Put exit behavior to test the behavior of here:
51
+ at_exit do
52
+ puts "Stopping Que..."
53
+ Que.stop!
54
+ end
55
+
56
+ stop = false
57
+ trap('INT'){stop = true}
58
+
59
+ loop do
60
+ sleep 0.01
61
+ break if stop
62
+ end
63
+ end
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: que
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chris Hanks
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2013-11-30 00:00:00.000000000 Z
11
+ date: 2013-12-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
15
- version_requirements: !ruby/object:Gem::Requirement
15
+ requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: '1.3'
20
- requirement: !ruby/object:Gem::Requirement
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
21
23
  requirements:
22
- - - ~>
24
+ - - "~>"
23
25
  - !ruby/object:Gem::Version
24
26
  version: '1.3'
25
- prerelease: false
26
- type: :development
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: multi_json
29
- version_requirements: !ruby/object:Gem::Requirement
29
+ requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: '1.0'
34
- requirement: !ruby/object:Gem::Requirement
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
35
37
  requirements:
36
- - - ~>
38
+ - - "~>"
37
39
  - !ruby/object:Gem::Version
38
40
  version: '1.0'
39
- prerelease: false
40
- type: :runtime
41
41
  description: A job queue that uses PostgreSQL's advisory locks for speed and reliability.
42
42
  email:
43
43
  - christopher.m.hanks@gmail.com
@@ -45,8 +45,8 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
- - .gitignore
49
- - .rspec
48
+ - ".gitignore"
49
+ - ".rspec"
50
50
  - CHANGELOG.md
51
51
  - Gemfile
52
52
  - LICENSE.txt
@@ -83,30 +83,30 @@ files:
83
83
  - spec/unit/work_spec.rb
84
84
  - spec/unit/worker_spec.rb
85
85
  - tasks/benchmark.rb
86
- - tasks/benchmark_queues.rb
87
86
  - tasks/rspec.rb
87
+ - tasks/safe_shutdown.rb
88
88
  homepage: https://github.com/chanks/que
89
89
  licenses:
90
90
  - MIT
91
91
  metadata: {}
92
- post_install_message:
92
+ post_install_message:
93
93
  rdoc_options: []
94
94
  require_paths:
95
95
  - lib
96
96
  required_ruby_version: !ruby/object:Gem::Requirement
97
97
  requirements:
98
- - - '>='
98
+ - - ">="
99
99
  - !ruby/object:Gem::Version
100
100
  version: '0'
101
101
  required_rubygems_version: !ruby/object:Gem::Requirement
102
102
  requirements:
103
- - - '>='
103
+ - - ">="
104
104
  - !ruby/object:Gem::Version
105
105
  version: '0'
106
106
  requirements: []
107
- rubyforge_project:
107
+ rubyforge_project:
108
108
  rubygems_version: 2.1.5
109
- signing_key:
109
+ signing_key:
110
110
  specification_version: 4
111
111
  summary: A PostgreSQL-based Job Queue
112
112
  test_files:
@@ -1,398 +0,0 @@
1
- task :benchmark_queues do
2
- # The following is a somewhat simplistic benchmark (aren't they all) meant
3
- # to compare the speed and concurrency of the locking mechanisms used by Que
4
- # (standard and lateral queries), DelayedJob and QueueClassic - it does this
5
- # by simply sending the raw queries that each system sends during operation.
6
-
7
- # It is NOT meant to benchmark the overall performance of each system (which
8
- # would include the time each spends working in Ruby), but to see which one
9
- # supports the highest concurrency under load, assuming that there will be
10
- # many workers and that Postgres will be the bottleneck. I'm unsure how
11
- # useful it is for this, but it's a start.
12
-
13
- JOB_COUNT = (ENV['JOB_COUNT'] || 1000).to_i
14
- WORKER_COUNT = (ENV['WORKER_COUNT'] || 10).to_i
15
- SYNCHRONOUS_COMMIT = ENV['SYNCHRONOUS_COMMIT'] || 'on'
16
-
17
- require 'uri'
18
- require 'pg'
19
- require 'connection_pool'
20
-
21
- uri = URI.parse ENV["DATABASE_URL"] || "postgres://postgres:@localhost/que-test"
22
-
23
- new_connection = proc do
24
- PG::Connection.open :host => uri.host,
25
- :user => uri.user,
26
- :password => uri.password,
27
- :port => uri.port || 5432,
28
- :dbname => uri.path[1..-1]
29
- end
30
-
31
- pg = new_connection.call
32
-
33
-
34
-
35
- # Necessary setup, mostly for QueueClassic. I apologize for this - I hope your editor supports code folding.
36
- pg.async_exec <<-SQL
37
- SET SESSION client_min_messages = 'WARNING';
38
-
39
- -- Que table.
40
- DROP TABLE IF EXISTS que_jobs;
41
- CREATE TABLE que_jobs
42
- (
43
- priority integer NOT NULL DEFAULT 1,
44
- run_at timestamptz NOT NULL DEFAULT now(),
45
- job_id bigserial NOT NULL,
46
- job_class text NOT NULL,
47
- args json NOT NULL DEFAULT '[]'::json,
48
- error_count integer NOT NULL DEFAULT 0,
49
- last_error text,
50
-
51
- CONSTRAINT que_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
52
- );
53
-
54
- DROP TABLE IF EXISTS que_lateral_jobs;
55
- CREATE TABLE que_lateral_jobs
56
- (
57
- priority integer NOT NULL DEFAULT 1,
58
- run_at timestamptz NOT NULL DEFAULT now(),
59
- job_id bigserial NOT NULL,
60
- job_class text NOT NULL,
61
- args json NOT NULL DEFAULT '[]'::json,
62
- error_count integer NOT NULL DEFAULT 0,
63
- last_error text,
64
-
65
- CONSTRAINT que_lateral_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
66
- );
67
-
68
- DROP TABLE IF EXISTS delayed_jobs;
69
- -- DelayedJob table.
70
- CREATE TABLE delayed_jobs
71
- (
72
- id serial NOT NULL,
73
- priority integer NOT NULL DEFAULT 0,
74
- attempts integer NOT NULL DEFAULT 0,
75
- handler text NOT NULL,
76
- last_error text,
77
- run_at timestamp without time zone,
78
- locked_at timestamp without time zone,
79
- failed_at timestamp without time zone,
80
- locked_by character varying(255),
81
- queue character varying(255),
82
- created_at timestamp without time zone,
83
- updated_at timestamp without time zone,
84
- CONSTRAINT delayed_jobs_pkey PRIMARY KEY (id)
85
- );
86
- ALTER TABLE delayed_jobs
87
- OWNER TO postgres;
88
-
89
- CREATE INDEX delayed_jobs_priority
90
- ON delayed_jobs
91
- USING btree
92
- (priority, run_at);
93
-
94
-
95
-
96
- -- QueueClassic table and functions.
97
- DROP FUNCTION IF EXISTS lock_head(tname varchar);
98
- DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer);
99
- DROP FUNCTION IF EXISTS queue_classic_notify() cascade;
100
- DROP TABLE IF EXISTS queue_classic_jobs;
101
-
102
- CREATE TABLE queue_classic_jobs (
103
- id bigserial PRIMARY KEY,
104
- q_name text not null check (length(q_name) > 0),
105
- method text not null check (length(method) > 0),
106
- args text not null,
107
- locked_at timestamptz
108
- );
109
-
110
- alter table queue_classic_jobs alter column args type json using (args::json);
111
-
112
- create function queue_classic_notify() returns trigger as $$ begin
113
- perform pg_notify(new.q_name, '');
114
- return null;
115
- end $$ language plpgsql;
116
-
117
- create trigger queue_classic_notify
118
- after insert on queue_classic_jobs
119
- for each row
120
- execute procedure queue_classic_notify();
121
-
122
- CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
123
-
124
- CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
125
- RETURNS SETOF queue_classic_jobs AS $$
126
- DECLARE
127
- unlocked bigint;
128
- relative_top integer;
129
- job_count integer;
130
- BEGIN
131
- -- The purpose is to release contention for the first spot in the table.
132
- -- The select count(*) is going to slow down dequeue performance but allow
133
- -- for more workers. Would love to see some optimization here...
134
-
135
- EXECUTE 'SELECT count(*) FROM '
136
- || '(SELECT * FROM queue_classic_jobs WHERE q_name = '
137
- || quote_literal(q_name)
138
- || ' LIMIT '
139
- || quote_literal(top_boundary)
140
- || ') limited'
141
- INTO job_count;
142
-
143
- SELECT TRUNC(random() * (top_boundary - 1))
144
- INTO relative_top;
145
-
146
- IF job_count < top_boundary THEN
147
- relative_top = 0;
148
- END IF;
149
-
150
- LOOP
151
- BEGIN
152
- EXECUTE 'SELECT id FROM queue_classic_jobs '
153
- || ' WHERE locked_at IS NULL'
154
- || ' AND q_name = '
155
- || quote_literal(q_name)
156
- || ' ORDER BY id ASC'
157
- || ' LIMIT 1'
158
- || ' OFFSET ' || quote_literal(relative_top)
159
- || ' FOR UPDATE NOWAIT'
160
- INTO unlocked;
161
- EXIT;
162
- EXCEPTION
163
- WHEN lock_not_available THEN
164
- -- do nothing. loop again and hope we get a lock
165
- END;
166
- END LOOP;
167
-
168
- RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
169
- || ' SET locked_at = (CURRENT_TIMESTAMP)'
170
- || ' WHERE id = $1'
171
- || ' AND locked_at is NULL'
172
- || ' RETURNING *'
173
- USING unlocked;
174
-
175
- RETURN;
176
- END;
177
- $$ LANGUAGE plpgsql;
178
-
179
- CREATE OR REPLACE FUNCTION lock_head(tname varchar)
180
- RETURNS SETOF queue_classic_jobs AS $$
181
- BEGIN
182
- RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
183
- END;
184
- $$ LANGUAGE plpgsql;
185
-
186
-
187
-
188
-
189
-
190
- INSERT INTO que_jobs (job_class, args, priority)
191
- SELECT 'Que::Job', ('[' || i || ',{}]')::json, 1
192
- FROM generate_Series(1,#{JOB_COUNT}) AS i;
193
-
194
- INSERT INTO que_lateral_jobs (job_class, args, priority)
195
- SELECT 'Que::Job', ('[' || i || ',{}]')::json, 1
196
- FROM generate_Series(1,#{JOB_COUNT}) AS i;
197
-
198
- INSERT INTO delayed_jobs (handler, run_at, created_at, updated_at)
199
- SELECT '--- !ruby/struct:NewsletterJob\ntext: lorem ipsum...\nemails: blah@blah.com\n', now(), now(), now()
200
- FROM generate_Series(1,#{JOB_COUNT}) AS i;
201
-
202
- INSERT INTO queue_classic_jobs (q_name, method, args)
203
- SELECT 'default', 'Kernel.puts', '["hello world"]'
204
- FROM generate_Series(1,#{JOB_COUNT}) AS i;
205
-
206
-
207
-
208
-
209
- -- Necessary tables and functions made, now stock them with jobs and analyze.
210
- ANALYZE;
211
- SQL
212
-
213
-
214
- queries = {
215
- :que => (
216
- <<-SQL
217
- WITH RECURSIVE cte AS (
218
- SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
219
- FROM (
220
- SELECT job
221
- FROM que_jobs AS job
222
- WHERE run_at <= now()
223
- ORDER BY priority, run_at, job_id
224
- LIMIT 1
225
- ) AS t1
226
- UNION ALL (
227
- SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
228
- FROM (
229
- SELECT (
230
- SELECT job
231
- FROM que_jobs AS job
232
- WHERE run_at <= now() AND (priority, run_at, job_id) > (cte.priority, cte.run_at, cte.job_id)
233
- ORDER BY priority, run_at, job_id
234
- LIMIT 1
235
- ) AS job
236
- FROM cte
237
- WHERE NOT cte.locked
238
- LIMIT 1
239
- ) AS t1
240
- )
241
- )
242
- SELECT job_id, priority, run_at, args, job_class, error_count
243
- FROM cte
244
- WHERE locked
245
- LIMIT 1
246
- SQL
247
- ),
248
- :que_lateral => (
249
- <<-SQL
250
- WITH RECURSIVE cte AS (
251
- SELECT *, pg_try_advisory_lock(s.job_id) AS locked
252
- FROM (
253
- SELECT *
254
- FROM que_lateral_jobs
255
- WHERE run_at <= now()
256
- ORDER BY priority, run_at, job_id
257
- LIMIT 1
258
- ) s
259
- UNION ALL (
260
- SELECT j.*, pg_try_advisory_lock(j.job_id) AS locked
261
- FROM (
262
- SELECT *
263
- FROM cte
264
- WHERE NOT locked
265
- ) t,
266
- LATERAL (
267
- SELECT *
268
- FROM que_lateral_jobs
269
- WHERE run_at <= now()
270
- AND (priority, run_at, job_id) > (t.priority, t.run_at, t.job_id)
271
- ORDER BY priority, run_at, job_id
272
- LIMIT 1
273
- ) j
274
- )
275
- )
276
- SELECT *
277
- FROM cte
278
- WHERE locked
279
- LIMIT 1
280
- SQL
281
- ),
282
- :delayed_job => (
283
- # From delayed_job_active_record
284
- <<-SQL
285
- UPDATE delayed_jobs
286
- SET locked_at = now(),
287
- locked_by = $1::text
288
- WHERE id IN (
289
- SELECT id
290
- FROM delayed_jobs
291
- WHERE (
292
- (run_at <= now() AND (locked_at IS NULL OR locked_at < now()) OR locked_by = $1) AND failed_at IS NULL
293
- )
294
- ORDER BY priority ASC, run_at ASC
295
- LIMIT 1
296
- FOR UPDATE
297
- )
298
- RETURNING *
299
- SQL
300
- )
301
- }
302
-
303
- connections = WORKER_COUNT.times.map do
304
- conn = new_connection.call
305
- conn.async_exec "SET SESSION synchronous_commit = #{SYNCHRONOUS_COMMIT}"
306
- queries.each do |name, sql|
307
- conn.prepare(name.to_s, sql)
308
- end
309
- conn
310
- end
311
-
312
-
313
-
314
- # Track the ids that are worked, to make sure they're all hit.
315
- $results = {
316
- :delayed_job => [],
317
- :queue_classic => [],
318
- :que => [],
319
- :que_lateral => []
320
- }
321
-
322
- def work_job(type, conn)
323
- case type
324
- when :delayed_job
325
- return unless r = conn.exec_prepared("delayed_job", [conn.object_id]).first
326
- $results[type] << r['id']
327
- conn.async_exec "DELETE FROM delayed_jobs WHERE id = $1", [r['id']]
328
-
329
- when :queue_classic
330
- return unless r = conn.async_exec("SELECT * FROM lock_head($1, $2)", ['default', 9]).first
331
- $results[type] << r['id']
332
- conn.async_exec "DELETE FROM queue_classic_jobs WHERE id = $1", [r['id']]
333
-
334
- when :que
335
- begin
336
- return unless r = conn.exec_prepared("que").first
337
- # Have to double-check that the job is valid, as explained at length in Que::Job.work.
338
- return true unless conn.async_exec("SELECT * FROM que_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]).first
339
- conn.async_exec "DELETE FROM que_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]
340
- $results[type] << r['job_id']
341
- ensure
342
- conn.async_exec "SELECT pg_advisory_unlock_all()" if r
343
- end
344
-
345
- when :que_lateral
346
- begin
347
- return unless r = conn.exec_prepared("que_lateral").first
348
- return true unless conn.async_exec("SELECT * FROM que_lateral_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]).first
349
- conn.async_exec "DELETE FROM que_lateral_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]
350
- $results[type] << r['job_id']
351
- ensure
352
- conn.async_exec "SELECT pg_advisory_unlock_all()" if r
353
- end
354
-
355
- end
356
- end
357
-
358
- puts "Benchmarking #{JOB_COUNT} jobs, #{WORKER_COUNT} workers and synchronous_commit = #{SYNCHRONOUS_COMMIT}..."
359
-
360
- {
361
- :delayed_job => :delayed_jobs,
362
- :queue_classic => :queue_classic_jobs,
363
- :que => :que_jobs,
364
- :que_lateral => :que_lateral_jobs
365
- }.each do |type, table|
366
- print "Benchmarking #{type}... "
367
- start = Time.now
368
-
369
- threads = connections.map do |conn|
370
- Thread.new do
371
- loop do
372
- begin
373
- break unless work_job(type, conn)
374
- rescue
375
- # DelayedJob deadlocks sometimes.
376
- end
377
- end
378
- end
379
- end
380
-
381
- threads.each &:join
382
- time = Time.now - start
383
- puts "#{JOB_COUNT} jobs in #{time} seconds = #{(JOB_COUNT / time).round} jobs per second"
384
-
385
-
386
- # These checks are commented out because I can't seem to get DelayedJob to
387
- # pass them (Que and QueueClassic don't have the same problem). It seems
388
- # to repeat some jobs multiple times on every run, and its run times are
389
- # also highly variable.
390
-
391
- # worked = $results[type].map(&:to_i).sort
392
- # puts "Jobs worked more than once! #{worked.inspect}" unless worked == worked.uniq
393
- # puts "Jobs worked less than once! #{worked.inspect}" unless worked.length == JOB_COUNT
394
-
395
- puts "Jobs left in DB" unless pg.async_exec("SELECT count(*) FROM #{table}").first['count'].to_i == 0
396
- puts "Advisory locks left over!" if pg.async_exec("SELECT * FROM pg_locks WHERE locktype = 'advisory'").first
397
- end
398
- end