queue_classic 3.2.0.RC1 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.github/workflows/codeql-analysis.yml +72 -0
- data/.github/workflows/main.yaml +71 -0
- data/.gitignore +2 -0
- data/{changelog → CHANGELOG.md} +87 -34
- data/CODE_OF_CONDUCT.md +46 -0
- data/Gemfile +9 -5
- data/README.md +103 -126
- data/Rakefile +2 -0
- data/lib/generators/queue_classic/install_generator.rb +6 -0
- data/lib/generators/queue_classic/templates/add_queue_classic.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_4_0_0.rb +11 -0
- data/lib/queue_classic/config.rb +2 -1
- data/lib/queue_classic/conn_adapter.rb +29 -15
- data/lib/queue_classic/queue.rb +66 -12
- data/lib/queue_classic/railtie.rb +2 -0
- data/lib/queue_classic/setup.rb +24 -7
- data/lib/queue_classic/tasks.rb +4 -5
- data/lib/queue_classic/version.rb +3 -1
- data/lib/queue_classic/worker.rb +15 -6
- data/lib/queue_classic.rb +4 -11
- data/queue_classic.gemspec +3 -2
- data/sql/create_table.sql +7 -16
- data/sql/ddl.sql +6 -82
- data/sql/downgrade_from_4_0_0.sql +88 -0
- data/sql/update_to_3_0_0.sql +5 -5
- data/sql/update_to_3_1_0.sql +6 -6
- data/sql/update_to_4_0_0.sql +6 -0
- data/test/benchmark_test.rb +2 -0
- data/test/config_test.rb +2 -0
- data/test/helper.rb +34 -0
- data/test/lib/queue_classic_rails_connection_test.rb +9 -6
- data/test/lib/queue_classic_test.rb +2 -0
- data/test/lib/queue_classic_test_with_activerecord_typecast.rb +21 -0
- data/test/queue_test.rb +62 -2
- data/test/rails-tests/.gitignore +2 -0
- data/test/rails-tests/rails523.sh +23 -0
- data/test/worker_test.rb +138 -17
- metadata +43 -13
- data/.travis.yml +0 -15
data/lib/queue_classic/queue.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative 'conn_adapter'
|
2
4
|
require 'json'
|
3
5
|
require 'time'
|
@@ -5,8 +7,8 @@ require 'time'
|
|
5
7
|
module QC
|
6
8
|
# The queue class maps a queue abstraction onto a database table.
|
7
9
|
class Queue
|
8
|
-
|
9
10
|
attr_reader :name, :top_bound
|
11
|
+
|
10
12
|
def initialize(name, top_bound=nil)
|
11
13
|
@name = name
|
12
14
|
@top_bound = top_bound || QC.top_bound
|
@@ -38,7 +40,16 @@ module QC
|
|
38
40
|
def enqueue(method, *args)
|
39
41
|
QC.log_yield(:measure => 'queue.enqueue') do
|
40
42
|
s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
|
41
|
-
|
43
|
+
begin
|
44
|
+
retries ||= 0
|
45
|
+
conn_adapter.execute(s, name, method, JSON.dump(args))
|
46
|
+
rescue PG::Error
|
47
|
+
if (retries += 1) < 2
|
48
|
+
retry
|
49
|
+
else
|
50
|
+
raise
|
51
|
+
end
|
52
|
+
end
|
42
53
|
end
|
43
54
|
end
|
44
55
|
|
@@ -64,21 +75,49 @@ module QC
|
|
64
75
|
s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
|
65
76
|
VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
|
66
77
|
RETURNING id"
|
67
|
-
|
78
|
+
begin
|
79
|
+
retries ||= 0
|
80
|
+
conn_adapter.execute(s, name, method, JSON.dump(args))
|
81
|
+
rescue PG::Error
|
82
|
+
if (retries += 1) < 2
|
83
|
+
retry
|
84
|
+
else
|
85
|
+
raise
|
86
|
+
end
|
87
|
+
end
|
68
88
|
end
|
69
89
|
end
|
70
90
|
|
71
91
|
def lock
|
72
92
|
QC.log_yield(:measure => 'queue.lock') do
|
73
|
-
s =
|
74
|
-
|
93
|
+
s = <<~SQL
|
94
|
+
WITH selected_job AS (
|
95
|
+
SELECT id
|
96
|
+
FROM queue_classic_jobs
|
97
|
+
WHERE
|
98
|
+
locked_at IS NULL AND
|
99
|
+
q_name = $1 AND
|
100
|
+
scheduled_at <= now()
|
101
|
+
LIMIT 1
|
102
|
+
FOR NO KEY UPDATE SKIP LOCKED
|
103
|
+
)
|
104
|
+
UPDATE queue_classic_jobs
|
105
|
+
SET
|
106
|
+
locked_at = now(),
|
107
|
+
locked_by = pg_backend_pid()
|
108
|
+
FROM selected_job
|
109
|
+
WHERE queue_classic_jobs.id = selected_job.id
|
110
|
+
RETURNING *
|
111
|
+
SQL
|
112
|
+
|
113
|
+
if r = conn_adapter.execute(s, name)
|
75
114
|
{}.tap do |job|
|
76
115
|
job[:id] = r["id"]
|
77
116
|
job[:q_name] = r["q_name"]
|
78
117
|
job[:method] = r["method"]
|
79
118
|
job[:args] = JSON.parse(r["args"])
|
80
119
|
if r["scheduled_at"]
|
81
|
-
job[:scheduled_at] = Time.parse(r["scheduled_at"])
|
120
|
+
job[:scheduled_at] = r["scheduled_at"].kind_of?(Time) ? r["scheduled_at"] : Time.parse(r["scheduled_at"])
|
82
121
|
ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
|
83
122
|
QC.measure("time-to-lock=#{ttl}ms source=#{name}")
|
84
123
|
end
|
@@ -89,14 +128,14 @@ module QC
|
|
89
128
|
|
90
129
|
def unlock(id)
|
91
130
|
QC.log_yield(:measure => 'queue.unlock') do
|
92
|
-
s = "UPDATE #{QC.table_name}
|
131
|
+
s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1"
|
93
132
|
conn_adapter.execute(s, id)
|
94
133
|
end
|
95
134
|
end
|
96
135
|
|
97
136
|
def delete(id)
|
98
137
|
QC.log_yield(:measure => 'queue.delete') do
|
99
|
-
conn_adapter.execute("DELETE FROM #{QC.table_name}
|
138
|
+
conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id)
|
100
139
|
end
|
101
140
|
end
|
102
141
|
|
@@ -107,13 +146,28 @@ module QC
|
|
107
146
|
end
|
108
147
|
end
|
109
148
|
|
149
|
+
# Count the number of jobs in a specific queue. This returns all
|
150
|
+
# jobs, including ones that are scheduled in the future.
|
110
151
|
def count
|
111
|
-
|
112
|
-
|
113
|
-
|
152
|
+
_count('queue.count', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1")
|
153
|
+
end
|
154
|
+
|
155
|
+
# Count the number of jobs in a specific queue, except ones scheduled in the future
|
156
|
+
def count_ready
|
157
|
+
_count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at <= now()")
|
158
|
+
end
|
159
|
+
|
160
|
+
# Count the number of jobs in a specific queue scheduled in the future
|
161
|
+
def count_scheduled
|
162
|
+
_count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at > now()")
|
163
|
+
end
|
164
|
+
|
165
|
+
private
|
166
|
+
def _count(metric_name, sql)
|
167
|
+
QC.log_yield(measure: metric_name) do
|
168
|
+
r = conn_adapter.execute(sql, name)
|
114
169
|
r["count"].to_i
|
115
170
|
end
|
116
171
|
end
|
117
|
-
|
118
172
|
end
|
119
173
|
end
|
data/lib/queue_classic/setup.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QC
|
2
4
|
module Setup
|
3
5
|
Root = File.expand_path("../..", File.dirname(__FILE__))
|
@@ -8,51 +10,66 @@ module QC
|
|
8
10
|
DowngradeFrom_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
|
9
11
|
UpgradeTo_3_1_0 = File.join(Root, "/sql/update_to_3_1_0.sql")
|
10
12
|
DowngradeFrom_3_1_0 = File.join(Root, "/sql/downgrade_from_3_1_0.sql")
|
13
|
+
UpgradeTo_4_0_0 = File.join(Root, "/sql/update_to_4_0_0.sql")
|
14
|
+
DowngradeFrom_4_0_0 = File.join(Root, "/sql/downgrade_from_4_0_0.sql")
|
11
15
|
|
12
16
|
def self.create(c = QC::default_conn_adapter.connection)
|
13
|
-
conn = QC::ConnAdapter.new(c)
|
17
|
+
conn = QC::ConnAdapter.new(connection: c)
|
14
18
|
conn.execute(File.read(CreateTable))
|
15
19
|
conn.execute(File.read(SqlFunctions))
|
16
20
|
conn.disconnect if c.nil? #Don't close a conn we didn't create.
|
17
21
|
end
|
18
22
|
|
19
23
|
def self.drop(c = QC::default_conn_adapter.connection)
|
20
|
-
conn = QC::ConnAdapter.new(c)
|
24
|
+
conn = QC::ConnAdapter.new(connection: c)
|
21
25
|
conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
|
22
26
|
conn.execute(File.read(DropSqlFunctions))
|
23
27
|
conn.disconnect if c.nil? #Don't close a conn we didn't create.
|
24
28
|
end
|
25
29
|
|
26
30
|
def self.update(c = QC::default_conn_adapter.connection)
|
27
|
-
conn = QC::ConnAdapter.new(c)
|
31
|
+
conn = QC::ConnAdapter.new(connection: c)
|
28
32
|
conn.execute(File.read(UpgradeTo_3_0_0))
|
29
33
|
conn.execute(File.read(UpgradeTo_3_1_0))
|
34
|
+
conn.execute(File.read(UpgradeTo_4_0_0))
|
30
35
|
conn.execute(File.read(DropSqlFunctions))
|
31
36
|
conn.execute(File.read(SqlFunctions))
|
32
37
|
end
|
33
38
|
|
34
39
|
def self.update_to_3_0_0(c = QC::default_conn_adapter.connection)
|
35
|
-
conn = QC::ConnAdapter.new(c)
|
40
|
+
conn = QC::ConnAdapter.new(connection: c)
|
36
41
|
conn.execute(File.read(UpgradeTo_3_0_0))
|
37
42
|
conn.execute(File.read(DropSqlFunctions))
|
38
43
|
conn.execute(File.read(SqlFunctions))
|
39
44
|
end
|
40
45
|
|
41
46
|
def self.downgrade_from_3_0_0(c = QC::default_conn_adapter.connection)
|
42
|
-
conn = QC::ConnAdapter.new(c)
|
47
|
+
conn = QC::ConnAdapter.new(connection: c)
|
43
48
|
conn.execute(File.read(DowngradeFrom_3_0_0))
|
44
49
|
end
|
45
50
|
|
46
51
|
def self.update_to_3_1_0(c = QC::default_conn_adapter.connection)
|
47
|
-
conn = QC::ConnAdapter.new(c)
|
52
|
+
conn = QC::ConnAdapter.new(connection: c)
|
48
53
|
conn.execute(File.read(UpgradeTo_3_1_0))
|
49
54
|
conn.execute(File.read(DropSqlFunctions))
|
50
55
|
conn.execute(File.read(SqlFunctions))
|
51
56
|
end
|
52
57
|
|
53
58
|
def self.downgrade_from_3_1_0(c = QC::default_conn_adapter.connection)
|
54
|
-
conn = QC::ConnAdapter.new(c)
|
59
|
+
conn = QC::ConnAdapter.new(connection: c)
|
55
60
|
conn.execute(File.read(DowngradeFrom_3_1_0))
|
56
61
|
end
|
62
|
+
|
63
|
+
def self.update_to_4_0_0(c = QC::default_conn_adapter.connection)
|
64
|
+
conn = QC::ConnAdapter.new(connection: c)
|
65
|
+
conn.execute(File.read(UpgradeTo_4_0_0))
|
66
|
+
conn.execute(File.read(DropSqlFunctions))
|
67
|
+
conn.execute(File.read(SqlFunctions))
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.downgrade_from_4_0_0(c = QC::default_conn_adapter.connection)
|
71
|
+
conn = QC::ConnAdapter.new(connection: c)
|
72
|
+
conn.execute(File.read(DowngradeFrom_4_0_0))
|
73
|
+
end
|
57
74
|
end
|
58
75
|
end
|
data/lib/queue_classic/tasks.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
task :environment
|
2
4
|
|
3
5
|
namespace :jobs do
|
@@ -11,11 +13,8 @@ namespace :qc do
|
|
11
13
|
@worker = QC.default_worker_class.new
|
12
14
|
|
13
15
|
trap('INT') do
|
14
|
-
$stderr.puts("Received INT. Shutting down.")
|
15
|
-
|
16
|
-
$stderr.puts("Worker has stopped running. Exit.")
|
17
|
-
exit(1)
|
18
|
-
end
|
16
|
+
$stderr.puts("Received INT. Shutting down.")
|
17
|
+
abort("Worker has stopped running. Exit.") unless @worker.running
|
19
18
|
@worker.stop
|
20
19
|
end
|
21
20
|
|
data/lib/queue_classic/worker.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# -*- coding: utf-8 -*-
|
2
4
|
require_relative 'queue'
|
3
5
|
require_relative 'conn_adapter'
|
@@ -12,7 +14,7 @@ module QC
|
|
12
14
|
# This method takes a single hash argument. The following keys are read:
|
13
15
|
# fork_worker:: Worker forks each job execution.
|
14
16
|
# wait_interval:: Time to wait between failed lock attempts
|
15
|
-
# connection::
|
17
|
+
# connection:: PG::Connection object.
|
16
18
|
# q_name:: Name of a single queue to process.
|
17
19
|
# q_names:: Names of queues to process. Will process left to right.
|
18
20
|
# top_bound:: Offset to the head of the queue. 1 == strict FIFO.
|
@@ -21,7 +23,7 @@ module QC
|
|
21
23
|
@wait_interval = args[:wait_interval] || QC.wait_time
|
22
24
|
|
23
25
|
if args[:connection]
|
24
|
-
@conn_adapter = ConnAdapter.new(args[:connection])
|
26
|
+
@conn_adapter = ConnAdapter.new(connection: args[:connection])
|
25
27
|
else
|
26
28
|
@conn_adapter = QC.default_conn_adapter
|
27
29
|
end
|
@@ -109,10 +111,13 @@ module QC
|
|
109
111
|
finished = false
|
110
112
|
begin
|
111
113
|
call(job).tap do
|
112
|
-
queue
|
114
|
+
handle_success(queue, job)
|
113
115
|
finished = true
|
114
116
|
end
|
115
|
-
rescue => e
|
117
|
+
rescue StandardError, ScriptError, NoMemoryError => e
|
118
|
+
# We really only want to unlock the job for signal and system exit
|
119
|
+
# exceptions. If we encounter a ScriptError or a NoMemoryError any
|
120
|
+
# future run will likely encounter the same error.
|
116
121
|
handle_failure(job, e)
|
117
122
|
finished = true
|
118
123
|
ensure
|
@@ -134,8 +139,12 @@ module QC
|
|
134
139
|
receiver.send(message, *args)
|
135
140
|
end
|
136
141
|
|
137
|
-
|
138
|
-
|
142
|
+
def handle_success(queue, job)
|
143
|
+
queue.delete(job[:id])
|
144
|
+
end
|
145
|
+
|
146
|
+
# This method will be called when a StandardError, ScriptError or
|
147
|
+
# NoMemoryError is raised during the execution of the job.
|
139
148
|
def handle_failure(job,e)
|
140
149
|
$stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
|
141
150
|
end
|
data/lib/queue_classic.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative "queue_classic/config"
|
2
4
|
|
3
5
|
module QC
|
@@ -49,15 +51,7 @@ Please use the method QC.#{config_method} instead.
|
|
49
51
|
end
|
50
52
|
|
51
53
|
def self.default_conn_adapter
|
52
|
-
|
53
|
-
return t[:qc_conn_adapter] if t[:qc_conn_adapter]
|
54
|
-
adapter = if rails_connection_sharing_enabled?
|
55
|
-
ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
|
56
|
-
else
|
57
|
-
ConnAdapter.new
|
58
|
-
end
|
59
|
-
|
60
|
-
t[:qc_conn_adapter] = adapter
|
54
|
+
Thread.current[:qc_conn_adapter] ||= ConnAdapter.new(active_record_connection_share: rails_connection_sharing_enabled?)
|
61
55
|
end
|
62
56
|
|
63
57
|
def self.default_conn_adapter=(conn)
|
@@ -100,8 +94,7 @@ Please use the method QC.#{config_method} instead.
|
|
100
94
|
# This will unlock all jobs any postgres' PID that is not existing anymore
|
101
95
|
# to prevent any infinitely locked jobs
|
102
96
|
def self.unlock_jobs_of_dead_workers
|
103
|
-
|
104
|
-
default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT #{pid_column} FROM pg_stat_activity);")
|
97
|
+
default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);")
|
105
98
|
end
|
106
99
|
|
107
100
|
# private class methods
|
data/queue_classic.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.description = "queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic only requires pg."
|
11
11
|
spec.summary = "Simple, efficient worker queue for Ruby & PostgreSQL."
|
12
12
|
spec.authors = ["Ryan Smith (♠ ace hacker)"]
|
13
|
-
spec.homepage = "
|
13
|
+
spec.homepage = "https://github.com/QueueClassic/queue_classic"
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0")
|
@@ -20,5 +20,6 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.require_paths = %w[lib]
|
22
22
|
|
23
|
-
spec.add_dependency "pg", ">=
|
23
|
+
spec.add_dependency "pg", ">= 1.1", "< 2.0"
|
24
|
+
spec.add_development_dependency "activerecord", ">= 5.0.0", "< 6.1"
|
24
25
|
end
|
data/sql/create_table.sql
CHANGED
@@ -1,26 +1,17 @@
|
|
1
|
-
|
1
|
+
DO $$ BEGIN
|
2
2
|
|
3
3
|
CREATE TABLE queue_classic_jobs (
|
4
4
|
id bigserial PRIMARY KEY,
|
5
|
-
q_name text
|
6
|
-
method text
|
7
|
-
args
|
5
|
+
q_name text NOT NULL CHECK (length(q_name) > 0),
|
6
|
+
method text NOT NULL CHECK (length(method) > 0),
|
7
|
+
args jsonb NOT NULL,
|
8
8
|
locked_at timestamptz,
|
9
9
|
locked_by integer,
|
10
|
-
created_at timestamptz
|
11
|
-
scheduled_at timestamptz
|
10
|
+
created_at timestamptz DEFAULT now(),
|
11
|
+
scheduled_at timestamptz DEFAULT now()
|
12
12
|
);
|
13
13
|
|
14
|
-
|
15
|
-
if exists (select 1 from pg_type where typname = 'jsonb') then
|
16
|
-
alter table queue_classic_jobs alter column args type jsonb using args::jsonb;
|
17
|
-
-- Otherwise, use json type for the args column if available
|
18
|
-
elsif exists (select 1 from pg_type where typname = 'json') then
|
19
|
-
alter table queue_classic_jobs alter column args type json using args::json;
|
20
|
-
end if;
|
21
|
-
|
22
|
-
end $$ language plpgsql;
|
14
|
+
END $$ LANGUAGE plpgsql;
|
23
15
|
|
24
16
|
CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
|
25
17
|
CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
|
26
|
-
|
data/sql/ddl.sql
CHANGED
@@ -1,84 +1,8 @@
|
|
1
|
-
-- We are declaring the return type to be queue_classic_jobs.
|
2
|
-
-- This is ok since I am assuming that all of the users added queues will
|
3
|
-
-- have identical columns to queue_classic_jobs.
|
4
|
-
-- When QC supports queues with columns other than the default, we will have to change this.
|
5
|
-
|
6
|
-
CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
|
7
|
-
RETURNS SETOF queue_classic_jobs AS $$
|
8
|
-
DECLARE
|
9
|
-
unlocked bigint;
|
10
|
-
relative_top integer;
|
11
|
-
job_count integer;
|
12
|
-
BEGIN
|
13
|
-
-- The purpose is to release contention for the first spot in the table.
|
14
|
-
-- The select count(*) is going to slow down dequeue performance but allow
|
15
|
-
-- for more workers. Would love to see some optimization here...
|
16
|
-
|
17
|
-
EXECUTE 'SELECT count(*) FROM '
|
18
|
-
|| '(SELECT * FROM queue_classic_jobs '
|
19
|
-
|| ' WHERE locked_at IS NULL'
|
20
|
-
|| ' AND q_name = '
|
21
|
-
|| quote_literal(q_name)
|
22
|
-
|| ' AND scheduled_at <= '
|
23
|
-
|| quote_literal(now())
|
24
|
-
|| ' LIMIT '
|
25
|
-
|| quote_literal(top_boundary)
|
26
|
-
|| ') limited'
|
27
|
-
INTO job_count;
|
28
|
-
|
29
|
-
SELECT TRUNC(random() * (top_boundary - 1))
|
30
|
-
INTO relative_top;
|
31
|
-
|
32
|
-
IF job_count < top_boundary THEN
|
33
|
-
relative_top = 0;
|
34
|
-
END IF;
|
35
|
-
|
36
|
-
LOOP
|
37
|
-
BEGIN
|
38
|
-
EXECUTE 'SELECT id FROM queue_classic_jobs '
|
39
|
-
|| ' WHERE locked_at IS NULL'
|
40
|
-
|| ' AND q_name = '
|
41
|
-
|| quote_literal(q_name)
|
42
|
-
|| ' AND scheduled_at <= '
|
43
|
-
|| quote_literal(now())
|
44
|
-
|| ' ORDER BY id ASC'
|
45
|
-
|| ' LIMIT 1'
|
46
|
-
|| ' OFFSET ' || quote_literal(relative_top)
|
47
|
-
|| ' FOR UPDATE NOWAIT'
|
48
|
-
INTO unlocked;
|
49
|
-
EXIT;
|
50
|
-
EXCEPTION
|
51
|
-
WHEN lock_not_available THEN
|
52
|
-
-- do nothing. loop again and hope we get a lock
|
53
|
-
END;
|
54
|
-
END LOOP;
|
55
|
-
|
56
|
-
RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
|
57
|
-
|| ' SET locked_at = (CURRENT_TIMESTAMP),'
|
58
|
-
|| ' locked_by = (select pg_backend_pid())'
|
59
|
-
|| ' WHERE id = $1'
|
60
|
-
|| ' AND locked_at is NULL'
|
61
|
-
|| ' RETURNING *'
|
62
|
-
USING unlocked;
|
63
|
-
|
64
|
-
RETURN;
|
65
|
-
END;
|
66
|
-
$$ LANGUAGE plpgsql;
|
67
|
-
|
68
|
-
CREATE OR REPLACE FUNCTION lock_head(tname varchar)
|
69
|
-
RETURNS SETOF queue_classic_jobs AS $$
|
70
|
-
BEGIN
|
71
|
-
RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
|
72
|
-
END;
|
73
|
-
$$ LANGUAGE plpgsql;
|
74
|
-
|
75
1
|
-- queue_classic_notify function and trigger
|
76
|
-
|
77
|
-
perform pg_notify(new.q_name, '');
|
78
|
-
|
79
|
-
end $$ language plpgsql;
|
2
|
+
CREATE FUNCTION queue_classic_notify() RETURNS TRIGGER AS $$ BEGIN
|
3
|
+
perform pg_notify(new.q_name, ''); RETURN NULL;
|
4
|
+
END $$ LANGUAGE plpgsql;
|
80
5
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
execute procedure queue_classic_notify();
|
6
|
+
CREATE TRIGGER queue_classic_notify
|
7
|
+
AFTER INSERT ON queue_classic_jobs FOR EACH ROW
|
8
|
+
EXECUTE PROCEDURE queue_classic_notify();
|
@@ -0,0 +1,88 @@
|
|
1
|
+
DO $$DECLARE r record;
|
2
|
+
BEGIN
|
3
|
+
-- If jsonb type is available, do nothing as we're downgrading from 4.0.0
|
4
|
+
IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'jsonb') THEN
|
5
|
+
-- do nothing - it should already be already jsonb
|
6
|
+
-- Otherwise, use json type for the args column if available
|
7
|
+
ELSIF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'json') THEN
|
8
|
+
-- this should only happen if someone downgrades QC and their database < pg 9.4
|
9
|
+
ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE json USING args::json;
|
10
|
+
END IF;
|
11
|
+
|
12
|
+
|
13
|
+
END$$;
|
14
|
+
|
15
|
+
|
16
|
+
--
|
17
|
+
-- Re install the lock_head function
|
18
|
+
--
|
19
|
+
|
20
|
+
-- We are declaring the return type to be queue_classic_jobs.
|
21
|
+
-- This is ok since I am assuming that all of the users added queues will
|
22
|
+
-- have identical columns to queue_classic_jobs.
|
23
|
+
-- When QC supports queues with columns other than the default, we will have to change this.
|
24
|
+
|
25
|
+
CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
|
26
|
+
RETURNS SETOF queue_classic_jobs AS $$
|
27
|
+
DECLARE
|
28
|
+
unlocked bigint;
|
29
|
+
relative_top integer;
|
30
|
+
job_count integer;
|
31
|
+
BEGIN
|
32
|
+
-- The purpose is to release contention for the first spot in the table.
|
33
|
+
-- The select count(*) is going to slow down dequeue performance but allow
|
34
|
+
-- for more workers. Would love to see some optimization here...
|
35
|
+
|
36
|
+
EXECUTE 'SELECT count(*) FROM '
|
37
|
+
|| '(SELECT * FROM queue_classic_jobs '
|
38
|
+
|| ' WHERE locked_at IS NULL'
|
39
|
+
|| ' AND q_name = '
|
40
|
+
|| quote_literal(q_name)
|
41
|
+
|| ' AND scheduled_at <= '
|
42
|
+
|| quote_literal(now())
|
43
|
+
|| ' LIMIT '
|
44
|
+
|| quote_literal(top_boundary)
|
45
|
+
|| ') limited'
|
46
|
+
INTO job_count;
|
47
|
+
|
48
|
+
SELECT TRUNC(random() * (top_boundary - 1))
|
49
|
+
INTO relative_top;
|
50
|
+
|
51
|
+
IF job_count < top_boundary THEN
|
52
|
+
relative_top = 0;
|
53
|
+
END IF;
|
54
|
+
|
55
|
+
LOOP
|
56
|
+
BEGIN
|
57
|
+
EXECUTE 'SELECT id FROM queue_classic_jobs '
|
58
|
+
|| ' WHERE locked_at IS NULL'
|
59
|
+
|| ' AND q_name = '
|
60
|
+
|| quote_literal(q_name)
|
61
|
+
|| ' AND scheduled_at <= '
|
62
|
+
|| quote_literal(now())
|
63
|
+
|| ' ORDER BY id ASC'
|
64
|
+
|| ' LIMIT 1'
|
65
|
+
|| ' OFFSET ' || quote_literal(relative_top)
|
66
|
+
|| ' FOR UPDATE NOWAIT'
|
67
|
+
INTO unlocked;
|
68
|
+
EXIT;
|
69
|
+
EXCEPTION
|
70
|
+
WHEN lock_not_available THEN
|
71
|
+
-- do nothing. loop again and hope we get a lock
|
72
|
+
END;
|
73
|
+
END LOOP;
|
74
|
+
|
75
|
+
RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
|
76
|
+
|| ' SET locked_at = (CURRENT_TIMESTAMP),'
|
77
|
+
|| ' locked_by = (select pg_backend_pid())'
|
78
|
+
|| ' WHERE id = $1'
|
79
|
+
|| ' AND locked_at is NULL'
|
80
|
+
|| ' RETURNING *'
|
81
|
+
USING unlocked;
|
82
|
+
|
83
|
+
RETURN;
|
84
|
+
END $$ LANGUAGE plpgsql;
|
85
|
+
|
86
|
+
CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$ BEGIN
|
87
|
+
RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
|
88
|
+
END $$ LANGUAGE plpgsql;
|
data/sql/update_to_3_0_0.sql
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
DO $$DECLARE r record;
|
2
2
|
BEGIN
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
3
|
+
BEGIN
|
4
|
+
ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz DEFAULT now();
|
5
|
+
EXCEPTION
|
6
|
+
WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
|
7
|
+
END;
|
8
8
|
END$$;
|
9
9
|
|
10
10
|
DO $$DECLARE r record;
|
data/sql/update_to_3_1_0.sql
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
DO $$DECLARE r record;
|
2
2
|
BEGIN
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
3
|
+
BEGIN
|
4
|
+
ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz DEFAULT now();
|
5
|
+
CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
|
6
|
+
EXCEPTION
|
7
|
+
WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
|
8
|
+
END;
|
9
9
|
END$$;
|
data/test/benchmark_test.rb
CHANGED
data/test/config_test.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -1,6 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "bundler"
|
4
|
+
require "minitest/reporters"
|
5
|
+
|
2
6
|
Bundler.setup :default, :test
|
3
7
|
|
8
|
+
if ENV['CIRCLECI'] == "true"
|
9
|
+
Minitest::Reporters.use! Minitest::Reporters::JUnitReporter.new
|
10
|
+
else
|
11
|
+
Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
|
12
|
+
end
|
13
|
+
|
4
14
|
ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
|
5
15
|
|
6
16
|
require_relative '../lib/queue_classic'
|
@@ -58,4 +68,28 @@ class QCTest < Minitest::Test
|
|
58
68
|
ensure
|
59
69
|
original_environment.each { |name, value| ENV[name] = value }
|
60
70
|
end
|
71
|
+
|
72
|
+
def stub_any_instance(class_name, method_name, definition)
|
73
|
+
new_method_name = "new_#{method_name}"
|
74
|
+
original_method_name = "original_#{method_name}"
|
75
|
+
|
76
|
+
method_present = class_name.instance_methods(false).include? method_name
|
77
|
+
|
78
|
+
if method_present
|
79
|
+
class_name.send(:alias_method, original_method_name, method_name)
|
80
|
+
class_name.send(:define_method, new_method_name, definition)
|
81
|
+
class_name.send(:alias_method, method_name, new_method_name)
|
82
|
+
|
83
|
+
yield
|
84
|
+
else
|
85
|
+
message = "#{class_name} does not have method #{method_name}."
|
86
|
+
message << "\nAvailable methods: #{class_name.instance_methods(false)}"
|
87
|
+
raise ArgumentError.new message
|
88
|
+
end
|
89
|
+
ensure
|
90
|
+
if method_present
|
91
|
+
class_name.send(:alias_method, method_name, original_method_name)
|
92
|
+
class_name.send(:undef_method, new_method_name)
|
93
|
+
end
|
94
|
+
end
|
61
95
|
end
|