queue_classic 3.2.0.RC1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.github/workflows/codeql-analysis.yml +72 -0
- data/.github/workflows/main.yaml +71 -0
- data/.gitignore +2 -0
- data/{changelog → CHANGELOG.md} +87 -34
- data/CODE_OF_CONDUCT.md +46 -0
- data/Gemfile +9 -5
- data/README.md +103 -126
- data/Rakefile +2 -0
- data/lib/generators/queue_classic/install_generator.rb +6 -0
- data/lib/generators/queue_classic/templates/add_queue_classic.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb +3 -1
- data/lib/generators/queue_classic/templates/update_queue_classic_4_0_0.rb +11 -0
- data/lib/queue_classic/config.rb +2 -1
- data/lib/queue_classic/conn_adapter.rb +29 -15
- data/lib/queue_classic/queue.rb +66 -12
- data/lib/queue_classic/railtie.rb +2 -0
- data/lib/queue_classic/setup.rb +24 -7
- data/lib/queue_classic/tasks.rb +4 -5
- data/lib/queue_classic/version.rb +3 -1
- data/lib/queue_classic/worker.rb +15 -6
- data/lib/queue_classic.rb +4 -11
- data/queue_classic.gemspec +3 -2
- data/sql/create_table.sql +7 -16
- data/sql/ddl.sql +6 -82
- data/sql/downgrade_from_4_0_0.sql +88 -0
- data/sql/update_to_3_0_0.sql +5 -5
- data/sql/update_to_3_1_0.sql +6 -6
- data/sql/update_to_4_0_0.sql +6 -0
- data/test/benchmark_test.rb +2 -0
- data/test/config_test.rb +2 -0
- data/test/helper.rb +34 -0
- data/test/lib/queue_classic_rails_connection_test.rb +9 -6
- data/test/lib/queue_classic_test.rb +2 -0
- data/test/lib/queue_classic_test_with_activerecord_typecast.rb +21 -0
- data/test/queue_test.rb +62 -2
- data/test/rails-tests/.gitignore +2 -0
- data/test/rails-tests/rails523.sh +23 -0
- data/test/worker_test.rb +138 -17
- metadata +43 -13
- data/.travis.yml +0 -15
data/lib/queue_classic/queue.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative 'conn_adapter'
|
2
4
|
require 'json'
|
3
5
|
require 'time'
|
@@ -5,8 +7,8 @@ require 'time'
|
|
5
7
|
module QC
|
6
8
|
# The queue class maps a queue abstraction onto a database table.
|
7
9
|
class Queue
|
8
|
-
|
9
10
|
attr_reader :name, :top_bound
|
11
|
+
|
10
12
|
def initialize(name, top_bound=nil)
|
11
13
|
@name = name
|
12
14
|
@top_bound = top_bound || QC.top_bound
|
@@ -38,7 +40,16 @@ module QC
|
|
38
40
|
def enqueue(method, *args)
|
39
41
|
QC.log_yield(:measure => 'queue.enqueue') do
|
40
42
|
s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
|
41
|
-
|
43
|
+
begin
|
44
|
+
retries ||= 0
|
45
|
+
conn_adapter.execute(s, name, method, JSON.dump(args))
|
46
|
+
rescue PG::Error
|
47
|
+
if (retries += 1) < 2
|
48
|
+
retry
|
49
|
+
else
|
50
|
+
raise
|
51
|
+
end
|
52
|
+
end
|
42
53
|
end
|
43
54
|
end
|
44
55
|
|
@@ -64,21 +75,49 @@ module QC
|
|
64
75
|
s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
|
65
76
|
VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
|
66
77
|
RETURNING id"
|
67
|
-
|
78
|
+
begin
|
79
|
+
retries ||= 0
|
80
|
+
conn_adapter.execute(s, name, method, JSON.dump(args))
|
81
|
+
rescue PG::Error
|
82
|
+
if (retries += 1) < 2
|
83
|
+
retry
|
84
|
+
else
|
85
|
+
raise
|
86
|
+
end
|
87
|
+
end
|
68
88
|
end
|
69
89
|
end
|
70
90
|
|
71
91
|
def lock
|
72
92
|
QC.log_yield(:measure => 'queue.lock') do
|
73
|
-
s =
|
74
|
-
|
93
|
+
s = <<~SQL
|
94
|
+
WITH selected_job AS (
|
95
|
+
SELECT id
|
96
|
+
FROM queue_classic_jobs
|
97
|
+
WHERE
|
98
|
+
locked_at IS NULL AND
|
99
|
+
q_name = $1 AND
|
100
|
+
scheduled_at <= now()
|
101
|
+
LIMIT 1
|
102
|
+
FOR NO KEY UPDATE SKIP LOCKED
|
103
|
+
)
|
104
|
+
UPDATE queue_classic_jobs
|
105
|
+
SET
|
106
|
+
locked_at = now(),
|
107
|
+
locked_by = pg_backend_pid()
|
108
|
+
FROM selected_job
|
109
|
+
WHERE queue_classic_jobs.id = selected_job.id
|
110
|
+
RETURNING *
|
111
|
+
SQL
|
112
|
+
|
113
|
+
if r = conn_adapter.execute(s, name)
|
75
114
|
{}.tap do |job|
|
76
115
|
job[:id] = r["id"]
|
77
116
|
job[:q_name] = r["q_name"]
|
78
117
|
job[:method] = r["method"]
|
79
118
|
job[:args] = JSON.parse(r["args"])
|
80
119
|
if r["scheduled_at"]
|
81
|
-
job[:scheduled_at] = Time.parse(r["scheduled_at"])
|
120
|
+
job[:scheduled_at] = r["scheduled_at"].kind_of?(Time) ? r["scheduled_at"] : Time.parse(r["scheduled_at"])
|
82
121
|
ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
|
83
122
|
QC.measure("time-to-lock=#{ttl}ms source=#{name}")
|
84
123
|
end
|
@@ -89,14 +128,14 @@ module QC
|
|
89
128
|
|
90
129
|
def unlock(id)
|
91
130
|
QC.log_yield(:measure => 'queue.unlock') do
|
92
|
-
s = "UPDATE #{QC.table_name}
|
131
|
+
s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1"
|
93
132
|
conn_adapter.execute(s, id)
|
94
133
|
end
|
95
134
|
end
|
96
135
|
|
97
136
|
def delete(id)
|
98
137
|
QC.log_yield(:measure => 'queue.delete') do
|
99
|
-
conn_adapter.execute("DELETE FROM #{QC.table_name}
|
138
|
+
conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id)
|
100
139
|
end
|
101
140
|
end
|
102
141
|
|
@@ -107,13 +146,28 @@ module QC
|
|
107
146
|
end
|
108
147
|
end
|
109
148
|
|
149
|
+
# Count the number of jobs in a specific queue. This returns all
|
150
|
+
# jobs, including ones that are scheduled in the future.
|
110
151
|
def count
|
111
|
-
|
112
|
-
|
113
|
-
|
152
|
+
_count('queue.count', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1")
|
153
|
+
end
|
154
|
+
|
155
|
+
# Count the number of jobs in a specific queue, except ones scheduled in the future
|
156
|
+
def count_ready
|
157
|
+
_count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at <= now()")
|
158
|
+
end
|
159
|
+
|
160
|
+
# Count the number of jobs in a specific queue scheduled in the future
|
161
|
+
def count_scheduled
|
162
|
+
_count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at > now()")
|
163
|
+
end
|
164
|
+
|
165
|
+
private
|
166
|
+
def _count(metric_name, sql)
|
167
|
+
QC.log_yield(measure: metric_name) do
|
168
|
+
r = conn_adapter.execute(sql, name)
|
114
169
|
r["count"].to_i
|
115
170
|
end
|
116
171
|
end
|
117
|
-
|
118
172
|
end
|
119
173
|
end
|
data/lib/queue_classic/setup.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module QC
|
2
4
|
module Setup
|
3
5
|
Root = File.expand_path("../..", File.dirname(__FILE__))
|
@@ -8,51 +10,66 @@ module QC
|
|
8
10
|
DowngradeFrom_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
|
9
11
|
UpgradeTo_3_1_0 = File.join(Root, "/sql/update_to_3_1_0.sql")
|
10
12
|
DowngradeFrom_3_1_0 = File.join(Root, "/sql/downgrade_from_3_1_0.sql")
|
13
|
+
UpgradeTo_4_0_0 = File.join(Root, "/sql/update_to_4_0_0.sql")
|
14
|
+
DowngradeFrom_4_0_0 = File.join(Root, "/sql/downgrade_from_4_0_0.sql")
|
11
15
|
|
12
16
|
def self.create(c = QC::default_conn_adapter.connection)
|
13
|
-
conn = QC::ConnAdapter.new(c)
|
17
|
+
conn = QC::ConnAdapter.new(connection: c)
|
14
18
|
conn.execute(File.read(CreateTable))
|
15
19
|
conn.execute(File.read(SqlFunctions))
|
16
20
|
conn.disconnect if c.nil? #Don't close a conn we didn't create.
|
17
21
|
end
|
18
22
|
|
19
23
|
def self.drop(c = QC::default_conn_adapter.connection)
|
20
|
-
conn = QC::ConnAdapter.new(c)
|
24
|
+
conn = QC::ConnAdapter.new(connection: c)
|
21
25
|
conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
|
22
26
|
conn.execute(File.read(DropSqlFunctions))
|
23
27
|
conn.disconnect if c.nil? #Don't close a conn we didn't create.
|
24
28
|
end
|
25
29
|
|
26
30
|
def self.update(c = QC::default_conn_adapter.connection)
|
27
|
-
conn = QC::ConnAdapter.new(c)
|
31
|
+
conn = QC::ConnAdapter.new(connection: c)
|
28
32
|
conn.execute(File.read(UpgradeTo_3_0_0))
|
29
33
|
conn.execute(File.read(UpgradeTo_3_1_0))
|
34
|
+
conn.execute(File.read(UpgradeTo_4_0_0))
|
30
35
|
conn.execute(File.read(DropSqlFunctions))
|
31
36
|
conn.execute(File.read(SqlFunctions))
|
32
37
|
end
|
33
38
|
|
34
39
|
def self.update_to_3_0_0(c = QC::default_conn_adapter.connection)
|
35
|
-
conn = QC::ConnAdapter.new(c)
|
40
|
+
conn = QC::ConnAdapter.new(connection: c)
|
36
41
|
conn.execute(File.read(UpgradeTo_3_0_0))
|
37
42
|
conn.execute(File.read(DropSqlFunctions))
|
38
43
|
conn.execute(File.read(SqlFunctions))
|
39
44
|
end
|
40
45
|
|
41
46
|
def self.downgrade_from_3_0_0(c = QC::default_conn_adapter.connection)
|
42
|
-
conn = QC::ConnAdapter.new(c)
|
47
|
+
conn = QC::ConnAdapter.new(connection: c)
|
43
48
|
conn.execute(File.read(DowngradeFrom_3_0_0))
|
44
49
|
end
|
45
50
|
|
46
51
|
def self.update_to_3_1_0(c = QC::default_conn_adapter.connection)
|
47
|
-
conn = QC::ConnAdapter.new(c)
|
52
|
+
conn = QC::ConnAdapter.new(connection: c)
|
48
53
|
conn.execute(File.read(UpgradeTo_3_1_0))
|
49
54
|
conn.execute(File.read(DropSqlFunctions))
|
50
55
|
conn.execute(File.read(SqlFunctions))
|
51
56
|
end
|
52
57
|
|
53
58
|
def self.downgrade_from_3_1_0(c = QC::default_conn_adapter.connection)
|
54
|
-
conn = QC::ConnAdapter.new(c)
|
59
|
+
conn = QC::ConnAdapter.new(connection: c)
|
55
60
|
conn.execute(File.read(DowngradeFrom_3_1_0))
|
56
61
|
end
|
62
|
+
|
63
|
+
def self.update_to_4_0_0(c = QC::default_conn_adapter.connection)
|
64
|
+
conn = QC::ConnAdapter.new(connection: c)
|
65
|
+
conn.execute(File.read(UpgradeTo_4_0_0))
|
66
|
+
conn.execute(File.read(DropSqlFunctions))
|
67
|
+
conn.execute(File.read(SqlFunctions))
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.downgrade_from_4_0_0(c = QC::default_conn_adapter.connection)
|
71
|
+
conn = QC::ConnAdapter.new(connection: c)
|
72
|
+
conn.execute(File.read(DowngradeFrom_4_0_0))
|
73
|
+
end
|
57
74
|
end
|
58
75
|
end
|
data/lib/queue_classic/tasks.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
task :environment
|
2
4
|
|
3
5
|
namespace :jobs do
|
@@ -11,11 +13,8 @@ namespace :qc do
|
|
11
13
|
@worker = QC.default_worker_class.new
|
12
14
|
|
13
15
|
trap('INT') do
|
14
|
-
$stderr.puts("Received INT. Shutting down.")
|
15
|
-
|
16
|
-
$stderr.puts("Worker has stopped running. Exit.")
|
17
|
-
exit(1)
|
18
|
-
end
|
16
|
+
$stderr.puts("Received INT. Shutting down.")
|
17
|
+
abort("Worker has stopped running. Exit.") unless @worker.running
|
19
18
|
@worker.stop
|
20
19
|
end
|
21
20
|
|
data/lib/queue_classic/worker.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# -*- coding: utf-8 -*-
|
2
4
|
require_relative 'queue'
|
3
5
|
require_relative 'conn_adapter'
|
@@ -12,7 +14,7 @@ module QC
|
|
12
14
|
# This method takes a single hash argument. The following keys are read:
|
13
15
|
# fork_worker:: Worker forks each job execution.
|
14
16
|
# wait_interval:: Time to wait between failed lock attempts
|
15
|
-
# connection::
|
17
|
+
# connection:: PG::Connection object.
|
16
18
|
# q_name:: Name of a single queue to process.
|
17
19
|
# q_names:: Names of queues to process. Will process left to right.
|
18
20
|
# top_bound:: Offset to the head of the queue. 1 == strict FIFO.
|
@@ -21,7 +23,7 @@ module QC
|
|
21
23
|
@wait_interval = args[:wait_interval] || QC.wait_time
|
22
24
|
|
23
25
|
if args[:connection]
|
24
|
-
@conn_adapter = ConnAdapter.new(args[:connection])
|
26
|
+
@conn_adapter = ConnAdapter.new(connection: args[:connection])
|
25
27
|
else
|
26
28
|
@conn_adapter = QC.default_conn_adapter
|
27
29
|
end
|
@@ -109,10 +111,13 @@ module QC
|
|
109
111
|
finished = false
|
110
112
|
begin
|
111
113
|
call(job).tap do
|
112
|
-
queue
|
114
|
+
handle_success(queue, job)
|
113
115
|
finished = true
|
114
116
|
end
|
115
|
-
rescue => e
|
117
|
+
rescue StandardError, ScriptError, NoMemoryError => e
|
118
|
+
# We really only want to unlock the job for signal and system exit
|
119
|
+
# exceptions. If we encounter a ScriptError or a NoMemoryError any
|
120
|
+
# future run will likely encounter the same error.
|
116
121
|
handle_failure(job, e)
|
117
122
|
finished = true
|
118
123
|
ensure
|
@@ -134,8 +139,12 @@ module QC
|
|
134
139
|
receiver.send(message, *args)
|
135
140
|
end
|
136
141
|
|
137
|
-
|
138
|
-
|
142
|
+
def handle_success(queue, job)
|
143
|
+
queue.delete(job[:id])
|
144
|
+
end
|
145
|
+
|
146
|
+
# This method will be called when a StandardError, ScriptError or
|
147
|
+
# NoMemoryError is raised during the execution of the job.
|
139
148
|
def handle_failure(job,e)
|
140
149
|
$stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
|
141
150
|
end
|
data/lib/queue_classic.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative "queue_classic/config"
|
2
4
|
|
3
5
|
module QC
|
@@ -49,15 +51,7 @@ Please use the method QC.#{config_method} instead.
|
|
49
51
|
end
|
50
52
|
|
51
53
|
def self.default_conn_adapter
|
52
|
-
|
53
|
-
return t[:qc_conn_adapter] if t[:qc_conn_adapter]
|
54
|
-
adapter = if rails_connection_sharing_enabled?
|
55
|
-
ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
|
56
|
-
else
|
57
|
-
ConnAdapter.new
|
58
|
-
end
|
59
|
-
|
60
|
-
t[:qc_conn_adapter] = adapter
|
54
|
+
Thread.current[:qc_conn_adapter] ||= ConnAdapter.new(active_record_connection_share: rails_connection_sharing_enabled?)
|
61
55
|
end
|
62
56
|
|
63
57
|
def self.default_conn_adapter=(conn)
|
@@ -100,8 +94,7 @@ Please use the method QC.#{config_method} instead.
|
|
100
94
|
# This will unlock all jobs any postgres' PID that is not existing anymore
|
101
95
|
# to prevent any infinitely locked jobs
|
102
96
|
def self.unlock_jobs_of_dead_workers
|
103
|
-
|
104
|
-
default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT #{pid_column} FROM pg_stat_activity);")
|
97
|
+
default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);")
|
105
98
|
end
|
106
99
|
|
107
100
|
# private class methods
|
data/queue_classic.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.description = "queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic only requires pg."
|
11
11
|
spec.summary = "Simple, efficient worker queue for Ruby & PostgreSQL."
|
12
12
|
spec.authors = ["Ryan Smith (♠ ace hacker)"]
|
13
|
-
spec.homepage = "
|
13
|
+
spec.homepage = "https://github.com/QueueClassic/queue_classic"
|
14
14
|
spec.license = "MIT"
|
15
15
|
|
16
16
|
spec.files = `git ls-files -z`.split("\x0")
|
@@ -20,5 +20,6 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.require_paths = %w[lib]
|
22
22
|
|
23
|
-
spec.add_dependency "pg", ">=
|
23
|
+
spec.add_dependency "pg", ">= 1.1", "< 2.0"
|
24
|
+
spec.add_development_dependency "activerecord", ">= 5.0.0", "< 6.1"
|
24
25
|
end
|
data/sql/create_table.sql
CHANGED
@@ -1,26 +1,17 @@
|
|
1
|
-
|
1
|
+
DO $$ BEGIN
|
2
2
|
|
3
3
|
CREATE TABLE queue_classic_jobs (
|
4
4
|
id bigserial PRIMARY KEY,
|
5
|
-
q_name text
|
6
|
-
method text
|
7
|
-
args
|
5
|
+
q_name text NOT NULL CHECK (length(q_name) > 0),
|
6
|
+
method text NOT NULL CHECK (length(method) > 0),
|
7
|
+
args jsonb NOT NULL,
|
8
8
|
locked_at timestamptz,
|
9
9
|
locked_by integer,
|
10
|
-
created_at timestamptz
|
11
|
-
scheduled_at timestamptz
|
10
|
+
created_at timestamptz DEFAULT now(),
|
11
|
+
scheduled_at timestamptz DEFAULT now()
|
12
12
|
);
|
13
13
|
|
14
|
-
|
15
|
-
if exists (select 1 from pg_type where typname = 'jsonb') then
|
16
|
-
alter table queue_classic_jobs alter column args type jsonb using args::jsonb;
|
17
|
-
-- Otherwise, use json type for the args column if available
|
18
|
-
elsif exists (select 1 from pg_type where typname = 'json') then
|
19
|
-
alter table queue_classic_jobs alter column args type json using args::json;
|
20
|
-
end if;
|
21
|
-
|
22
|
-
end $$ language plpgsql;
|
14
|
+
END $$ LANGUAGE plpgsql;
|
23
15
|
|
24
16
|
CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
|
25
17
|
CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
|
26
|
-
|
data/sql/ddl.sql
CHANGED
@@ -1,84 +1,8 @@
|
|
1
|
-
-- We are declaring the return type to be queue_classic_jobs.
|
2
|
-
-- This is ok since I am assuming that all of the users added queues will
|
3
|
-
-- have identical columns to queue_classic_jobs.
|
4
|
-
-- When QC supports queues with columns other than the default, we will have to change this.
|
5
|
-
|
6
|
-
CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
|
7
|
-
RETURNS SETOF queue_classic_jobs AS $$
|
8
|
-
DECLARE
|
9
|
-
unlocked bigint;
|
10
|
-
relative_top integer;
|
11
|
-
job_count integer;
|
12
|
-
BEGIN
|
13
|
-
-- The purpose is to release contention for the first spot in the table.
|
14
|
-
-- The select count(*) is going to slow down dequeue performance but allow
|
15
|
-
-- for more workers. Would love to see some optimization here...
|
16
|
-
|
17
|
-
EXECUTE 'SELECT count(*) FROM '
|
18
|
-
|| '(SELECT * FROM queue_classic_jobs '
|
19
|
-
|| ' WHERE locked_at IS NULL'
|
20
|
-
|| ' AND q_name = '
|
21
|
-
|| quote_literal(q_name)
|
22
|
-
|| ' AND scheduled_at <= '
|
23
|
-
|| quote_literal(now())
|
24
|
-
|| ' LIMIT '
|
25
|
-
|| quote_literal(top_boundary)
|
26
|
-
|| ') limited'
|
27
|
-
INTO job_count;
|
28
|
-
|
29
|
-
SELECT TRUNC(random() * (top_boundary - 1))
|
30
|
-
INTO relative_top;
|
31
|
-
|
32
|
-
IF job_count < top_boundary THEN
|
33
|
-
relative_top = 0;
|
34
|
-
END IF;
|
35
|
-
|
36
|
-
LOOP
|
37
|
-
BEGIN
|
38
|
-
EXECUTE 'SELECT id FROM queue_classic_jobs '
|
39
|
-
|| ' WHERE locked_at IS NULL'
|
40
|
-
|| ' AND q_name = '
|
41
|
-
|| quote_literal(q_name)
|
42
|
-
|| ' AND scheduled_at <= '
|
43
|
-
|| quote_literal(now())
|
44
|
-
|| ' ORDER BY id ASC'
|
45
|
-
|| ' LIMIT 1'
|
46
|
-
|| ' OFFSET ' || quote_literal(relative_top)
|
47
|
-
|| ' FOR UPDATE NOWAIT'
|
48
|
-
INTO unlocked;
|
49
|
-
EXIT;
|
50
|
-
EXCEPTION
|
51
|
-
WHEN lock_not_available THEN
|
52
|
-
-- do nothing. loop again and hope we get a lock
|
53
|
-
END;
|
54
|
-
END LOOP;
|
55
|
-
|
56
|
-
RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
|
57
|
-
|| ' SET locked_at = (CURRENT_TIMESTAMP),'
|
58
|
-
|| ' locked_by = (select pg_backend_pid())'
|
59
|
-
|| ' WHERE id = $1'
|
60
|
-
|| ' AND locked_at is NULL'
|
61
|
-
|| ' RETURNING *'
|
62
|
-
USING unlocked;
|
63
|
-
|
64
|
-
RETURN;
|
65
|
-
END;
|
66
|
-
$$ LANGUAGE plpgsql;
|
67
|
-
|
68
|
-
CREATE OR REPLACE FUNCTION lock_head(tname varchar)
|
69
|
-
RETURNS SETOF queue_classic_jobs AS $$
|
70
|
-
BEGIN
|
71
|
-
RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
|
72
|
-
END;
|
73
|
-
$$ LANGUAGE plpgsql;
|
74
|
-
|
75
1
|
-- queue_classic_notify function and trigger
|
76
|
-
|
77
|
-
perform pg_notify(new.q_name, '');
|
78
|
-
|
79
|
-
end $$ language plpgsql;
|
2
|
+
CREATE FUNCTION queue_classic_notify() RETURNS TRIGGER AS $$ BEGIN
|
3
|
+
perform pg_notify(new.q_name, ''); RETURN NULL;
|
4
|
+
END $$ LANGUAGE plpgsql;
|
80
5
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
execute procedure queue_classic_notify();
|
6
|
+
CREATE TRIGGER queue_classic_notify
|
7
|
+
AFTER INSERT ON queue_classic_jobs FOR EACH ROW
|
8
|
+
EXECUTE PROCEDURE queue_classic_notify();
|
@@ -0,0 +1,88 @@
|
|
1
|
+
DO $$DECLARE r record;
|
2
|
+
BEGIN
|
3
|
+
-- If jsonb type is available, do nothing as we're downgrading from 4.0.0
|
4
|
+
IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'jsonb') THEN
|
5
|
+
-- do nothing - it should already be already jsonb
|
6
|
+
-- Otherwise, use json type for the args column if available
|
7
|
+
ELSIF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'json') THEN
|
8
|
+
-- this should only happen if someone downgrades QC and their database < pg 9.4
|
9
|
+
ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE json USING args::json;
|
10
|
+
END IF;
|
11
|
+
|
12
|
+
|
13
|
+
END$$;
|
14
|
+
|
15
|
+
|
16
|
+
--
|
17
|
+
-- Re install the lock_head function
|
18
|
+
--
|
19
|
+
|
20
|
+
-- We are declaring the return type to be queue_classic_jobs.
|
21
|
+
-- This is ok since I am assuming that all of the users added queues will
|
22
|
+
-- have identical columns to queue_classic_jobs.
|
23
|
+
-- When QC supports queues with columns other than the default, we will have to change this.
|
24
|
+
|
25
|
+
CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
|
26
|
+
RETURNS SETOF queue_classic_jobs AS $$
|
27
|
+
DECLARE
|
28
|
+
unlocked bigint;
|
29
|
+
relative_top integer;
|
30
|
+
job_count integer;
|
31
|
+
BEGIN
|
32
|
+
-- The purpose is to release contention for the first spot in the table.
|
33
|
+
-- The select count(*) is going to slow down dequeue performance but allow
|
34
|
+
-- for more workers. Would love to see some optimization here...
|
35
|
+
|
36
|
+
EXECUTE 'SELECT count(*) FROM '
|
37
|
+
|| '(SELECT * FROM queue_classic_jobs '
|
38
|
+
|| ' WHERE locked_at IS NULL'
|
39
|
+
|| ' AND q_name = '
|
40
|
+
|| quote_literal(q_name)
|
41
|
+
|| ' AND scheduled_at <= '
|
42
|
+
|| quote_literal(now())
|
43
|
+
|| ' LIMIT '
|
44
|
+
|| quote_literal(top_boundary)
|
45
|
+
|| ') limited'
|
46
|
+
INTO job_count;
|
47
|
+
|
48
|
+
SELECT TRUNC(random() * (top_boundary - 1))
|
49
|
+
INTO relative_top;
|
50
|
+
|
51
|
+
IF job_count < top_boundary THEN
|
52
|
+
relative_top = 0;
|
53
|
+
END IF;
|
54
|
+
|
55
|
+
LOOP
|
56
|
+
BEGIN
|
57
|
+
EXECUTE 'SELECT id FROM queue_classic_jobs '
|
58
|
+
|| ' WHERE locked_at IS NULL'
|
59
|
+
|| ' AND q_name = '
|
60
|
+
|| quote_literal(q_name)
|
61
|
+
|| ' AND scheduled_at <= '
|
62
|
+
|| quote_literal(now())
|
63
|
+
|| ' ORDER BY id ASC'
|
64
|
+
|| ' LIMIT 1'
|
65
|
+
|| ' OFFSET ' || quote_literal(relative_top)
|
66
|
+
|| ' FOR UPDATE NOWAIT'
|
67
|
+
INTO unlocked;
|
68
|
+
EXIT;
|
69
|
+
EXCEPTION
|
70
|
+
WHEN lock_not_available THEN
|
71
|
+
-- do nothing. loop again and hope we get a lock
|
72
|
+
END;
|
73
|
+
END LOOP;
|
74
|
+
|
75
|
+
RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
|
76
|
+
|| ' SET locked_at = (CURRENT_TIMESTAMP),'
|
77
|
+
|| ' locked_by = (select pg_backend_pid())'
|
78
|
+
|| ' WHERE id = $1'
|
79
|
+
|| ' AND locked_at is NULL'
|
80
|
+
|| ' RETURNING *'
|
81
|
+
USING unlocked;
|
82
|
+
|
83
|
+
RETURN;
|
84
|
+
END $$ LANGUAGE plpgsql;
|
85
|
+
|
86
|
+
CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$ BEGIN
|
87
|
+
RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
|
88
|
+
END $$ LANGUAGE plpgsql;
|
data/sql/update_to_3_0_0.sql
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
DO $$DECLARE r record;
|
2
2
|
BEGIN
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
3
|
+
BEGIN
|
4
|
+
ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz DEFAULT now();
|
5
|
+
EXCEPTION
|
6
|
+
WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
|
7
|
+
END;
|
8
8
|
END$$;
|
9
9
|
|
10
10
|
DO $$DECLARE r record;
|
data/sql/update_to_3_1_0.sql
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
DO $$DECLARE r record;
|
2
2
|
BEGIN
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
3
|
+
BEGIN
|
4
|
+
ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz DEFAULT now();
|
5
|
+
CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
|
6
|
+
EXCEPTION
|
7
|
+
WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
|
8
|
+
END;
|
9
9
|
END$$;
|
data/test/benchmark_test.rb
CHANGED
data/test/config_test.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -1,6 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "bundler"
|
4
|
+
require "minitest/reporters"
|
5
|
+
|
2
6
|
Bundler.setup :default, :test
|
3
7
|
|
8
|
+
if ENV['CIRCLECI'] == "true"
|
9
|
+
Minitest::Reporters.use! Minitest::Reporters::JUnitReporter.new
|
10
|
+
else
|
11
|
+
Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
|
12
|
+
end
|
13
|
+
|
4
14
|
ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
|
5
15
|
|
6
16
|
require_relative '../lib/queue_classic'
|
@@ -58,4 +68,28 @@ class QCTest < Minitest::Test
|
|
58
68
|
ensure
|
59
69
|
original_environment.each { |name, value| ENV[name] = value }
|
60
70
|
end
|
71
|
+
|
72
|
+
def stub_any_instance(class_name, method_name, definition)
|
73
|
+
new_method_name = "new_#{method_name}"
|
74
|
+
original_method_name = "original_#{method_name}"
|
75
|
+
|
76
|
+
method_present = class_name.instance_methods(false).include? method_name
|
77
|
+
|
78
|
+
if method_present
|
79
|
+
class_name.send(:alias_method, original_method_name, method_name)
|
80
|
+
class_name.send(:define_method, new_method_name, definition)
|
81
|
+
class_name.send(:alias_method, method_name, new_method_name)
|
82
|
+
|
83
|
+
yield
|
84
|
+
else
|
85
|
+
message = "#{class_name} does not have method #{method_name}."
|
86
|
+
message << "\nAvailable methods: #{class_name.instance_methods(false)}"
|
87
|
+
raise ArgumentError.new message
|
88
|
+
end
|
89
|
+
ensure
|
90
|
+
if method_present
|
91
|
+
class_name.send(:alias_method, method_name, original_method_name)
|
92
|
+
class_name.send(:undef_method, new_method_name)
|
93
|
+
end
|
94
|
+
end
|
61
95
|
end
|