queue_classic 3.2.0.RC1 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/codeql-analysis.yml +72 -0
  3. data/.github/workflows/main.yaml +71 -0
  4. data/.gitignore +2 -0
  5. data/{changelog → CHANGELOG.md} +87 -34
  6. data/CODE_OF_CONDUCT.md +46 -0
  7. data/Gemfile +9 -5
  8. data/README.md +103 -126
  9. data/Rakefile +2 -0
  10. data/lib/generators/queue_classic/install_generator.rb +6 -0
  11. data/lib/generators/queue_classic/templates/add_queue_classic.rb +3 -1
  12. data/lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb +3 -1
  13. data/lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb +3 -1
  14. data/lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb +3 -1
  15. data/lib/generators/queue_classic/templates/update_queue_classic_4_0_0.rb +11 -0
  16. data/lib/queue_classic/config.rb +2 -1
  17. data/lib/queue_classic/conn_adapter.rb +29 -15
  18. data/lib/queue_classic/queue.rb +66 -12
  19. data/lib/queue_classic/railtie.rb +2 -0
  20. data/lib/queue_classic/setup.rb +24 -7
  21. data/lib/queue_classic/tasks.rb +4 -5
  22. data/lib/queue_classic/version.rb +3 -1
  23. data/lib/queue_classic/worker.rb +15 -6
  24. data/lib/queue_classic.rb +4 -11
  25. data/queue_classic.gemspec +3 -2
  26. data/sql/create_table.sql +7 -16
  27. data/sql/ddl.sql +6 -82
  28. data/sql/downgrade_from_4_0_0.sql +88 -0
  29. data/sql/update_to_3_0_0.sql +5 -5
  30. data/sql/update_to_3_1_0.sql +6 -6
  31. data/sql/update_to_4_0_0.sql +6 -0
  32. data/test/benchmark_test.rb +2 -0
  33. data/test/config_test.rb +2 -0
  34. data/test/helper.rb +34 -0
  35. data/test/lib/queue_classic_rails_connection_test.rb +9 -6
  36. data/test/lib/queue_classic_test.rb +2 -0
  37. data/test/lib/queue_classic_test_with_activerecord_typecast.rb +21 -0
  38. data/test/queue_test.rb +62 -2
  39. data/test/rails-tests/.gitignore +2 -0
  40. data/test/rails-tests/rails523.sh +23 -0
  41. data/test/worker_test.rb +138 -17
  42. metadata +43 -13
  43. data/.travis.yml +0 -15
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative 'conn_adapter'
2
4
  require 'json'
3
5
  require 'time'
@@ -5,8 +7,8 @@ require 'time'
5
7
  module QC
6
8
  # The queue class maps a queue abstraction onto a database table.
7
9
  class Queue
8
-
9
10
  attr_reader :name, :top_bound
11
+
10
12
  def initialize(name, top_bound=nil)
11
13
  @name = name
12
14
  @top_bound = top_bound || QC.top_bound
@@ -38,7 +40,16 @@ module QC
38
40
  def enqueue(method, *args)
39
41
  QC.log_yield(:measure => 'queue.enqueue') do
40
42
  s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
41
- conn_adapter.execute(s, name, method, JSON.dump(args))
43
+ begin
44
+ retries ||= 0
45
+ conn_adapter.execute(s, name, method, JSON.dump(args))
46
+ rescue PG::Error
47
+ if (retries += 1) < 2
48
+ retry
49
+ else
50
+ raise
51
+ end
52
+ end
42
53
  end
43
54
  end
44
55
 
@@ -64,21 +75,49 @@ module QC
64
75
  s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
65
76
  VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
66
77
  RETURNING id"
67
- conn_adapter.execute(s, name, method, JSON.dump(args))
78
+ begin
79
+ retries ||= 0
80
+ conn_adapter.execute(s, name, method, JSON.dump(args))
81
+ rescue PG::Error
82
+ if (retries += 1) < 2
83
+ retry
84
+ else
85
+ raise
86
+ end
87
+ end
68
88
  end
69
89
  end
70
90
 
71
91
  def lock
72
92
  QC.log_yield(:measure => 'queue.lock') do
73
- s = "SELECT * FROM lock_head($1, $2)"
74
- if r = conn_adapter.execute(s, name, top_bound)
93
+ s = <<~SQL
94
+ WITH selected_job AS (
95
+ SELECT id
96
+ FROM queue_classic_jobs
97
+ WHERE
98
+ locked_at IS NULL AND
99
+ q_name = $1 AND
100
+ scheduled_at <= now()
101
+ LIMIT 1
102
+ FOR NO KEY UPDATE SKIP LOCKED
103
+ )
104
+ UPDATE queue_classic_jobs
105
+ SET
106
+ locked_at = now(),
107
+ locked_by = pg_backend_pid()
108
+ FROM selected_job
109
+ WHERE queue_classic_jobs.id = selected_job.id
110
+ RETURNING *
111
+ SQL
112
+
113
+ if r = conn_adapter.execute(s, name)
75
114
  {}.tap do |job|
76
115
  job[:id] = r["id"]
77
116
  job[:q_name] = r["q_name"]
78
117
  job[:method] = r["method"]
79
118
  job[:args] = JSON.parse(r["args"])
80
119
  if r["scheduled_at"]
81
- job[:scheduled_at] = Time.parse(r["scheduled_at"])
120
+ job[:scheduled_at] = r["scheduled_at"].kind_of?(Time) ? r["scheduled_at"] : Time.parse(r["scheduled_at"])
82
121
  ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
83
122
  QC.measure("time-to-lock=#{ttl}ms source=#{name}")
84
123
  end
@@ -89,14 +128,14 @@ module QC
89
128
 
90
129
  def unlock(id)
91
130
  QC.log_yield(:measure => 'queue.unlock') do
92
- s = "UPDATE #{QC.table_name} set locked_at = null where id = $1"
131
+ s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1"
93
132
  conn_adapter.execute(s, id)
94
133
  end
95
134
  end
96
135
 
97
136
  def delete(id)
98
137
  QC.log_yield(:measure => 'queue.delete') do
99
- conn_adapter.execute("DELETE FROM #{QC.table_name} where id = $1", id)
138
+ conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id)
100
139
  end
101
140
  end
102
141
 
@@ -107,13 +146,28 @@ module QC
107
146
  end
108
147
  end
109
148
 
149
+ # Count the number of jobs in a specific queue. This returns all
150
+ # jobs, including ones that are scheduled in the future.
110
151
  def count
111
- QC.log_yield(:measure => 'queue.count') do
112
- s = "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1"
113
- r = conn_adapter.execute(s, name)
152
+ _count('queue.count', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1")
153
+ end
154
+
155
+ # Count the number of jobs in a specific queue, except ones scheduled in the future
156
+ def count_ready
157
+ _count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at <= now()")
158
+ end
159
+
160
+ # Count the number of jobs in a specific queue scheduled in the future
161
+ def count_scheduled
162
+ _count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at > now()")
163
+ end
164
+
165
+ private
166
+ def _count(metric_name, sql)
167
+ QC.log_yield(measure: metric_name) do
168
+ r = conn_adapter.execute(sql, name)
114
169
  r["count"].to_i
115
170
  end
116
171
  end
117
-
118
172
  end
119
173
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'rails/railtie'
2
4
 
3
5
  module QC
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QC
2
4
  module Setup
3
5
  Root = File.expand_path("../..", File.dirname(__FILE__))
@@ -8,51 +10,66 @@ module QC
8
10
  DowngradeFrom_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
9
11
  UpgradeTo_3_1_0 = File.join(Root, "/sql/update_to_3_1_0.sql")
10
12
  DowngradeFrom_3_1_0 = File.join(Root, "/sql/downgrade_from_3_1_0.sql")
13
+ UpgradeTo_4_0_0 = File.join(Root, "/sql/update_to_4_0_0.sql")
14
+ DowngradeFrom_4_0_0 = File.join(Root, "/sql/downgrade_from_4_0_0.sql")
11
15
 
12
16
  def self.create(c = QC::default_conn_adapter.connection)
13
- conn = QC::ConnAdapter.new(c)
17
+ conn = QC::ConnAdapter.new(connection: c)
14
18
  conn.execute(File.read(CreateTable))
15
19
  conn.execute(File.read(SqlFunctions))
16
20
  conn.disconnect if c.nil? #Don't close a conn we didn't create.
17
21
  end
18
22
 
19
23
  def self.drop(c = QC::default_conn_adapter.connection)
20
- conn = QC::ConnAdapter.new(c)
24
+ conn = QC::ConnAdapter.new(connection: c)
21
25
  conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
22
26
  conn.execute(File.read(DropSqlFunctions))
23
27
  conn.disconnect if c.nil? #Don't close a conn we didn't create.
24
28
  end
25
29
 
26
30
  def self.update(c = QC::default_conn_adapter.connection)
27
- conn = QC::ConnAdapter.new(c)
31
+ conn = QC::ConnAdapter.new(connection: c)
28
32
  conn.execute(File.read(UpgradeTo_3_0_0))
29
33
  conn.execute(File.read(UpgradeTo_3_1_0))
34
+ conn.execute(File.read(UpgradeTo_4_0_0))
30
35
  conn.execute(File.read(DropSqlFunctions))
31
36
  conn.execute(File.read(SqlFunctions))
32
37
  end
33
38
 
34
39
  def self.update_to_3_0_0(c = QC::default_conn_adapter.connection)
35
- conn = QC::ConnAdapter.new(c)
40
+ conn = QC::ConnAdapter.new(connection: c)
36
41
  conn.execute(File.read(UpgradeTo_3_0_0))
37
42
  conn.execute(File.read(DropSqlFunctions))
38
43
  conn.execute(File.read(SqlFunctions))
39
44
  end
40
45
 
41
46
  def self.downgrade_from_3_0_0(c = QC::default_conn_adapter.connection)
42
- conn = QC::ConnAdapter.new(c)
47
+ conn = QC::ConnAdapter.new(connection: c)
43
48
  conn.execute(File.read(DowngradeFrom_3_0_0))
44
49
  end
45
50
 
46
51
  def self.update_to_3_1_0(c = QC::default_conn_adapter.connection)
47
- conn = QC::ConnAdapter.new(c)
52
+ conn = QC::ConnAdapter.new(connection: c)
48
53
  conn.execute(File.read(UpgradeTo_3_1_0))
49
54
  conn.execute(File.read(DropSqlFunctions))
50
55
  conn.execute(File.read(SqlFunctions))
51
56
  end
52
57
 
53
58
  def self.downgrade_from_3_1_0(c = QC::default_conn_adapter.connection)
54
- conn = QC::ConnAdapter.new(c)
59
+ conn = QC::ConnAdapter.new(connection: c)
55
60
  conn.execute(File.read(DowngradeFrom_3_1_0))
56
61
  end
62
+
63
+ def self.update_to_4_0_0(c = QC::default_conn_adapter.connection)
64
+ conn = QC::ConnAdapter.new(connection: c)
65
+ conn.execute(File.read(UpgradeTo_4_0_0))
66
+ conn.execute(File.read(DropSqlFunctions))
67
+ conn.execute(File.read(SqlFunctions))
68
+ end
69
+
70
+ def self.downgrade_from_4_0_0(c = QC::default_conn_adapter.connection)
71
+ conn = QC::ConnAdapter.new(connection: c)
72
+ conn.execute(File.read(DowngradeFrom_4_0_0))
73
+ end
57
74
  end
58
75
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  task :environment
2
4
 
3
5
  namespace :jobs do
@@ -11,11 +13,8 @@ namespace :qc do
11
13
  @worker = QC.default_worker_class.new
12
14
 
13
15
  trap('INT') do
14
- $stderr.puts("Received INT. Shutting down.")
15
- if !@worker.running
16
- $stderr.puts("Worker has stopped running. Exit.")
17
- exit(1)
18
- end
16
+ $stderr.puts("Received INT. Shutting down.")
17
+ abort("Worker has stopped running. Exit.") unless @worker.running
19
18
  @worker.stop
20
19
  end
21
20
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QC
2
- VERSION = "3.2.0.RC1"
4
+ VERSION = "4.0.0"
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # -*- coding: utf-8 -*-
2
4
  require_relative 'queue'
3
5
  require_relative 'conn_adapter'
@@ -12,7 +14,7 @@ module QC
12
14
  # This method takes a single hash argument. The following keys are read:
13
15
  # fork_worker:: Worker forks each job execution.
14
16
  # wait_interval:: Time to wait between failed lock attempts
15
- # connection:: PGConn object.
17
+ # connection:: PG::Connection object.
16
18
  # q_name:: Name of a single queue to process.
17
19
  # q_names:: Names of queues to process. Will process left to right.
18
20
  # top_bound:: Offset to the head of the queue. 1 == strict FIFO.
@@ -21,7 +23,7 @@ module QC
21
23
  @wait_interval = args[:wait_interval] || QC.wait_time
22
24
 
23
25
  if args[:connection]
24
- @conn_adapter = ConnAdapter.new(args[:connection])
26
+ @conn_adapter = ConnAdapter.new(connection: args[:connection])
25
27
  else
26
28
  @conn_adapter = QC.default_conn_adapter
27
29
  end
@@ -109,10 +111,13 @@ module QC
109
111
  finished = false
110
112
  begin
111
113
  call(job).tap do
112
- queue.delete(job[:id])
114
+ handle_success(queue, job)
113
115
  finished = true
114
116
  end
115
- rescue => e
117
+ rescue StandardError, ScriptError, NoMemoryError => e
118
+ # We really only want to unlock the job for signal and system exit
119
+ # exceptions. If we encounter a ScriptError or a NoMemoryError any
120
+ # future run will likely encounter the same error.
116
121
  handle_failure(job, e)
117
122
  finished = true
118
123
  ensure
@@ -134,8 +139,12 @@ module QC
134
139
  receiver.send(message, *args)
135
140
  end
136
141
 
137
- # This method will be called when an exception
138
- # is raised during the execution of the job.
142
+ def handle_success(queue, job)
143
+ queue.delete(job[:id])
144
+ end
145
+
146
+ # This method will be called when a StandardError, ScriptError or
147
+ # NoMemoryError is raised during the execution of the job.
139
148
  def handle_failure(job,e)
140
149
  $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
141
150
  end
data/lib/queue_classic.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative "queue_classic/config"
2
4
 
3
5
  module QC
@@ -49,15 +51,7 @@ Please use the method QC.#{config_method} instead.
49
51
  end
50
52
 
51
53
  def self.default_conn_adapter
52
- t = Thread.current
53
- return t[:qc_conn_adapter] if t[:qc_conn_adapter]
54
- adapter = if rails_connection_sharing_enabled?
55
- ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
56
- else
57
- ConnAdapter.new
58
- end
59
-
60
- t[:qc_conn_adapter] = adapter
54
+ Thread.current[:qc_conn_adapter] ||= ConnAdapter.new(active_record_connection_share: rails_connection_sharing_enabled?)
61
55
  end
62
56
 
63
57
  def self.default_conn_adapter=(conn)
@@ -100,8 +94,7 @@ Please use the method QC.#{config_method} instead.
100
94
  # This will unlock all jobs any postgres' PID that is not existing anymore
101
95
  # to prevent any infinitely locked jobs
102
96
  def self.unlock_jobs_of_dead_workers
103
- pid_column = default_conn_adapter.server_version < 90200 ? "procpid" : "pid"
104
- default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT #{pid_column} FROM pg_stat_activity);")
97
+ default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);")
105
98
  end
106
99
 
107
100
  # private class methods
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
10
10
  spec.description = "queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic only requires pg."
11
11
  spec.summary = "Simple, efficient worker queue for Ruby & PostgreSQL."
12
12
  spec.authors = ["Ryan Smith (♠ ace hacker)"]
13
- spec.homepage = "http://github.com/QueueClassic/queue_classic"
13
+ spec.homepage = "https://github.com/QueueClassic/queue_classic"
14
14
  spec.license = "MIT"
15
15
 
16
16
  spec.files = `git ls-files -z`.split("\x0")
@@ -20,5 +20,6 @@ Gem::Specification.new do |spec|
20
20
 
21
21
  spec.require_paths = %w[lib]
22
22
 
23
- spec.add_dependency "pg", ">= 0.17", "< 0.19"
23
+ spec.add_dependency "pg", ">= 1.1", "< 2.0"
24
+ spec.add_development_dependency "activerecord", ">= 5.0.0", "< 6.1"
24
25
  end
data/sql/create_table.sql CHANGED
@@ -1,26 +1,17 @@
1
- do $$ begin
1
+ DO $$ BEGIN
2
2
 
3
3
  CREATE TABLE queue_classic_jobs (
4
4
  id bigserial PRIMARY KEY,
5
- q_name text not null check (length(q_name) > 0),
6
- method text not null check (length(method) > 0),
7
- args text not null,
5
+ q_name text NOT NULL CHECK (length(q_name) > 0),
6
+ method text NOT NULL CHECK (length(method) > 0),
7
+ args jsonb NOT NULL,
8
8
  locked_at timestamptz,
9
9
  locked_by integer,
10
- created_at timestamptz default now(),
11
- scheduled_at timestamptz default now()
10
+ created_at timestamptz DEFAULT now(),
11
+ scheduled_at timestamptz DEFAULT now()
12
12
  );
13
13
 
14
- -- If jsonb type is available, use it for the args column
15
- if exists (select 1 from pg_type where typname = 'jsonb') then
16
- alter table queue_classic_jobs alter column args type jsonb using args::jsonb;
17
- -- Otherwise, use json type for the args column if available
18
- elsif exists (select 1 from pg_type where typname = 'json') then
19
- alter table queue_classic_jobs alter column args type json using args::json;
20
- end if;
21
-
22
- end $$ language plpgsql;
14
+ END $$ LANGUAGE plpgsql;
23
15
 
24
16
  CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
25
17
  CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
26
-
data/sql/ddl.sql CHANGED
@@ -1,84 +1,8 @@
1
- -- We are declaring the return type to be queue_classic_jobs.
2
- -- This is ok since I am assuming that all of the users added queues will
3
- -- have identical columns to queue_classic_jobs.
4
- -- When QC supports queues with columns other than the default, we will have to change this.
5
-
6
- CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
7
- RETURNS SETOF queue_classic_jobs AS $$
8
- DECLARE
9
- unlocked bigint;
10
- relative_top integer;
11
- job_count integer;
12
- BEGIN
13
- -- The purpose is to release contention for the first spot in the table.
14
- -- The select count(*) is going to slow down dequeue performance but allow
15
- -- for more workers. Would love to see some optimization here...
16
-
17
- EXECUTE 'SELECT count(*) FROM '
18
- || '(SELECT * FROM queue_classic_jobs '
19
- || ' WHERE locked_at IS NULL'
20
- || ' AND q_name = '
21
- || quote_literal(q_name)
22
- || ' AND scheduled_at <= '
23
- || quote_literal(now())
24
- || ' LIMIT '
25
- || quote_literal(top_boundary)
26
- || ') limited'
27
- INTO job_count;
28
-
29
- SELECT TRUNC(random() * (top_boundary - 1))
30
- INTO relative_top;
31
-
32
- IF job_count < top_boundary THEN
33
- relative_top = 0;
34
- END IF;
35
-
36
- LOOP
37
- BEGIN
38
- EXECUTE 'SELECT id FROM queue_classic_jobs '
39
- || ' WHERE locked_at IS NULL'
40
- || ' AND q_name = '
41
- || quote_literal(q_name)
42
- || ' AND scheduled_at <= '
43
- || quote_literal(now())
44
- || ' ORDER BY id ASC'
45
- || ' LIMIT 1'
46
- || ' OFFSET ' || quote_literal(relative_top)
47
- || ' FOR UPDATE NOWAIT'
48
- INTO unlocked;
49
- EXIT;
50
- EXCEPTION
51
- WHEN lock_not_available THEN
52
- -- do nothing. loop again and hope we get a lock
53
- END;
54
- END LOOP;
55
-
56
- RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
57
- || ' SET locked_at = (CURRENT_TIMESTAMP),'
58
- || ' locked_by = (select pg_backend_pid())'
59
- || ' WHERE id = $1'
60
- || ' AND locked_at is NULL'
61
- || ' RETURNING *'
62
- USING unlocked;
63
-
64
- RETURN;
65
- END;
66
- $$ LANGUAGE plpgsql;
67
-
68
- CREATE OR REPLACE FUNCTION lock_head(tname varchar)
69
- RETURNS SETOF queue_classic_jobs AS $$
70
- BEGIN
71
- RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
72
- END;
73
- $$ LANGUAGE plpgsql;
74
-
75
1
  -- queue_classic_notify function and trigger
76
- create function queue_classic_notify() returns trigger as $$ begin
77
- perform pg_notify(new.q_name, '');
78
- return null;
79
- end $$ language plpgsql;
2
+ CREATE FUNCTION queue_classic_notify() RETURNS TRIGGER AS $$ BEGIN
3
+ perform pg_notify(new.q_name, ''); RETURN NULL;
4
+ END $$ LANGUAGE plpgsql;
80
5
 
81
- create trigger queue_classic_notify
82
- after insert on queue_classic_jobs
83
- for each row
84
- execute procedure queue_classic_notify();
6
+ CREATE TRIGGER queue_classic_notify
7
+ AFTER INSERT ON queue_classic_jobs FOR EACH ROW
8
+ EXECUTE PROCEDURE queue_classic_notify();
@@ -0,0 +1,88 @@
1
+ DO $$DECLARE r record;
2
+ BEGIN
3
+ -- If jsonb type is available, do nothing as we're downgrading from 4.0.0
4
+ IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'jsonb') THEN
5
+ -- do nothing - it should already be already jsonb
6
+ -- Otherwise, use json type for the args column if available
7
+ ELSIF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'json') THEN
8
+ -- this should only happen if someone downgrades QC and their database < pg 9.4
9
+ ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE json USING args::json;
10
+ END IF;
11
+
12
+
13
+ END$$;
14
+
15
+
16
+ --
17
+ -- Re install the lock_head function
18
+ --
19
+
20
+ -- We are declaring the return type to be queue_classic_jobs.
21
+ -- This is ok since I am assuming that all of the users added queues will
22
+ -- have identical columns to queue_classic_jobs.
23
+ -- When QC supports queues with columns other than the default, we will have to change this.
24
+
25
+ CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
26
+ RETURNS SETOF queue_classic_jobs AS $$
27
+ DECLARE
28
+ unlocked bigint;
29
+ relative_top integer;
30
+ job_count integer;
31
+ BEGIN
32
+ -- The purpose is to release contention for the first spot in the table.
33
+ -- The select count(*) is going to slow down dequeue performance but allow
34
+ -- for more workers. Would love to see some optimization here...
35
+
36
+ EXECUTE 'SELECT count(*) FROM '
37
+ || '(SELECT * FROM queue_classic_jobs '
38
+ || ' WHERE locked_at IS NULL'
39
+ || ' AND q_name = '
40
+ || quote_literal(q_name)
41
+ || ' AND scheduled_at <= '
42
+ || quote_literal(now())
43
+ || ' LIMIT '
44
+ || quote_literal(top_boundary)
45
+ || ') limited'
46
+ INTO job_count;
47
+
48
+ SELECT TRUNC(random() * (top_boundary - 1))
49
+ INTO relative_top;
50
+
51
+ IF job_count < top_boundary THEN
52
+ relative_top = 0;
53
+ END IF;
54
+
55
+ LOOP
56
+ BEGIN
57
+ EXECUTE 'SELECT id FROM queue_classic_jobs '
58
+ || ' WHERE locked_at IS NULL'
59
+ || ' AND q_name = '
60
+ || quote_literal(q_name)
61
+ || ' AND scheduled_at <= '
62
+ || quote_literal(now())
63
+ || ' ORDER BY id ASC'
64
+ || ' LIMIT 1'
65
+ || ' OFFSET ' || quote_literal(relative_top)
66
+ || ' FOR UPDATE NOWAIT'
67
+ INTO unlocked;
68
+ EXIT;
69
+ EXCEPTION
70
+ WHEN lock_not_available THEN
71
+ -- do nothing. loop again and hope we get a lock
72
+ END;
73
+ END LOOP;
74
+
75
+ RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
76
+ || ' SET locked_at = (CURRENT_TIMESTAMP),'
77
+ || ' locked_by = (select pg_backend_pid())'
78
+ || ' WHERE id = $1'
79
+ || ' AND locked_at is NULL'
80
+ || ' RETURNING *'
81
+ USING unlocked;
82
+
83
+ RETURN;
84
+ END $$ LANGUAGE plpgsql;
85
+
86
+ CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$ BEGIN
87
+ RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
88
+ END $$ LANGUAGE plpgsql;
@@ -1,10 +1,10 @@
1
1
  DO $$DECLARE r record;
2
2
  BEGIN
3
- BEGIN
4
- ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz default now();
5
- EXCEPTION
6
- WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
7
- END;
3
+ BEGIN
4
+ ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz DEFAULT now();
5
+ EXCEPTION
6
+ WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
7
+ END;
8
8
  END$$;
9
9
 
10
10
  DO $$DECLARE r record;
@@ -1,9 +1,9 @@
1
1
  DO $$DECLARE r record;
2
2
  BEGIN
3
- BEGIN
4
- ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz default now();
5
- CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
6
- EXCEPTION
7
- WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
8
- END;
3
+ BEGIN
4
+ ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz DEFAULT now();
5
+ CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
6
+ EXCEPTION
7
+ WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
8
+ END;
9
9
  END$$;
@@ -0,0 +1,6 @@
1
+ DO $$DECLARE r record;
2
+ BEGIN
3
+ ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE jsonb USING args::jsonb;
4
+ DROP FUNCTION IF EXISTS lock_head(tname varchar);
5
+ DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer);
6
+ END$$;
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative 'helper'
2
4
 
3
5
  if ENV["QC_BENCHMARK"]
data/test/config_test.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative 'helper'
2
4
 
3
5
  class ConfigTest < QCTest
data/test/helper.rb CHANGED
@@ -1,6 +1,16 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "bundler"
4
+ require "minitest/reporters"
5
+
2
6
  Bundler.setup :default, :test
3
7
 
8
+ if ENV['CIRCLECI'] == "true"
9
+ Minitest::Reporters.use! Minitest::Reporters::JUnitReporter.new
10
+ else
11
+ Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
12
+ end
13
+
4
14
  ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
5
15
 
6
16
  require_relative '../lib/queue_classic'
@@ -58,4 +68,28 @@ class QCTest < Minitest::Test
58
68
  ensure
59
69
  original_environment.each { |name, value| ENV[name] = value }
60
70
  end
71
+
72
+ def stub_any_instance(class_name, method_name, definition)
73
+ new_method_name = "new_#{method_name}"
74
+ original_method_name = "original_#{method_name}"
75
+
76
+ method_present = class_name.instance_methods(false).include? method_name
77
+
78
+ if method_present
79
+ class_name.send(:alias_method, original_method_name, method_name)
80
+ class_name.send(:define_method, new_method_name, definition)
81
+ class_name.send(:alias_method, method_name, new_method_name)
82
+
83
+ yield
84
+ else
85
+ message = "#{class_name} does not have method #{method_name}."
86
+ message << "\nAvailable methods: #{class_name.instance_methods(false)}"
87
+ raise ArgumentError.new message
88
+ end
89
+ ensure
90
+ if method_present
91
+ class_name.send(:alias_method, method_name, original_method_name)
92
+ class_name.send(:undef_method, new_method_name)
93
+ end
94
+ end
61
95
  end