queue_classic 3.2.0.RC1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/.github/workflows/codeql-analysis.yml +72 -0
  3. data/.github/workflows/main.yaml +71 -0
  4. data/.gitignore +2 -0
  5. data/{changelog → CHANGELOG.md} +87 -34
  6. data/CODE_OF_CONDUCT.md +46 -0
  7. data/Gemfile +9 -5
  8. data/README.md +103 -126
  9. data/Rakefile +2 -0
  10. data/lib/generators/queue_classic/install_generator.rb +6 -0
  11. data/lib/generators/queue_classic/templates/add_queue_classic.rb +3 -1
  12. data/lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb +3 -1
  13. data/lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb +3 -1
  14. data/lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb +3 -1
  15. data/lib/generators/queue_classic/templates/update_queue_classic_4_0_0.rb +11 -0
  16. data/lib/queue_classic/config.rb +2 -1
  17. data/lib/queue_classic/conn_adapter.rb +29 -15
  18. data/lib/queue_classic/queue.rb +66 -12
  19. data/lib/queue_classic/railtie.rb +2 -0
  20. data/lib/queue_classic/setup.rb +24 -7
  21. data/lib/queue_classic/tasks.rb +4 -5
  22. data/lib/queue_classic/version.rb +3 -1
  23. data/lib/queue_classic/worker.rb +15 -6
  24. data/lib/queue_classic.rb +4 -11
  25. data/queue_classic.gemspec +3 -2
  26. data/sql/create_table.sql +7 -16
  27. data/sql/ddl.sql +6 -82
  28. data/sql/downgrade_from_4_0_0.sql +88 -0
  29. data/sql/update_to_3_0_0.sql +5 -5
  30. data/sql/update_to_3_1_0.sql +6 -6
  31. data/sql/update_to_4_0_0.sql +6 -0
  32. data/test/benchmark_test.rb +2 -0
  33. data/test/config_test.rb +2 -0
  34. data/test/helper.rb +34 -0
  35. data/test/lib/queue_classic_rails_connection_test.rb +9 -6
  36. data/test/lib/queue_classic_test.rb +2 -0
  37. data/test/lib/queue_classic_test_with_activerecord_typecast.rb +21 -0
  38. data/test/queue_test.rb +62 -2
  39. data/test/rails-tests/.gitignore +2 -0
  40. data/test/rails-tests/rails523.sh +23 -0
  41. data/test/worker_test.rb +138 -17
  42. metadata +43 -13
  43. data/.travis.yml +0 -15
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative 'conn_adapter'
2
4
  require 'json'
3
5
  require 'time'
@@ -5,8 +7,8 @@ require 'time'
5
7
  module QC
6
8
  # The queue class maps a queue abstraction onto a database table.
7
9
  class Queue
8
-
9
10
  attr_reader :name, :top_bound
11
+
10
12
  def initialize(name, top_bound=nil)
11
13
  @name = name
12
14
  @top_bound = top_bound || QC.top_bound
@@ -38,7 +40,16 @@ module QC
38
40
  def enqueue(method, *args)
39
41
  QC.log_yield(:measure => 'queue.enqueue') do
40
42
  s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
41
- conn_adapter.execute(s, name, method, JSON.dump(args))
43
+ begin
44
+ retries ||= 0
45
+ conn_adapter.execute(s, name, method, JSON.dump(args))
46
+ rescue PG::Error
47
+ if (retries += 1) < 2
48
+ retry
49
+ else
50
+ raise
51
+ end
52
+ end
42
53
  end
43
54
  end
44
55
 
@@ -64,21 +75,49 @@ module QC
64
75
  s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
65
76
  VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
66
77
  RETURNING id"
67
- conn_adapter.execute(s, name, method, JSON.dump(args))
78
+ begin
79
+ retries ||= 0
80
+ conn_adapter.execute(s, name, method, JSON.dump(args))
81
+ rescue PG::Error
82
+ if (retries += 1) < 2
83
+ retry
84
+ else
85
+ raise
86
+ end
87
+ end
68
88
  end
69
89
  end
70
90
 
71
91
  def lock
72
92
  QC.log_yield(:measure => 'queue.lock') do
73
- s = "SELECT * FROM lock_head($1, $2)"
74
- if r = conn_adapter.execute(s, name, top_bound)
93
+ s = <<~SQL
94
+ WITH selected_job AS (
95
+ SELECT id
96
+ FROM queue_classic_jobs
97
+ WHERE
98
+ locked_at IS NULL AND
99
+ q_name = $1 AND
100
+ scheduled_at <= now()
101
+ LIMIT 1
102
+ FOR NO KEY UPDATE SKIP LOCKED
103
+ )
104
+ UPDATE queue_classic_jobs
105
+ SET
106
+ locked_at = now(),
107
+ locked_by = pg_backend_pid()
108
+ FROM selected_job
109
+ WHERE queue_classic_jobs.id = selected_job.id
110
+ RETURNING *
111
+ SQL
112
+
113
+ if r = conn_adapter.execute(s, name)
75
114
  {}.tap do |job|
76
115
  job[:id] = r["id"]
77
116
  job[:q_name] = r["q_name"]
78
117
  job[:method] = r["method"]
79
118
  job[:args] = JSON.parse(r["args"])
80
119
  if r["scheduled_at"]
81
- job[:scheduled_at] = Time.parse(r["scheduled_at"])
120
+ job[:scheduled_at] = r["scheduled_at"].kind_of?(Time) ? r["scheduled_at"] : Time.parse(r["scheduled_at"])
82
121
  ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
83
122
  QC.measure("time-to-lock=#{ttl}ms source=#{name}")
84
123
  end
@@ -89,14 +128,14 @@ module QC
89
128
 
90
129
  def unlock(id)
91
130
  QC.log_yield(:measure => 'queue.unlock') do
92
- s = "UPDATE #{QC.table_name} set locked_at = null where id = $1"
131
+ s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1"
93
132
  conn_adapter.execute(s, id)
94
133
  end
95
134
  end
96
135
 
97
136
  def delete(id)
98
137
  QC.log_yield(:measure => 'queue.delete') do
99
- conn_adapter.execute("DELETE FROM #{QC.table_name} where id = $1", id)
138
+ conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id)
100
139
  end
101
140
  end
102
141
 
@@ -107,13 +146,28 @@ module QC
107
146
  end
108
147
  end
109
148
 
149
+ # Count the number of jobs in a specific queue. This returns all
150
+ # jobs, including ones that are scheduled in the future.
110
151
  def count
111
- QC.log_yield(:measure => 'queue.count') do
112
- s = "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1"
113
- r = conn_adapter.execute(s, name)
152
+ _count('queue.count', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1")
153
+ end
154
+
155
+ # Count the number of jobs in a specific queue, except ones scheduled in the future
156
+ def count_ready
157
+ _count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at <= now()")
158
+ end
159
+
160
+ # Count the number of jobs in a specific queue scheduled in the future
161
+ def count_scheduled
162
+ _count('queue.count_scheduled', "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1 AND scheduled_at > now()")
163
+ end
164
+
165
+ private
166
+ def _count(metric_name, sql)
167
+ QC.log_yield(measure: metric_name) do
168
+ r = conn_adapter.execute(sql, name)
114
169
  r["count"].to_i
115
170
  end
116
171
  end
117
-
118
172
  end
119
173
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'rails/railtie'
2
4
 
3
5
  module QC
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QC
2
4
  module Setup
3
5
  Root = File.expand_path("../..", File.dirname(__FILE__))
@@ -8,51 +10,66 @@ module QC
8
10
  DowngradeFrom_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
9
11
  UpgradeTo_3_1_0 = File.join(Root, "/sql/update_to_3_1_0.sql")
10
12
  DowngradeFrom_3_1_0 = File.join(Root, "/sql/downgrade_from_3_1_0.sql")
13
+ UpgradeTo_4_0_0 = File.join(Root, "/sql/update_to_4_0_0.sql")
14
+ DowngradeFrom_4_0_0 = File.join(Root, "/sql/downgrade_from_4_0_0.sql")
11
15
 
12
16
  def self.create(c = QC::default_conn_adapter.connection)
13
- conn = QC::ConnAdapter.new(c)
17
+ conn = QC::ConnAdapter.new(connection: c)
14
18
  conn.execute(File.read(CreateTable))
15
19
  conn.execute(File.read(SqlFunctions))
16
20
  conn.disconnect if c.nil? #Don't close a conn we didn't create.
17
21
  end
18
22
 
19
23
  def self.drop(c = QC::default_conn_adapter.connection)
20
- conn = QC::ConnAdapter.new(c)
24
+ conn = QC::ConnAdapter.new(connection: c)
21
25
  conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
22
26
  conn.execute(File.read(DropSqlFunctions))
23
27
  conn.disconnect if c.nil? #Don't close a conn we didn't create.
24
28
  end
25
29
 
26
30
  def self.update(c = QC::default_conn_adapter.connection)
27
- conn = QC::ConnAdapter.new(c)
31
+ conn = QC::ConnAdapter.new(connection: c)
28
32
  conn.execute(File.read(UpgradeTo_3_0_0))
29
33
  conn.execute(File.read(UpgradeTo_3_1_0))
34
+ conn.execute(File.read(UpgradeTo_4_0_0))
30
35
  conn.execute(File.read(DropSqlFunctions))
31
36
  conn.execute(File.read(SqlFunctions))
32
37
  end
33
38
 
34
39
  def self.update_to_3_0_0(c = QC::default_conn_adapter.connection)
35
- conn = QC::ConnAdapter.new(c)
40
+ conn = QC::ConnAdapter.new(connection: c)
36
41
  conn.execute(File.read(UpgradeTo_3_0_0))
37
42
  conn.execute(File.read(DropSqlFunctions))
38
43
  conn.execute(File.read(SqlFunctions))
39
44
  end
40
45
 
41
46
  def self.downgrade_from_3_0_0(c = QC::default_conn_adapter.connection)
42
- conn = QC::ConnAdapter.new(c)
47
+ conn = QC::ConnAdapter.new(connection: c)
43
48
  conn.execute(File.read(DowngradeFrom_3_0_0))
44
49
  end
45
50
 
46
51
  def self.update_to_3_1_0(c = QC::default_conn_adapter.connection)
47
- conn = QC::ConnAdapter.new(c)
52
+ conn = QC::ConnAdapter.new(connection: c)
48
53
  conn.execute(File.read(UpgradeTo_3_1_0))
49
54
  conn.execute(File.read(DropSqlFunctions))
50
55
  conn.execute(File.read(SqlFunctions))
51
56
  end
52
57
 
53
58
  def self.downgrade_from_3_1_0(c = QC::default_conn_adapter.connection)
54
- conn = QC::ConnAdapter.new(c)
59
+ conn = QC::ConnAdapter.new(connection: c)
55
60
  conn.execute(File.read(DowngradeFrom_3_1_0))
56
61
  end
62
+
63
+ def self.update_to_4_0_0(c = QC::default_conn_adapter.connection)
64
+ conn = QC::ConnAdapter.new(connection: c)
65
+ conn.execute(File.read(UpgradeTo_4_0_0))
66
+ conn.execute(File.read(DropSqlFunctions))
67
+ conn.execute(File.read(SqlFunctions))
68
+ end
69
+
70
+ def self.downgrade_from_4_0_0(c = QC::default_conn_adapter.connection)
71
+ conn = QC::ConnAdapter.new(connection: c)
72
+ conn.execute(File.read(DowngradeFrom_4_0_0))
73
+ end
57
74
  end
58
75
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  task :environment
2
4
 
3
5
  namespace :jobs do
@@ -11,11 +13,8 @@ namespace :qc do
11
13
  @worker = QC.default_worker_class.new
12
14
 
13
15
  trap('INT') do
14
- $stderr.puts("Received INT. Shutting down.")
15
- if !@worker.running
16
- $stderr.puts("Worker has stopped running. Exit.")
17
- exit(1)
18
- end
16
+ $stderr.puts("Received INT. Shutting down.")
17
+ abort("Worker has stopped running. Exit.") unless @worker.running
19
18
  @worker.stop
20
19
  end
21
20
 
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module QC
2
- VERSION = "3.2.0.RC1"
4
+ VERSION = "4.0.0"
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  # -*- coding: utf-8 -*-
2
4
  require_relative 'queue'
3
5
  require_relative 'conn_adapter'
@@ -12,7 +14,7 @@ module QC
12
14
  # This method takes a single hash argument. The following keys are read:
13
15
  # fork_worker:: Worker forks each job execution.
14
16
  # wait_interval:: Time to wait between failed lock attempts
15
- # connection:: PGConn object.
17
+ # connection:: PG::Connection object.
16
18
  # q_name:: Name of a single queue to process.
17
19
  # q_names:: Names of queues to process. Will process left to right.
18
20
  # top_bound:: Offset to the head of the queue. 1 == strict FIFO.
@@ -21,7 +23,7 @@ module QC
21
23
  @wait_interval = args[:wait_interval] || QC.wait_time
22
24
 
23
25
  if args[:connection]
24
- @conn_adapter = ConnAdapter.new(args[:connection])
26
+ @conn_adapter = ConnAdapter.new(connection: args[:connection])
25
27
  else
26
28
  @conn_adapter = QC.default_conn_adapter
27
29
  end
@@ -109,10 +111,13 @@ module QC
109
111
  finished = false
110
112
  begin
111
113
  call(job).tap do
112
- queue.delete(job[:id])
114
+ handle_success(queue, job)
113
115
  finished = true
114
116
  end
115
- rescue => e
117
+ rescue StandardError, ScriptError, NoMemoryError => e
118
+ # We really only want to unlock the job for signal and system exit
119
+ # exceptions. If we encounter a ScriptError or a NoMemoryError any
120
+ # future run will likely encounter the same error.
116
121
  handle_failure(job, e)
117
122
  finished = true
118
123
  ensure
@@ -134,8 +139,12 @@ module QC
134
139
  receiver.send(message, *args)
135
140
  end
136
141
 
137
- # This method will be called when an exception
138
- # is raised during the execution of the job.
142
+ def handle_success(queue, job)
143
+ queue.delete(job[:id])
144
+ end
145
+
146
+ # This method will be called when a StandardError, ScriptError or
147
+ # NoMemoryError is raised during the execution of the job.
139
148
  def handle_failure(job,e)
140
149
  $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
141
150
  end
data/lib/queue_classic.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative "queue_classic/config"
2
4
 
3
5
  module QC
@@ -49,15 +51,7 @@ Please use the method QC.#{config_method} instead.
49
51
  end
50
52
 
51
53
  def self.default_conn_adapter
52
- t = Thread.current
53
- return t[:qc_conn_adapter] if t[:qc_conn_adapter]
54
- adapter = if rails_connection_sharing_enabled?
55
- ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
56
- else
57
- ConnAdapter.new
58
- end
59
-
60
- t[:qc_conn_adapter] = adapter
54
+ Thread.current[:qc_conn_adapter] ||= ConnAdapter.new(active_record_connection_share: rails_connection_sharing_enabled?)
61
55
  end
62
56
 
63
57
  def self.default_conn_adapter=(conn)
@@ -100,8 +94,7 @@ Please use the method QC.#{config_method} instead.
100
94
  # This will unlock all jobs any postgres' PID that is not existing anymore
101
95
  # to prevent any infinitely locked jobs
102
96
  def self.unlock_jobs_of_dead_workers
103
- pid_column = default_conn_adapter.server_version < 90200 ? "procpid" : "pid"
104
- default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT #{pid_column} FROM pg_stat_activity);")
97
+ default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);")
105
98
  end
106
99
 
107
100
  # private class methods
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
10
10
  spec.description = "queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic only requires pg."
11
11
  spec.summary = "Simple, efficient worker queue for Ruby & PostgreSQL."
12
12
  spec.authors = ["Ryan Smith (♠ ace hacker)"]
13
- spec.homepage = "http://github.com/QueueClassic/queue_classic"
13
+ spec.homepage = "https://github.com/QueueClassic/queue_classic"
14
14
  spec.license = "MIT"
15
15
 
16
16
  spec.files = `git ls-files -z`.split("\x0")
@@ -20,5 +20,6 @@ Gem::Specification.new do |spec|
20
20
 
21
21
  spec.require_paths = %w[lib]
22
22
 
23
- spec.add_dependency "pg", ">= 0.17", "< 0.19"
23
+ spec.add_dependency "pg", ">= 1.1", "< 2.0"
24
+ spec.add_development_dependency "activerecord", ">= 5.0.0", "< 6.1"
24
25
  end
data/sql/create_table.sql CHANGED
@@ -1,26 +1,17 @@
1
- do $$ begin
1
+ DO $$ BEGIN
2
2
 
3
3
  CREATE TABLE queue_classic_jobs (
4
4
  id bigserial PRIMARY KEY,
5
- q_name text not null check (length(q_name) > 0),
6
- method text not null check (length(method) > 0),
7
- args text not null,
5
+ q_name text NOT NULL CHECK (length(q_name) > 0),
6
+ method text NOT NULL CHECK (length(method) > 0),
7
+ args jsonb NOT NULL,
8
8
  locked_at timestamptz,
9
9
  locked_by integer,
10
- created_at timestamptz default now(),
11
- scheduled_at timestamptz default now()
10
+ created_at timestamptz DEFAULT now(),
11
+ scheduled_at timestamptz DEFAULT now()
12
12
  );
13
13
 
14
- -- If jsonb type is available, use it for the args column
15
- if exists (select 1 from pg_type where typname = 'jsonb') then
16
- alter table queue_classic_jobs alter column args type jsonb using args::jsonb;
17
- -- Otherwise, use json type for the args column if available
18
- elsif exists (select 1 from pg_type where typname = 'json') then
19
- alter table queue_classic_jobs alter column args type json using args::json;
20
- end if;
21
-
22
- end $$ language plpgsql;
14
+ END $$ LANGUAGE plpgsql;
23
15
 
24
16
  CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
25
17
  CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
26
-
data/sql/ddl.sql CHANGED
@@ -1,84 +1,8 @@
1
- -- We are declaring the return type to be queue_classic_jobs.
2
- -- This is ok since I am assuming that all of the users added queues will
3
- -- have identical columns to queue_classic_jobs.
4
- -- When QC supports queues with columns other than the default, we will have to change this.
5
-
6
- CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
7
- RETURNS SETOF queue_classic_jobs AS $$
8
- DECLARE
9
- unlocked bigint;
10
- relative_top integer;
11
- job_count integer;
12
- BEGIN
13
- -- The purpose is to release contention for the first spot in the table.
14
- -- The select count(*) is going to slow down dequeue performance but allow
15
- -- for more workers. Would love to see some optimization here...
16
-
17
- EXECUTE 'SELECT count(*) FROM '
18
- || '(SELECT * FROM queue_classic_jobs '
19
- || ' WHERE locked_at IS NULL'
20
- || ' AND q_name = '
21
- || quote_literal(q_name)
22
- || ' AND scheduled_at <= '
23
- || quote_literal(now())
24
- || ' LIMIT '
25
- || quote_literal(top_boundary)
26
- || ') limited'
27
- INTO job_count;
28
-
29
- SELECT TRUNC(random() * (top_boundary - 1))
30
- INTO relative_top;
31
-
32
- IF job_count < top_boundary THEN
33
- relative_top = 0;
34
- END IF;
35
-
36
- LOOP
37
- BEGIN
38
- EXECUTE 'SELECT id FROM queue_classic_jobs '
39
- || ' WHERE locked_at IS NULL'
40
- || ' AND q_name = '
41
- || quote_literal(q_name)
42
- || ' AND scheduled_at <= '
43
- || quote_literal(now())
44
- || ' ORDER BY id ASC'
45
- || ' LIMIT 1'
46
- || ' OFFSET ' || quote_literal(relative_top)
47
- || ' FOR UPDATE NOWAIT'
48
- INTO unlocked;
49
- EXIT;
50
- EXCEPTION
51
- WHEN lock_not_available THEN
52
- -- do nothing. loop again and hope we get a lock
53
- END;
54
- END LOOP;
55
-
56
- RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
57
- || ' SET locked_at = (CURRENT_TIMESTAMP),'
58
- || ' locked_by = (select pg_backend_pid())'
59
- || ' WHERE id = $1'
60
- || ' AND locked_at is NULL'
61
- || ' RETURNING *'
62
- USING unlocked;
63
-
64
- RETURN;
65
- END;
66
- $$ LANGUAGE plpgsql;
67
-
68
- CREATE OR REPLACE FUNCTION lock_head(tname varchar)
69
- RETURNS SETOF queue_classic_jobs AS $$
70
- BEGIN
71
- RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
72
- END;
73
- $$ LANGUAGE plpgsql;
74
-
75
1
  -- queue_classic_notify function and trigger
76
- create function queue_classic_notify() returns trigger as $$ begin
77
- perform pg_notify(new.q_name, '');
78
- return null;
79
- end $$ language plpgsql;
2
+ CREATE FUNCTION queue_classic_notify() RETURNS TRIGGER AS $$ BEGIN
3
+ perform pg_notify(new.q_name, ''); RETURN NULL;
4
+ END $$ LANGUAGE plpgsql;
80
5
 
81
- create trigger queue_classic_notify
82
- after insert on queue_classic_jobs
83
- for each row
84
- execute procedure queue_classic_notify();
6
+ CREATE TRIGGER queue_classic_notify
7
+ AFTER INSERT ON queue_classic_jobs FOR EACH ROW
8
+ EXECUTE PROCEDURE queue_classic_notify();
@@ -0,0 +1,88 @@
1
+ DO $$DECLARE r record;
2
+ BEGIN
3
+ -- If jsonb type is available, do nothing as we're downgrading from 4.0.0
4
+ IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'jsonb') THEN
5
+ -- do nothing - it should already be already jsonb
6
+ -- Otherwise, use json type for the args column if available
7
+ ELSIF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'json') THEN
8
+ -- this should only happen if someone downgrades QC and their database < pg 9.4
9
+ ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE json USING args::json;
10
+ END IF;
11
+
12
+
13
+ END$$;
14
+
15
+
16
+ --
17
+ -- Re install the lock_head function
18
+ --
19
+
20
+ -- We are declaring the return type to be queue_classic_jobs.
21
+ -- This is ok since I am assuming that all of the users added queues will
22
+ -- have identical columns to queue_classic_jobs.
23
+ -- When QC supports queues with columns other than the default, we will have to change this.
24
+
25
+ CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
26
+ RETURNS SETOF queue_classic_jobs AS $$
27
+ DECLARE
28
+ unlocked bigint;
29
+ relative_top integer;
30
+ job_count integer;
31
+ BEGIN
32
+ -- The purpose is to release contention for the first spot in the table.
33
+ -- The select count(*) is going to slow down dequeue performance but allow
34
+ -- for more workers. Would love to see some optimization here...
35
+
36
+ EXECUTE 'SELECT count(*) FROM '
37
+ || '(SELECT * FROM queue_classic_jobs '
38
+ || ' WHERE locked_at IS NULL'
39
+ || ' AND q_name = '
40
+ || quote_literal(q_name)
41
+ || ' AND scheduled_at <= '
42
+ || quote_literal(now())
43
+ || ' LIMIT '
44
+ || quote_literal(top_boundary)
45
+ || ') limited'
46
+ INTO job_count;
47
+
48
+ SELECT TRUNC(random() * (top_boundary - 1))
49
+ INTO relative_top;
50
+
51
+ IF job_count < top_boundary THEN
52
+ relative_top = 0;
53
+ END IF;
54
+
55
+ LOOP
56
+ BEGIN
57
+ EXECUTE 'SELECT id FROM queue_classic_jobs '
58
+ || ' WHERE locked_at IS NULL'
59
+ || ' AND q_name = '
60
+ || quote_literal(q_name)
61
+ || ' AND scheduled_at <= '
62
+ || quote_literal(now())
63
+ || ' ORDER BY id ASC'
64
+ || ' LIMIT 1'
65
+ || ' OFFSET ' || quote_literal(relative_top)
66
+ || ' FOR UPDATE NOWAIT'
67
+ INTO unlocked;
68
+ EXIT;
69
+ EXCEPTION
70
+ WHEN lock_not_available THEN
71
+ -- do nothing. loop again and hope we get a lock
72
+ END;
73
+ END LOOP;
74
+
75
+ RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
76
+ || ' SET locked_at = (CURRENT_TIMESTAMP),'
77
+ || ' locked_by = (select pg_backend_pid())'
78
+ || ' WHERE id = $1'
79
+ || ' AND locked_at is NULL'
80
+ || ' RETURNING *'
81
+ USING unlocked;
82
+
83
+ RETURN;
84
+ END $$ LANGUAGE plpgsql;
85
+
86
+ CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$ BEGIN
87
+ RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
88
+ END $$ LANGUAGE plpgsql;
@@ -1,10 +1,10 @@
1
1
  DO $$DECLARE r record;
2
2
  BEGIN
3
- BEGIN
4
- ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz default now();
5
- EXCEPTION
6
- WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
7
- END;
3
+ BEGIN
4
+ ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz DEFAULT now();
5
+ EXCEPTION
6
+ WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
7
+ END;
8
8
  END$$;
9
9
 
10
10
  DO $$DECLARE r record;
@@ -1,9 +1,9 @@
1
1
  DO $$DECLARE r record;
2
2
  BEGIN
3
- BEGIN
4
- ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz default now();
5
- CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
6
- EXCEPTION
7
- WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
8
- END;
3
+ BEGIN
4
+ ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz DEFAULT now();
5
+ CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
6
+ EXCEPTION
7
+ WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
8
+ END;
9
9
  END$$;
@@ -0,0 +1,6 @@
1
+ DO $$DECLARE r record;
2
+ BEGIN
3
+ ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE jsonb USING args::jsonb;
4
+ DROP FUNCTION IF EXISTS lock_head(tname varchar);
5
+ DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer);
6
+ END$$;
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative 'helper'
2
4
 
3
5
  if ENV["QC_BENCHMARK"]
data/test/config_test.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require_relative 'helper'
2
4
 
3
5
  class ConfigTest < QCTest
data/test/helper.rb CHANGED
@@ -1,6 +1,16 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "bundler"
4
+ require "minitest/reporters"
5
+
2
6
  Bundler.setup :default, :test
3
7
 
8
+ if ENV['CIRCLECI'] == "true"
9
+ Minitest::Reporters.use! Minitest::Reporters::JUnitReporter.new
10
+ else
11
+ Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
12
+ end
13
+
4
14
  ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
5
15
 
6
16
  require_relative '../lib/queue_classic'
@@ -58,4 +68,28 @@ class QCTest < Minitest::Test
58
68
  ensure
59
69
  original_environment.each { |name, value| ENV[name] = value }
60
70
  end
71
+
72
+ def stub_any_instance(class_name, method_name, definition)
73
+ new_method_name = "new_#{method_name}"
74
+ original_method_name = "original_#{method_name}"
75
+
76
+ method_present = class_name.instance_methods(false).include? method_name
77
+
78
+ if method_present
79
+ class_name.send(:alias_method, original_method_name, method_name)
80
+ class_name.send(:define_method, new_method_name, definition)
81
+ class_name.send(:alias_method, method_name, new_method_name)
82
+
83
+ yield
84
+ else
85
+ message = "#{class_name} does not have method #{method_name}."
86
+ message << "\nAvailable methods: #{class_name.instance_methods(false)}"
87
+ raise ArgumentError.new message
88
+ end
89
+ ensure
90
+ if method_present
91
+ class_name.send(:alias_method, method_name, original_method_name)
92
+ class_name.send(:undef_method, new_method_name)
93
+ end
94
+ end
61
95
  end