skiplock 1.0.24 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -4
- data/bin/skiplock +1 -1
- data/lib/generators/skiplock/templates/migration.rb.erb +25 -19
- data/lib/skiplock/job.rb +5 -4
- data/lib/skiplock/manager.rb +6 -6
- data/lib/skiplock/version.rb +1 -1
- data/lib/skiplock/worker.rb +9 -19
- data/lib/skiplock.rb +9 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c119463794e6ac523fb385c7925181bf069d6cf37996fe4a2dacf101e6faa6c8
|
4
|
+
data.tar.gz: 0aee11414e05623185781b1a5779a9b7393474fde176ec6d9ee42151cd5453a4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ba80ed8e0bd4bf185ffa2e65f88d7f6a1f32e1124e75e82dac50ba1fecc12167687ffab2e5f8f9e7148ba24c000ae6114c04c67e175793c99a45db17dde8b0c
|
7
|
+
data.tar.gz: 0adb1907dd1e0c8338c298d32f6fe3e576f755a38962beed186c56ca4375882cbbf20646a7efd2cf37145588d7acf6dd4855156f418de9732254ec5b86704e9a
|
data/README.md
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
It only uses the `LISTEN/NOTIFY/SKIP LOCKED` features provided natively on PostgreSQL 9.5+ to efficiently and reliably dispatch jobs to worker processes and threads ensuring that each job can be completed successfully **only once**. No other polling or timer is needed.
|
6
6
|
|
7
|
-
The library is quite small compared to other PostgreSQL job queues (eg. *delay_job*, *queue_classic*, *que*, *good_job*) with less than
|
7
|
+
The library is quite small compared to other PostgreSQL job queues (eg. *delay_job*, *queue_classic*, *que*, *good_job*) with less than 600 lines of codes; and it still provides similar set of features and more...
|
8
8
|
|
9
9
|
#### Compatibility:
|
10
10
|
|
@@ -56,8 +56,8 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
|
|
56
56
|
max_retries: 20
|
57
57
|
logfile: skiplock.log
|
58
58
|
loglevel: info
|
59
|
+
namespace:
|
59
60
|
notification: custom
|
60
|
-
actioncable: false
|
61
61
|
extensions: false
|
62
62
|
purge_completion: true
|
63
63
|
queues:
|
@@ -72,8 +72,8 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
|
|
72
72
|
- **max_retries** (*integer*): sets the maximum attempt a job will be retrying before it is marked expired. See `Retry system` for more details
|
73
73
|
- **logfile** (*string*): filename for skiplock logs; empty logfile will disable logging
|
74
74
|
- **loglevel** (*string*): sets logging level (`debug, info, warn, error, fatal, unknown`)
|
75
|
+
- **namespace** (*string*): sets namespace for jobs (workers will only process jobs of specified namespace)
|
75
76
|
- **notification** (*string*): sets the library to be used for notifying errors and exceptions (`auto, airbrake, bugsnag, exception_notification, custom`); using `auto` will detect library if available. See `Notification system` for more details
|
76
|
-
- **actioncable** (*boolean*): enable or disable usage of ActionCable notification
|
77
77
|
- **extensions** (*multi*): enable or disable the class method extension. See `ClassMethod extension` for more details
|
78
78
|
- **purge_completion** (*boolean*): when set to **true** will delete jobs after they were completed successfully; if set to **false** then the completed jobs should be purged periodically to maximize performance (eg. clean up old jobs after 3 months); queued jobs can manually override using `purge` option
|
79
79
|
- **queues** (*hash*): defines the set of queues with priorities; lower priority takes precedence
|
@@ -87,10 +87,10 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
|
|
87
87
|
```
|
88
88
|
$ bundle exec skiplock -h
|
89
89
|
Usage: skiplock [options]
|
90
|
-
-a, --actioncable YESNO Actioncable notification
|
91
90
|
-e, --environment STRING Rails environment
|
92
91
|
-l, --logfile STRING Log filename
|
93
92
|
-L, --loglevel STRING Log level (debug, info, warn, error, fatal, unknown)
|
93
|
+
-n, --namespace STRING Job namespace
|
94
94
|
-s, --graceful-shutdown NUM Number of seconds to wait for graceful shutdown
|
95
95
|
-r, --max-retries NUM Number of maxixum retries
|
96
96
|
-t, --max-threads NUM Number of maximum threads
|
data/bin/skiplock
CHANGED
@@ -5,10 +5,10 @@ options = {}
|
|
5
5
|
begin
|
6
6
|
op = OptionParser.new do |opts|
|
7
7
|
opts.banner = "Usage: #{File.basename($0)} [options]"
|
8
|
-
opts.on('-a', '--actioncable YESNO', TrueClass, 'Actioncable notification')
|
9
8
|
opts.on('-e', '--environment STRING', String, 'Rails environment')
|
10
9
|
opts.on('-l', '--logfile STRING', String, 'Log filename')
|
11
10
|
opts.on('-L', '--loglevel STRING', String, 'Log level (debug, info, warn, error, fatal, unknown)')
|
11
|
+
opts.on('-n', '--namespace STRING', String, 'Job namespace')
|
12
12
|
opts.on('-s', '--graceful-shutdown NUM', Integer, 'Number of seconds to wait for graceful shutdown')
|
13
13
|
opts.on('-r', '--max-retries NUM', Integer, 'Number of maxixum retries')
|
14
14
|
opts.on('-t', '--max-threads NUM', Integer, 'Number of maximum threads')
|
@@ -7,11 +7,14 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
7
7
|
t.integer :expiries, null: false, default: 0
|
8
8
|
t.integer :failures, null: false, default: 0
|
9
9
|
t.integer :retries, null: false, default: 0
|
10
|
-
t.
|
10
|
+
t.string :namespace, null: false, default: '', index: true
|
11
|
+
t.date :day, null: false
|
12
|
+
t.index [ :namespace, :day ], unique: true
|
11
13
|
end
|
12
14
|
create_table 'skiplock.jobs', id: :uuid do |t|
|
13
15
|
t.uuid :worker_id, index: true
|
14
16
|
t.string :job_class, null: false
|
17
|
+
t.string :namespace, null: false, default: '', index: true
|
15
18
|
t.string :queue_name, index: true
|
16
19
|
t.string :locale
|
17
20
|
t.string :timezone
|
@@ -31,6 +34,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
31
34
|
t.integer :sid, null: false
|
32
35
|
t.integer :capacity, null: false
|
33
36
|
t.string :hostname, null: false, index: true
|
37
|
+
t.string :namespace, null: false, default: '', index: true
|
34
38
|
t.boolean :master, null: false, default: false, index: true
|
35
39
|
t.jsonb :data
|
36
40
|
t.timestamps null: false, index: true, default: -> { 'now()' }
|
@@ -41,27 +45,29 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
41
45
|
record RECORD;
|
42
46
|
BEGIN
|
43
47
|
record = NEW;
|
44
|
-
IF
|
48
|
+
IF TG_OP = 'DELETE' THEN
|
45
49
|
record = OLD;
|
46
|
-
IF
|
47
|
-
INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
50
|
+
IF record.running IS TRUE THEN
|
51
|
+
INSERT INTO skiplock.counters (namespace,day,completions) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
48
52
|
END IF;
|
49
|
-
ELSIF
|
50
|
-
IF
|
51
|
-
IF
|
52
|
-
INSERT INTO skiplock.counters (day,retries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET retries = skiplock.counters.retries + 1;
|
53
|
+
ELSIF TG_OP = 'UPDATE' THEN
|
54
|
+
IF OLD.running IS FALSE AND record.running IS TRUE THEN
|
55
|
+
IF record.executions > 0 THEN
|
56
|
+
INSERT INTO skiplock.counters (namespace,day,retries) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET retries = skiplock.counters.retries + 1;
|
53
57
|
ELSE
|
54
|
-
INSERT INTO skiplock.counters (day,dispatches) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET dispatches = skiplock.counters.dispatches + 1;
|
58
|
+
INSERT INTO skiplock.counters (namespace,day,dispatches) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET dispatches = skiplock.counters.dispatches + 1;
|
59
|
+
END IF;
|
60
|
+
ELSIF OLD.finished_at IS NULL AND record.finished_at IS NOT NULL THEN
|
61
|
+
INSERT INTO skiplock.counters (namespace,day,completions) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
62
|
+
ELSIF OLD.running IS TRUE AND record.running IS FALSE THEN
|
63
|
+
IF record.expired_at IS NOT NULL THEN
|
64
|
+
INSERT INTO skiplock.counters (namespace,day,expiries) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET expiries = skiplock.counters.expiries + 1;
|
65
|
+
ELSE
|
66
|
+
INSERT INTO skiplock.counters (namespace,day,failures) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET failures = skiplock.counters.failures + 1;
|
55
67
|
END IF;
|
56
|
-
ELSIF (OLD.finished_at IS NULL AND record.finished_at IS NOT NULL) THEN
|
57
|
-
INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
58
|
-
ELSIF (OLD.running = TRUE AND record.running = FALSE AND record.expired_at IS NOT NULL) THEN
|
59
|
-
INSERT INTO skiplock.counters (day,expiries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET expiries = skiplock.counters.expiries + 1;
|
60
|
-
ELSIF (OLD.running = TRUE AND record.running = FALSE AND record.expired_at IS NULL AND record.finished_at IS NULL) THEN
|
61
|
-
INSERT INTO skiplock.counters (day,failures) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET failures = skiplock.counters.failures + 1;
|
62
68
|
END IF;
|
63
69
|
END IF;
|
64
|
-
PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.job_class,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM CASE WHEN record.scheduled_at IS NULL THEN record.updated_at ELSE record.scheduled_at END) AS FLOAT)::TEXT));
|
70
|
+
PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.namespace,',',record.job_class,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM CASE WHEN record.scheduled_at IS NULL THEN record.updated_at ELSE record.scheduled_at END) AS FLOAT)::TEXT));
|
65
71
|
RETURN NULL;
|
66
72
|
END;
|
67
73
|
$$ LANGUAGE plpgsql
|
@@ -71,12 +77,12 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
71
77
|
DECLARE
|
72
78
|
record RECORD;
|
73
79
|
BEGIN
|
74
|
-
IF
|
80
|
+
IF TG_OP = 'DELETE' THEN
|
75
81
|
record = OLD;
|
76
82
|
ELSE
|
77
83
|
record = NEW;
|
78
84
|
END IF;
|
79
|
-
PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.master::TEXT,',',record.capacity,',',record.pid,',',record.sid,',',CAST(EXTRACT(EPOCH FROM record.created_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.updated_at) AS FLOAT)::TEXT));
|
85
|
+
PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.namespace,',',record.hostname,',',record.master::TEXT,',',record.capacity,',',record.pid,',',record.sid,',',CAST(EXTRACT(EPOCH FROM record.created_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.updated_at) AS FLOAT)::TEXT));
|
80
86
|
RETURN NULL;
|
81
87
|
END;
|
82
88
|
$$ LANGUAGE plpgsql;
|
@@ -87,7 +93,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
87
93
|
execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = FALSE AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
|
88
94
|
execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
|
89
95
|
execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
|
90
|
-
execute "CREATE UNIQUE INDEX workers_unique_master_index ON skiplock.workers(hostname) WHERE master = TRUE"
|
96
|
+
execute "CREATE UNIQUE INDEX workers_unique_master_index ON skiplock.workers(hostname,namespace) WHERE master = TRUE"
|
91
97
|
end
|
92
98
|
|
93
99
|
def down
|
data/lib/skiplock/job.rb
CHANGED
@@ -3,14 +3,15 @@ module Skiplock
|
|
3
3
|
self.implicit_order_column = 'updated_at'
|
4
4
|
attribute :activejob_error
|
5
5
|
attribute :exception
|
6
|
-
attribute :max_retries
|
7
|
-
attribute :purge
|
6
|
+
attribute :max_retries
|
7
|
+
attribute :purge
|
8
8
|
belongs_to :worker, inverse_of: :jobs, required: false
|
9
9
|
|
10
10
|
def self.dispatch(purge_completion: true, max_retries: 20)
|
11
|
+
namespace_query = Skiplock.namespace.nil? ? "namespace IS NULL" : "namespace = '#{Skiplock.namespace}'"
|
11
12
|
job = nil
|
12
13
|
self.connection.transaction do
|
13
|
-
job = self.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
14
|
+
job = self.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL AND #{namespace_query} ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
14
15
|
return if job.nil? || job.scheduled_at.to_f > Time.now.to_f
|
15
16
|
job = self.find_by_sql("UPDATE skiplock.jobs SET running = TRUE, updated_at = NOW() WHERE id = '#{job.id}' RETURNING *").first
|
16
17
|
end
|
@@ -33,7 +34,7 @@ module Skiplock
|
|
33
34
|
Thread.current[:skiplock_job]
|
34
35
|
else
|
35
36
|
serialize = activejob.serialize
|
36
|
-
self.create!(serialize.slice(*self.column_names).merge('id' => serialize['job_id'], 'data' => { 'arguments' => serialize['arguments'], 'options' => options }, 'scheduled_at' => timestamp))
|
37
|
+
self.create!(serialize.slice(*self.column_names).merge('id' => serialize['job_id'], 'data' => { 'arguments' => serialize['arguments'], 'options' => options }, 'namespace' => Skiplock.namespace, 'scheduled_at' => timestamp))
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
data/lib/skiplock/manager.rb
CHANGED
@@ -11,6 +11,7 @@ module Skiplock
|
|
11
11
|
@config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
|
12
12
|
end
|
13
13
|
ActiveJob::Base.__send__(:include, Skiplock::Patch)
|
14
|
+
Skiplock.namespace = @config[:namespace]
|
14
15
|
(caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0) ? async : Cron.setup
|
15
16
|
end
|
16
17
|
|
@@ -18,7 +19,7 @@ module Skiplock
|
|
18
19
|
setup_logger
|
19
20
|
configure
|
20
21
|
Worker.cleanup(@hostname)
|
21
|
-
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname
|
22
|
+
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
|
22
23
|
Cron.setup if @worker.master
|
23
24
|
@worker.start(**@config)
|
24
25
|
at_exit { @worker.shutdown }
|
@@ -40,13 +41,13 @@ module Skiplock
|
|
40
41
|
Signal.trap('TERM') { @shutdown = true }
|
41
42
|
Signal.trap('HUP') { setup_logger }
|
42
43
|
Worker.cleanup(@hostname)
|
43
|
-
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname
|
44
|
+
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
|
44
45
|
ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
|
45
46
|
(@config[:workers] - 1).times do |n|
|
46
47
|
fork do
|
47
48
|
sleep(0.25*n + 1)
|
48
49
|
ActiveRecord::Base.establish_connection
|
49
|
-
worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false
|
50
|
+
worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
|
50
51
|
worker.start(worker_num: n + 1, **@config)
|
51
52
|
loop do
|
52
53
|
sleep 0.5
|
@@ -76,7 +77,6 @@ module Skiplock
|
|
76
77
|
@logger.info "-"*(title.length)
|
77
78
|
@logger.info title
|
78
79
|
@logger.info "-"*(title.length)
|
79
|
-
@logger.info "ActionCable notification: #{@config[:actioncable]}#{' (not available)' unless defined?(ActionCable)}"
|
80
80
|
@logger.info " ClassMethod extensions: #{@config[:extensions]}"
|
81
81
|
@logger.info " Purge completion: #{@config[:purge_completion]}"
|
82
82
|
@logger.info " Notification: #{@config[:notification]}"
|
@@ -84,6 +84,7 @@ module Skiplock
|
|
84
84
|
@logger.info " Min threads: #{@config[:min_threads]}"
|
85
85
|
@logger.info " Max threads: #{@config[:max_threads]}"
|
86
86
|
@logger.info " Environment: #{Rails.env}"
|
87
|
+
@logger.info " Namespace: #{@config[:namespace] || '(nil)'}"
|
87
88
|
@logger.info " Loglevel: #{@config[:loglevel]}"
|
88
89
|
@logger.info " Logfile: #{@config[:logfile] || '(disabled)'}"
|
89
90
|
@logger.info " Workers: #{@config[:workers]}"
|
@@ -135,7 +136,6 @@ module Skiplock
|
|
135
136
|
else
|
136
137
|
@config[:notification] = 'custom'
|
137
138
|
end
|
138
|
-
@logger.error 'ActionCable is not found!' if @config[:actioncable] && !defined?(ActionCable)
|
139
139
|
Skiplock.on_errors.freeze
|
140
140
|
end
|
141
141
|
|
@@ -159,4 +159,4 @@ module Skiplock
|
|
159
159
|
Skiplock.on_errors.each { |p| p.call(ex) }
|
160
160
|
end
|
161
161
|
end
|
162
|
-
end
|
162
|
+
end
|
data/lib/skiplock/version.rb
CHANGED
data/lib/skiplock/worker.rb
CHANGED
@@ -5,27 +5,23 @@ module Skiplock
|
|
5
5
|
|
6
6
|
def self.cleanup(hostname = nil)
|
7
7
|
delete_ids = []
|
8
|
-
self.where(hostname: hostname || Socket.gethostname).each do |worker|
|
8
|
+
self.where(namespace: Skiplock.namespace, hostname: hostname || Socket.gethostname).each do |worker|
|
9
9
|
sid = Process.getsid(worker.pid) rescue nil
|
10
10
|
delete_ids << worker.id if worker.sid != sid || worker.updated_at < 10.minutes.ago
|
11
11
|
end
|
12
12
|
self.where(id: delete_ids).delete_all if delete_ids.count > 0
|
13
13
|
end
|
14
14
|
|
15
|
-
def self.generate(capacity:, hostname:, master: true
|
16
|
-
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity)
|
15
|
+
def self.generate(capacity:, hostname:, master: true)
|
16
|
+
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity, namespace: Skiplock.namespace)
|
17
17
|
rescue
|
18
|
-
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity)
|
19
|
-
ensure
|
20
|
-
ActionCable.server.broadcast('skiplock', { worker: { op: 'CREATE', id: worker.id, hostname: worker.hostname, master: worker.master, capacity: worker.capacity, pid: worker.pid, sid: worker.sid, created_at: worker.created_at.to_f, updated_at: worker.updated_at.to_f } }) if actioncable && defined?(ActionCable)
|
18
|
+
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity, namespace: Skiplock.namespace)
|
21
19
|
end
|
22
20
|
|
23
21
|
def shutdown
|
24
22
|
@running = false
|
25
23
|
@executor.shutdown
|
26
24
|
@executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
|
27
|
-
ActionCable.server.broadcast('skiplock', { worker: { op: 'DELETE', id: self.id, hostname: self.hostname, master: self.master, capacity: self.capacity, pid: self.pid, sid: self.sid, created_at: self.created_at.to_f, updated_at: self.updated_at.to_f } }) if @config[:actioncable] && defined?(ActionCable)
|
28
|
-
self.delete
|
29
25
|
Skiplock.logger.info "[Skiplock] Shutdown of #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} (PID: #{self.pid}) was completed."
|
30
26
|
end
|
31
27
|
|
@@ -33,6 +29,7 @@ module Skiplock
|
|
33
29
|
@num = worker_num
|
34
30
|
@config = config
|
35
31
|
@pg_config = ActiveRecord::Base.connection.raw_connection.conninfo_hash.compact
|
32
|
+
@namespace_query = Skiplock.namespace.nil? ? "namespace IS NULL" : "namespace = '#{Skiplock.namespace}'"
|
36
33
|
@queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
|
37
34
|
@running = true
|
38
35
|
@map = ::PG::TypeMapByOid.new
|
@@ -78,7 +75,7 @@ module Skiplock
|
|
78
75
|
if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 1 # reserves 1 slot in queue for Job.flush in case of pg_connection error
|
79
76
|
result = nil
|
80
77
|
@connection.transaction do |conn|
|
81
|
-
conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
|
78
|
+
conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL AND #{@namespace_query} ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
|
82
79
|
result = r.first
|
83
80
|
conn.exec("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *") { |r| result = r.first } if result && result['scheduled_at'].to_f <= Time.now.to_f
|
84
81
|
end
|
@@ -98,17 +95,10 @@ module Skiplock
|
|
98
95
|
notifications[payload[:relname]] << payload[:extra]
|
99
96
|
end
|
100
97
|
notifications['skiplock::jobs'].each do |n|
|
101
|
-
op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
|
102
|
-
|
103
|
-
next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
|
98
|
+
op, id, worker_id, namespace, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
|
99
|
+
next if op == 'DELETE' || running == 'true' || namespace != Skiplock.namespace || expired_at.to_f > 0 || finished_at.to_f > 0
|
104
100
|
next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < next_schedule_at
|
105
101
|
end
|
106
|
-
if self.master && @config[:actioncable] && defined?(ActionCable)
|
107
|
-
notifications['skiplock::workers'].each do |w|
|
108
|
-
op, id, hostname, master, capacity, pid, sid, created_at, updated_at = w.split(',')
|
109
|
-
ActionCable.server.broadcast('skiplock', { worker: { op: op, id: id, hostname: hostname, master: (master == 'true'), capacity: capacity.to_i, pid: pid.to_i, sid: sid.to_i, created_at: created_at.to_f, updated_at: updated_at.to_f } })
|
110
|
-
end
|
111
|
-
end
|
112
102
|
end
|
113
103
|
if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
|
114
104
|
@connection.exec("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'").clear
|
@@ -143,4 +133,4 @@ module Skiplock
|
|
143
133
|
end
|
144
134
|
end
|
145
135
|
end
|
146
|
-
end
|
136
|
+
end
|
data/lib/skiplock.rb
CHANGED
@@ -11,7 +11,7 @@ require 'skiplock/worker'
|
|
11
11
|
require 'skiplock/version'
|
12
12
|
|
13
13
|
module Skiplock
|
14
|
-
DEFAULT_CONFIG = { 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'logfile' => 'skiplock.log', 'loglevel' => 'info', '
|
14
|
+
DEFAULT_CONFIG = { 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'logfile' => 'skiplock.log', 'loglevel' => 'info', 'namespace' => nil, 'notification' => 'custom', 'extensions' => false, 'purge_completion' => true, 'queues' => { 'default' => 100, 'mailers' => 999 }, 'workers' => 0 }.freeze
|
15
15
|
|
16
16
|
def self.logger=(l)
|
17
17
|
@logger = l
|
@@ -21,6 +21,14 @@ module Skiplock
|
|
21
21
|
@logger
|
22
22
|
end
|
23
23
|
|
24
|
+
def self.namespace=(n)
|
25
|
+
@namespace = n
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.namespace
|
29
|
+
@namespace || ''
|
30
|
+
end
|
31
|
+
|
24
32
|
def self.on_error(&block)
|
25
33
|
@on_errors ||= []
|
26
34
|
@on_errors << block
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: skiplock
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tin Vo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-09-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activejob
|
@@ -109,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
requirements: []
|
112
|
-
rubygems_version: 3.
|
112
|
+
rubygems_version: 3.3.12
|
113
113
|
signing_key:
|
114
114
|
specification_version: 4
|
115
115
|
summary: ActiveJob Queue Adapter for PostgreSQL
|