skiplock 1.0.24 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -4
- data/bin/skiplock +1 -1
- data/lib/generators/skiplock/templates/migration.rb.erb +25 -19
- data/lib/skiplock/job.rb +5 -4
- data/lib/skiplock/manager.rb +6 -6
- data/lib/skiplock/version.rb +1 -1
- data/lib/skiplock/worker.rb +9 -19
- data/lib/skiplock.rb +9 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c119463794e6ac523fb385c7925181bf069d6cf37996fe4a2dacf101e6faa6c8
|
4
|
+
data.tar.gz: 0aee11414e05623185781b1a5779a9b7393474fde176ec6d9ee42151cd5453a4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ba80ed8e0bd4bf185ffa2e65f88d7f6a1f32e1124e75e82dac50ba1fecc12167687ffab2e5f8f9e7148ba24c000ae6114c04c67e175793c99a45db17dde8b0c
|
7
|
+
data.tar.gz: 0adb1907dd1e0c8338c298d32f6fe3e576f755a38962beed186c56ca4375882cbbf20646a7efd2cf37145588d7acf6dd4855156f418de9732254ec5b86704e9a
|
data/README.md
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
It only uses the `LISTEN/NOTIFY/SKIP LOCKED` features provided natively on PostgreSQL 9.5+ to efficiently and reliably dispatch jobs to worker processes and threads ensuring that each job can be completed successfully **only once**. No other polling or timer is needed.
|
6
6
|
|
7
|
-
The library is quite small compared to other PostgreSQL job queues (eg. *delay_job*, *queue_classic*, *que*, *good_job*) with less than
|
7
|
+
The library is quite small compared to other PostgreSQL job queues (eg. *delay_job*, *queue_classic*, *que*, *good_job*) with less than 600 lines of codes; and it still provides similar set of features and more...
|
8
8
|
|
9
9
|
#### Compatibility:
|
10
10
|
|
@@ -56,8 +56,8 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
|
|
56
56
|
max_retries: 20
|
57
57
|
logfile: skiplock.log
|
58
58
|
loglevel: info
|
59
|
+
namespace:
|
59
60
|
notification: custom
|
60
|
-
actioncable: false
|
61
61
|
extensions: false
|
62
62
|
purge_completion: true
|
63
63
|
queues:
|
@@ -72,8 +72,8 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
|
|
72
72
|
- **max_retries** (*integer*): sets the maximum attempt a job will be retrying before it is marked expired. See `Retry system` for more details
|
73
73
|
- **logfile** (*string*): filename for skiplock logs; empty logfile will disable logging
|
74
74
|
- **loglevel** (*string*): sets logging level (`debug, info, warn, error, fatal, unknown`)
|
75
|
+
- **namespace** (*string*): sets namespace for jobs (workers will only process jobs of specified namespace)
|
75
76
|
- **notification** (*string*): sets the library to be used for notifying errors and exceptions (`auto, airbrake, bugsnag, exception_notification, custom`); using `auto` will detect library if available. See `Notification system` for more details
|
76
|
-
- **actioncable** (*boolean*): enable or disable usage of ActionCable notification
|
77
77
|
- **extensions** (*multi*): enable or disable the class method extension. See `ClassMethod extension` for more details
|
78
78
|
- **purge_completion** (*boolean*): when set to **true** will delete jobs after they were completed successfully; if set to **false** then the completed jobs should be purged periodically to maximize performance (eg. clean up old jobs after 3 months); queued jobs can manually override using `purge` option
|
79
79
|
- **queues** (*hash*): defines the set of queues with priorities; lower priority takes precedence
|
@@ -87,10 +87,10 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
|
|
87
87
|
```
|
88
88
|
$ bundle exec skiplock -h
|
89
89
|
Usage: skiplock [options]
|
90
|
-
-a, --actioncable YESNO Actioncable notification
|
91
90
|
-e, --environment STRING Rails environment
|
92
91
|
-l, --logfile STRING Log filename
|
93
92
|
-L, --loglevel STRING Log level (debug, info, warn, error, fatal, unknown)
|
93
|
+
-n, --namespace STRING Job namespace
|
94
94
|
-s, --graceful-shutdown NUM Number of seconds to wait for graceful shutdown
|
95
95
|
-r, --max-retries NUM Number of maxixum retries
|
96
96
|
-t, --max-threads NUM Number of maximum threads
|
data/bin/skiplock
CHANGED
@@ -5,10 +5,10 @@ options = {}
|
|
5
5
|
begin
|
6
6
|
op = OptionParser.new do |opts|
|
7
7
|
opts.banner = "Usage: #{File.basename($0)} [options]"
|
8
|
-
opts.on('-a', '--actioncable YESNO', TrueClass, 'Actioncable notification')
|
9
8
|
opts.on('-e', '--environment STRING', String, 'Rails environment')
|
10
9
|
opts.on('-l', '--logfile STRING', String, 'Log filename')
|
11
10
|
opts.on('-L', '--loglevel STRING', String, 'Log level (debug, info, warn, error, fatal, unknown)')
|
11
|
+
opts.on('-n', '--namespace STRING', String, 'Job namespace')
|
12
12
|
opts.on('-s', '--graceful-shutdown NUM', Integer, 'Number of seconds to wait for graceful shutdown')
|
13
13
|
opts.on('-r', '--max-retries NUM', Integer, 'Number of maxixum retries')
|
14
14
|
opts.on('-t', '--max-threads NUM', Integer, 'Number of maximum threads')
|
@@ -7,11 +7,14 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
7
7
|
t.integer :expiries, null: false, default: 0
|
8
8
|
t.integer :failures, null: false, default: 0
|
9
9
|
t.integer :retries, null: false, default: 0
|
10
|
-
t.
|
10
|
+
t.string :namespace, null: false, default: '', index: true
|
11
|
+
t.date :day, null: false
|
12
|
+
t.index [ :namespace, :day ], unique: true
|
11
13
|
end
|
12
14
|
create_table 'skiplock.jobs', id: :uuid do |t|
|
13
15
|
t.uuid :worker_id, index: true
|
14
16
|
t.string :job_class, null: false
|
17
|
+
t.string :namespace, null: false, default: '', index: true
|
15
18
|
t.string :queue_name, index: true
|
16
19
|
t.string :locale
|
17
20
|
t.string :timezone
|
@@ -31,6 +34,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
31
34
|
t.integer :sid, null: false
|
32
35
|
t.integer :capacity, null: false
|
33
36
|
t.string :hostname, null: false, index: true
|
37
|
+
t.string :namespace, null: false, default: '', index: true
|
34
38
|
t.boolean :master, null: false, default: false, index: true
|
35
39
|
t.jsonb :data
|
36
40
|
t.timestamps null: false, index: true, default: -> { 'now()' }
|
@@ -41,27 +45,29 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
41
45
|
record RECORD;
|
42
46
|
BEGIN
|
43
47
|
record = NEW;
|
44
|
-
IF
|
48
|
+
IF TG_OP = 'DELETE' THEN
|
45
49
|
record = OLD;
|
46
|
-
IF
|
47
|
-
INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
50
|
+
IF record.running IS TRUE THEN
|
51
|
+
INSERT INTO skiplock.counters (namespace,day,completions) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
48
52
|
END IF;
|
49
|
-
ELSIF
|
50
|
-
IF
|
51
|
-
IF
|
52
|
-
INSERT INTO skiplock.counters (day,retries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET retries = skiplock.counters.retries + 1;
|
53
|
+
ELSIF TG_OP = 'UPDATE' THEN
|
54
|
+
IF OLD.running IS FALSE AND record.running IS TRUE THEN
|
55
|
+
IF record.executions > 0 THEN
|
56
|
+
INSERT INTO skiplock.counters (namespace,day,retries) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET retries = skiplock.counters.retries + 1;
|
53
57
|
ELSE
|
54
|
-
INSERT INTO skiplock.counters (day,dispatches) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET dispatches = skiplock.counters.dispatches + 1;
|
58
|
+
INSERT INTO skiplock.counters (namespace,day,dispatches) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET dispatches = skiplock.counters.dispatches + 1;
|
59
|
+
END IF;
|
60
|
+
ELSIF OLD.finished_at IS NULL AND record.finished_at IS NOT NULL THEN
|
61
|
+
INSERT INTO skiplock.counters (namespace,day,completions) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
62
|
+
ELSIF OLD.running IS TRUE AND record.running IS FALSE THEN
|
63
|
+
IF record.expired_at IS NOT NULL THEN
|
64
|
+
INSERT INTO skiplock.counters (namespace,day,expiries) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET expiries = skiplock.counters.expiries + 1;
|
65
|
+
ELSE
|
66
|
+
INSERT INTO skiplock.counters (namespace,day,failures) VALUES (record.namespace,NOW(),1) ON CONFLICT (namespace,day) DO UPDATE SET failures = skiplock.counters.failures + 1;
|
55
67
|
END IF;
|
56
|
-
ELSIF (OLD.finished_at IS NULL AND record.finished_at IS NOT NULL) THEN
|
57
|
-
INSERT INTO skiplock.counters (day,completions) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET completions = skiplock.counters.completions + 1;
|
58
|
-
ELSIF (OLD.running = TRUE AND record.running = FALSE AND record.expired_at IS NOT NULL) THEN
|
59
|
-
INSERT INTO skiplock.counters (day,expiries) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET expiries = skiplock.counters.expiries + 1;
|
60
|
-
ELSIF (OLD.running = TRUE AND record.running = FALSE AND record.expired_at IS NULL AND record.finished_at IS NULL) THEN
|
61
|
-
INSERT INTO skiplock.counters (day,failures) VALUES (NOW(),1) ON CONFLICT (day) DO UPDATE SET failures = skiplock.counters.failures + 1;
|
62
68
|
END IF;
|
63
69
|
END IF;
|
64
|
-
PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.job_class,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM CASE WHEN record.scheduled_at IS NULL THEN record.updated_at ELSE record.scheduled_at END) AS FLOAT)::TEXT));
|
70
|
+
PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.namespace,',',record.job_class,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM CASE WHEN record.scheduled_at IS NULL THEN record.updated_at ELSE record.scheduled_at END) AS FLOAT)::TEXT));
|
65
71
|
RETURN NULL;
|
66
72
|
END;
|
67
73
|
$$ LANGUAGE plpgsql
|
@@ -71,12 +77,12 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
71
77
|
DECLARE
|
72
78
|
record RECORD;
|
73
79
|
BEGIN
|
74
|
-
IF
|
80
|
+
IF TG_OP = 'DELETE' THEN
|
75
81
|
record = OLD;
|
76
82
|
ELSE
|
77
83
|
record = NEW;
|
78
84
|
END IF;
|
79
|
-
PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.master::TEXT,',',record.capacity,',',record.pid,',',record.sid,',',CAST(EXTRACT(EPOCH FROM record.created_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.updated_at) AS FLOAT)::TEXT));
|
85
|
+
PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.namespace,',',record.hostname,',',record.master::TEXT,',',record.capacity,',',record.pid,',',record.sid,',',CAST(EXTRACT(EPOCH FROM record.created_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.updated_at) AS FLOAT)::TEXT));
|
80
86
|
RETURN NULL;
|
81
87
|
END;
|
82
88
|
$$ LANGUAGE plpgsql;
|
@@ -87,7 +93,7 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
87
93
|
execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = FALSE AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
|
88
94
|
execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
|
89
95
|
execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
|
90
|
-
execute "CREATE UNIQUE INDEX workers_unique_master_index ON skiplock.workers(hostname) WHERE master = TRUE"
|
96
|
+
execute "CREATE UNIQUE INDEX workers_unique_master_index ON skiplock.workers(hostname,namespace) WHERE master = TRUE"
|
91
97
|
end
|
92
98
|
|
93
99
|
def down
|
data/lib/skiplock/job.rb
CHANGED
@@ -3,14 +3,15 @@ module Skiplock
|
|
3
3
|
self.implicit_order_column = 'updated_at'
|
4
4
|
attribute :activejob_error
|
5
5
|
attribute :exception
|
6
|
-
attribute :max_retries
|
7
|
-
attribute :purge
|
6
|
+
attribute :max_retries
|
7
|
+
attribute :purge
|
8
8
|
belongs_to :worker, inverse_of: :jobs, required: false
|
9
9
|
|
10
10
|
def self.dispatch(purge_completion: true, max_retries: 20)
|
11
|
+
namespace_query = Skiplock.namespace.nil? ? "namespace IS NULL" : "namespace = '#{Skiplock.namespace}'"
|
11
12
|
job = nil
|
12
13
|
self.connection.transaction do
|
13
|
-
job = self.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
14
|
+
job = self.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL AND #{namespace_query} ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
14
15
|
return if job.nil? || job.scheduled_at.to_f > Time.now.to_f
|
15
16
|
job = self.find_by_sql("UPDATE skiplock.jobs SET running = TRUE, updated_at = NOW() WHERE id = '#{job.id}' RETURNING *").first
|
16
17
|
end
|
@@ -33,7 +34,7 @@ module Skiplock
|
|
33
34
|
Thread.current[:skiplock_job]
|
34
35
|
else
|
35
36
|
serialize = activejob.serialize
|
36
|
-
self.create!(serialize.slice(*self.column_names).merge('id' => serialize['job_id'], 'data' => { 'arguments' => serialize['arguments'], 'options' => options }, 'scheduled_at' => timestamp))
|
37
|
+
self.create!(serialize.slice(*self.column_names).merge('id' => serialize['job_id'], 'data' => { 'arguments' => serialize['arguments'], 'options' => options }, 'namespace' => Skiplock.namespace, 'scheduled_at' => timestamp))
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
data/lib/skiplock/manager.rb
CHANGED
@@ -11,6 +11,7 @@ module Skiplock
|
|
11
11
|
@config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
|
12
12
|
end
|
13
13
|
ActiveJob::Base.__send__(:include, Skiplock::Patch)
|
14
|
+
Skiplock.namespace = @config[:namespace]
|
14
15
|
(caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0) ? async : Cron.setup
|
15
16
|
end
|
16
17
|
|
@@ -18,7 +19,7 @@ module Skiplock
|
|
18
19
|
setup_logger
|
19
20
|
configure
|
20
21
|
Worker.cleanup(@hostname)
|
21
|
-
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname
|
22
|
+
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
|
22
23
|
Cron.setup if @worker.master
|
23
24
|
@worker.start(**@config)
|
24
25
|
at_exit { @worker.shutdown }
|
@@ -40,13 +41,13 @@ module Skiplock
|
|
40
41
|
Signal.trap('TERM') { @shutdown = true }
|
41
42
|
Signal.trap('HUP') { setup_logger }
|
42
43
|
Worker.cleanup(@hostname)
|
43
|
-
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname
|
44
|
+
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
|
44
45
|
ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
|
45
46
|
(@config[:workers] - 1).times do |n|
|
46
47
|
fork do
|
47
48
|
sleep(0.25*n + 1)
|
48
49
|
ActiveRecord::Base.establish_connection
|
49
|
-
worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false
|
50
|
+
worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
|
50
51
|
worker.start(worker_num: n + 1, **@config)
|
51
52
|
loop do
|
52
53
|
sleep 0.5
|
@@ -76,7 +77,6 @@ module Skiplock
|
|
76
77
|
@logger.info "-"*(title.length)
|
77
78
|
@logger.info title
|
78
79
|
@logger.info "-"*(title.length)
|
79
|
-
@logger.info "ActionCable notification: #{@config[:actioncable]}#{' (not available)' unless defined?(ActionCable)}"
|
80
80
|
@logger.info " ClassMethod extensions: #{@config[:extensions]}"
|
81
81
|
@logger.info " Purge completion: #{@config[:purge_completion]}"
|
82
82
|
@logger.info " Notification: #{@config[:notification]}"
|
@@ -84,6 +84,7 @@ module Skiplock
|
|
84
84
|
@logger.info " Min threads: #{@config[:min_threads]}"
|
85
85
|
@logger.info " Max threads: #{@config[:max_threads]}"
|
86
86
|
@logger.info " Environment: #{Rails.env}"
|
87
|
+
@logger.info " Namespace: #{@config[:namespace] || '(nil)'}"
|
87
88
|
@logger.info " Loglevel: #{@config[:loglevel]}"
|
88
89
|
@logger.info " Logfile: #{@config[:logfile] || '(disabled)'}"
|
89
90
|
@logger.info " Workers: #{@config[:workers]}"
|
@@ -135,7 +136,6 @@ module Skiplock
|
|
135
136
|
else
|
136
137
|
@config[:notification] = 'custom'
|
137
138
|
end
|
138
|
-
@logger.error 'ActionCable is not found!' if @config[:actioncable] && !defined?(ActionCable)
|
139
139
|
Skiplock.on_errors.freeze
|
140
140
|
end
|
141
141
|
|
@@ -159,4 +159,4 @@ module Skiplock
|
|
159
159
|
Skiplock.on_errors.each { |p| p.call(ex) }
|
160
160
|
end
|
161
161
|
end
|
162
|
-
end
|
162
|
+
end
|
data/lib/skiplock/version.rb
CHANGED
data/lib/skiplock/worker.rb
CHANGED
@@ -5,27 +5,23 @@ module Skiplock
|
|
5
5
|
|
6
6
|
def self.cleanup(hostname = nil)
|
7
7
|
delete_ids = []
|
8
|
-
self.where(hostname: hostname || Socket.gethostname).each do |worker|
|
8
|
+
self.where(namespace: Skiplock.namespace, hostname: hostname || Socket.gethostname).each do |worker|
|
9
9
|
sid = Process.getsid(worker.pid) rescue nil
|
10
10
|
delete_ids << worker.id if worker.sid != sid || worker.updated_at < 10.minutes.ago
|
11
11
|
end
|
12
12
|
self.where(id: delete_ids).delete_all if delete_ids.count > 0
|
13
13
|
end
|
14
14
|
|
15
|
-
def self.generate(capacity:, hostname:, master: true
|
16
|
-
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity)
|
15
|
+
def self.generate(capacity:, hostname:, master: true)
|
16
|
+
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: master, hostname: hostname, capacity: capacity, namespace: Skiplock.namespace)
|
17
17
|
rescue
|
18
|
-
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity)
|
19
|
-
ensure
|
20
|
-
ActionCable.server.broadcast('skiplock', { worker: { op: 'CREATE', id: worker.id, hostname: worker.hostname, master: worker.master, capacity: worker.capacity, pid: worker.pid, sid: worker.sid, created_at: worker.created_at.to_f, updated_at: worker.updated_at.to_f } }) if actioncable && defined?(ActionCable)
|
18
|
+
worker = self.create!(pid: Process.pid, sid: Process.getsid(), master: false, hostname: hostname, capacity: capacity, namespace: Skiplock.namespace)
|
21
19
|
end
|
22
20
|
|
23
21
|
def shutdown
|
24
22
|
@running = false
|
25
23
|
@executor.shutdown
|
26
24
|
@executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
|
27
|
-
ActionCable.server.broadcast('skiplock', { worker: { op: 'DELETE', id: self.id, hostname: self.hostname, master: self.master, capacity: self.capacity, pid: self.pid, sid: self.sid, created_at: self.created_at.to_f, updated_at: self.updated_at.to_f } }) if @config[:actioncable] && defined?(ActionCable)
|
28
|
-
self.delete
|
29
25
|
Skiplock.logger.info "[Skiplock] Shutdown of #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} (PID: #{self.pid}) was completed."
|
30
26
|
end
|
31
27
|
|
@@ -33,6 +29,7 @@ module Skiplock
|
|
33
29
|
@num = worker_num
|
34
30
|
@config = config
|
35
31
|
@pg_config = ActiveRecord::Base.connection.raw_connection.conninfo_hash.compact
|
32
|
+
@namespace_query = Skiplock.namespace.nil? ? "namespace IS NULL" : "namespace = '#{Skiplock.namespace}'"
|
36
33
|
@queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
|
37
34
|
@running = true
|
38
35
|
@map = ::PG::TypeMapByOid.new
|
@@ -78,7 +75,7 @@ module Skiplock
|
|
78
75
|
if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 1 # reserves 1 slot in queue for Job.flush in case of pg_connection error
|
79
76
|
result = nil
|
80
77
|
@connection.transaction do |conn|
|
81
|
-
conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
|
78
|
+
conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL AND #{@namespace_query} ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
|
82
79
|
result = r.first
|
83
80
|
conn.exec("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *") { |r| result = r.first } if result && result['scheduled_at'].to_f <= Time.now.to_f
|
84
81
|
end
|
@@ -98,17 +95,10 @@ module Skiplock
|
|
98
95
|
notifications[payload[:relname]] << payload[:extra]
|
99
96
|
end
|
100
97
|
notifications['skiplock::jobs'].each do |n|
|
101
|
-
op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
|
102
|
-
|
103
|
-
next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
|
98
|
+
op, id, worker_id, namespace, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
|
99
|
+
next if op == 'DELETE' || running == 'true' || namespace != Skiplock.namespace || expired_at.to_f > 0 || finished_at.to_f > 0
|
104
100
|
next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < next_schedule_at
|
105
101
|
end
|
106
|
-
if self.master && @config[:actioncable] && defined?(ActionCable)
|
107
|
-
notifications['skiplock::workers'].each do |w|
|
108
|
-
op, id, hostname, master, capacity, pid, sid, created_at, updated_at = w.split(',')
|
109
|
-
ActionCable.server.broadcast('skiplock', { worker: { op: op, id: id, hostname: hostname, master: (master == 'true'), capacity: capacity.to_i, pid: pid.to_i, sid: sid.to_i, created_at: created_at.to_f, updated_at: updated_at.to_f } })
|
110
|
-
end
|
111
|
-
end
|
112
102
|
end
|
113
103
|
if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
|
114
104
|
@connection.exec("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'").clear
|
@@ -143,4 +133,4 @@ module Skiplock
|
|
143
133
|
end
|
144
134
|
end
|
145
135
|
end
|
146
|
-
end
|
136
|
+
end
|
data/lib/skiplock.rb
CHANGED
@@ -11,7 +11,7 @@ require 'skiplock/worker'
|
|
11
11
|
require 'skiplock/version'
|
12
12
|
|
13
13
|
module Skiplock
|
14
|
-
DEFAULT_CONFIG = { 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'logfile' => 'skiplock.log', 'loglevel' => 'info', '
|
14
|
+
DEFAULT_CONFIG = { 'graceful_shutdown' => 15, 'min_threads' => 1, 'max_threads' => 10, 'max_retries' => 20, 'logfile' => 'skiplock.log', 'loglevel' => 'info', 'namespace' => nil, 'notification' => 'custom', 'extensions' => false, 'purge_completion' => true, 'queues' => { 'default' => 100, 'mailers' => 999 }, 'workers' => 0 }.freeze
|
15
15
|
|
16
16
|
def self.logger=(l)
|
17
17
|
@logger = l
|
@@ -21,6 +21,14 @@ module Skiplock
|
|
21
21
|
@logger
|
22
22
|
end
|
23
23
|
|
24
|
+
def self.namespace=(n)
|
25
|
+
@namespace = n
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.namespace
|
29
|
+
@namespace || ''
|
30
|
+
end
|
31
|
+
|
24
32
|
def self.on_error(&block)
|
25
33
|
@on_errors ||= []
|
26
34
|
@on_errors << block
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: skiplock
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tin Vo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-09-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activejob
|
@@ -109,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
requirements: []
|
112
|
-
rubygems_version: 3.
|
112
|
+
rubygems_version: 3.3.12
|
113
113
|
signing_key:
|
114
114
|
specification_version: 4
|
115
115
|
summary: ActiveJob Queue Adapter for PostgreSQL
|