skiplock 1.0.4 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/generators/skiplock/templates/migration.rb.erb +37 -15
- data/lib/skiplock.rb +4 -0
- data/lib/skiplock/dispatcher.rb +3 -2
- data/lib/skiplock/job.rb +4 -4
- data/lib/skiplock/manager.rb +5 -0
- data/lib/skiplock/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 572dcd7b3a55c259ed181c8d893b38b0d27e682d7647347fd020e1f4cc3b7cd8
|
4
|
+
data.tar.gz: 227e1f528be6ee5c86eb79173f0922fbcf5e436caab3158eafa5c40ed7e658c3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3ddaba15001823730cf79cd644398ec14db0997d16413e694b4578ac599099e3ff39f754b18b103f6ed0005803e718b050ac2b9b45bb3b095977fef1f1fe46ef
|
7
|
+
data.tar.gz: 5ff4be776646109fad4ffe1476ee982f8fb47ed111e13cb836bb7c07988c9c4a8753ae43cfbd05f8694a2759264a59daedf094cf645325bada99dfa930d5d333
|
@@ -4,10 +4,10 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
4
4
|
create_table 'skiplock.jobs', id: :uuid do |t|
|
5
5
|
t.uuid :worker_id, index: true
|
6
6
|
t.string :job_class, null: false
|
7
|
-
t.string :
|
8
|
-
t.string :queue_name
|
7
|
+
t.string :queue_name, index: true
|
9
8
|
t.string :locale
|
10
9
|
t.string :timezone
|
10
|
+
t.string :cron
|
11
11
|
t.integer :priority
|
12
12
|
t.integer :executions
|
13
13
|
t.jsonb :exception_executions
|
@@ -18,26 +18,48 @@ class CreateSkiplockSchema < ActiveRecord::Migration<%= "[#{ActiveRecord::VERSIO
|
|
18
18
|
t.timestamp :scheduled_at
|
19
19
|
t.timestamps null: false, default: -> { 'now()' }
|
20
20
|
end
|
21
|
-
create_table '
|
21
|
+
create_table 'skiplock.workers', id: :uuid do |t|
|
22
22
|
t.integer :pid, null: false, index: true
|
23
23
|
t.integer :ppid, index: true
|
24
|
-
t.integer :
|
24
|
+
t.integer :capacity, null: false
|
25
25
|
t.string :hostname, null: false, index: true
|
26
26
|
t.jsonb :data
|
27
27
|
t.timestamps null: false, index: true, default: -> { 'now()' }
|
28
28
|
end
|
29
|
-
execute
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
29
|
+
execute <<~ENDFUNC
|
30
|
+
CREATE OR REPLACE FUNCTION skiplock.notify_jobs() RETURNS TRIGGER AS $$
|
31
|
+
DECLARE
|
32
|
+
record RECORD;
|
33
|
+
BEGIN
|
34
|
+
IF (TG_OP = 'DELETE') THEN
|
35
|
+
record = OLD;
|
36
|
+
ELSE
|
37
|
+
record = NEW;
|
38
|
+
END IF;
|
39
|
+
PERFORM pg_notify('skiplock::jobs', CONCAT(TG_OP,',',record.id::TEXT,',',record.worker_id::TEXT,',',record.queue_name,',',record.running::TEXT,',',CAST(EXTRACT(EPOCH FROM record.expired_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.finished_at) AS FLOAT)::TEXT,',',CAST(EXTRACT(EPOCH FROM record.scheduled_at) AS FLOAT)::TEXT));
|
40
|
+
RETURN NULL;
|
41
|
+
END;
|
36
42
|
$$ LANGUAGE plpgsql
|
37
|
-
|
38
|
-
execute
|
39
|
-
|
40
|
-
|
43
|
+
ENDFUNC
|
44
|
+
execute <<~ENDFUNC
|
45
|
+
CREATE OR REPLACE FUNCTION skiplock.notify_workers() RETURNS TRIGGER AS $$
|
46
|
+
DECLARE
|
47
|
+
record RECORD;
|
48
|
+
BEGIN
|
49
|
+
IF (TG_OP = 'DELETE') THEN
|
50
|
+
record = OLD;
|
51
|
+
ELSE
|
52
|
+
record = NEW;
|
53
|
+
END IF;
|
54
|
+
PERFORM pg_notify('skiplock::workers', CONCAT(TG_OP,',',record.id::TEXT,',',record.hostname,',',record.capacity,',',record.pid,',',record.ppid));
|
55
|
+
RETURN NULL;
|
56
|
+
END;
|
57
|
+
$$ LANGUAGE plpgsql;
|
58
|
+
ENDFUNC
|
59
|
+
execute "CREATE TRIGGER notify_job AFTER INSERT OR UPDATE OR DELETE ON skiplock.jobs FOR EACH ROW EXECUTE PROCEDURE skiplock.notify_jobs()"
|
60
|
+
execute "CREATE TRIGGER notify_worker AFTER INSERT OR UPDATE OR DELETE ON skiplock.workers FOR EACH ROW EXECUTE PROCEDURE skiplock.notify_workers()"
|
61
|
+
execute "CREATE INDEX jobs_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL"
|
62
|
+
execute "CREATE INDEX jobs_retry_index ON skiplock.jobs(scheduled_at) WHERE running = FALSE AND executions IS NOT NULL AND expired_at IS NULL AND finished_at IS NULL"
|
41
63
|
execute "CREATE INDEX jobs_cron_index ON skiplock.jobs(scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC) WHERE cron IS NOT NULL AND finished_at IS NULL"
|
42
64
|
execute "CREATE UNIQUE INDEX jobs_unique_cron_index ON skiplock.jobs (job_class) WHERE cron IS NOT NULL"
|
43
65
|
end
|
data/lib/skiplock.rb
CHANGED
data/lib/skiplock/dispatcher.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
module Skiplock
|
2
2
|
class Dispatcher
|
3
3
|
def initialize(master: true, worker_num: nil, worker_pids: [])
|
4
|
+
@queues_order_query = Skiplock::Settings['queues'].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ')
|
4
5
|
@executor = Concurrent::ThreadPoolExecutor.new(min_threads: Settings['min_threads'], max_threads: Settings['max_threads'], max_queue: Settings['max_threads'], idletime: 60, auto_terminate: true, fallback_policy: :discard)
|
5
6
|
@master = master
|
6
7
|
if @master
|
@@ -62,7 +63,7 @@ module Skiplock
|
|
62
63
|
if scheduled_at.to_f <= Time.now.to_f
|
63
64
|
@next_schedule_at = Time.now.to_f
|
64
65
|
elsif scheduled_at.to_f < @next_schedule_at
|
65
|
-
@next_schedule_at =
|
66
|
+
@next_schedule_at = scheduled_at.to_f
|
66
67
|
end
|
67
68
|
end
|
68
69
|
end
|
@@ -94,7 +95,7 @@ module Skiplock
|
|
94
95
|
|
95
96
|
def do_work
|
96
97
|
while @running
|
97
|
-
result = Job.dispatch(worker_id: @worker.id)
|
98
|
+
result = Job.dispatch(queues_order_query: @queues_order_query, worker_id: @worker.id)
|
98
99
|
next if result.is_a?(Hash)
|
99
100
|
@next_schedule_at = result if result.is_a?(Float)
|
100
101
|
break
|
data/lib/skiplock/job.rb
CHANGED
@@ -2,10 +2,10 @@ module Skiplock
|
|
2
2
|
class Job < ActiveRecord::Base
|
3
3
|
self.table_name = 'skiplock.jobs'
|
4
4
|
|
5
|
-
# Return:
|
6
|
-
def self.dispatch(worker_id: nil)
|
5
|
+
# Return: Skiplock::Job if it was executed; otherwise returns the next Job's schedule time in FLOAT
|
6
|
+
def self.dispatch(queues_order_query: nil, worker_id: nil)
|
7
7
|
self.connection.exec_query('BEGIN')
|
8
|
-
job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
8
|
+
job = self.find_by_sql("SELECT id, scheduled_at FROM #{self.table_name} WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{queues_order_query ? 'CASE ' + queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
9
9
|
if job.nil? || job.scheduled_at.to_f > Time.now.to_f
|
10
10
|
self.connection.exec_query('END')
|
11
11
|
return (job ? job.scheduled_at.to_f : Float::INFINITY)
|
@@ -54,7 +54,7 @@ module Skiplock
|
|
54
54
|
end
|
55
55
|
job
|
56
56
|
ensure
|
57
|
-
Thread.current[:
|
57
|
+
Thread.current[:skiplock_dispatch_job] = nil
|
58
58
|
end
|
59
59
|
|
60
60
|
def self.enqueue_at(activejob, timestamp)
|
data/lib/skiplock/manager.rb
CHANGED
@@ -31,7 +31,11 @@ module Skiplock
|
|
31
31
|
Settings['max_threads'] = 20 if Settings['max_threads'] > 20
|
32
32
|
Settings['min_threads'] = 0 if Settings['min_threads'] < 0
|
33
33
|
Settings['workers'] = 0 if Settings['workers'] < 0
|
34
|
+
Settings['queues'].values.each { |v| raise 'Queue value must be an integer' unless v.is_a?(Integer) }
|
34
35
|
Settings.freeze
|
36
|
+
rescue Exception => e
|
37
|
+
STDERR.puts "Invalid configuration 'config/skiplock.yml': #{e.message}"
|
38
|
+
exit
|
35
39
|
end
|
36
40
|
|
37
41
|
def self.standalone
|
@@ -46,6 +50,7 @@ module Skiplock
|
|
46
50
|
puts " Max threads: #{Settings['max_threads']}"
|
47
51
|
puts " Environment: #{Rails.env}"
|
48
52
|
puts " Logging: #{Settings['logging']}"
|
53
|
+
puts " Queues: #{Settings['queues'].map {|k,v| k + '(' + v.to_s + ')'}.join(', ')}"
|
49
54
|
puts " PID: #{Process.pid}"
|
50
55
|
puts "-"*(title.length)
|
51
56
|
if Settings['logging']
|
data/lib/skiplock/version.rb
CHANGED