skiplock 1.0.13 → 1.0.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/skiplock/job.rb +32 -18
- data/lib/skiplock/manager.rb +6 -15
- data/lib/skiplock/version.rb +1 -1
- data/lib/skiplock/worker.rb +87 -83
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 55c98833e2792a216b6389d399e992aae9660f23ea076fcf852fb164cbc32d55
|
4
|
+
data.tar.gz: 43c8c2f90ee55ec423242042d1ab2986c0cb9f70fcc04db56aaf2c566bbb0382
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 873492b815e6b5fc653630fed3bc64c5a363339551bbad304ecbad9453ea3738c24f3f0b874f4d6fe47cd4cc0ea5d46fb61b4c0dc322814d08dbc11002c24443
|
7
|
+
data.tar.gz: e84f7081e07ba0aff19834251de09e08df573d45ff159299fe08baf621b76845cd2fdf0461229da77b07cf707f151e7d8fc00d97b19a9bf30f1129376c2dd053
|
data/lib/skiplock/job.rb
CHANGED
@@ -2,6 +2,16 @@ module Skiplock
|
|
2
2
|
class Job < ActiveRecord::Base
|
3
3
|
self.implicit_order_column = 'created_at'
|
4
4
|
|
5
|
+
def self.dispatch(worker_id: nil, purge_completion: true, max_retries: 20)
|
6
|
+
job = nil
|
7
|
+
self.connection.transaction do
|
8
|
+
job = self.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST, priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
9
|
+
return if job.nil? || job.scheduled_at.to_f > Time.now.to_f
|
10
|
+
job = self.find_by_sql("UPDATE skiplock.jobs SET running = TRUE, worker_id = #{self.connection.quote(worker_id)}, updated_at = NOW() WHERE id = '#{job.id}' RETURNING *").first
|
11
|
+
end
|
12
|
+
self.dispatch(worker_id: worker_id, purge_completion: purge_completion, max_retries: max_retries) if job.execute(purge_completion: purge_completion, max_retries: max_retries)
|
13
|
+
end
|
14
|
+
|
5
15
|
def self.enqueue(activejob)
|
6
16
|
self.enqueue_at(activejob, nil)
|
7
17
|
end
|
@@ -50,7 +60,7 @@ module Skiplock
|
|
50
60
|
self.scheduled_at = Time.at(next_cron_at)
|
51
61
|
self.save!
|
52
62
|
else
|
53
|
-
Skiplock.logger.error
|
63
|
+
Skiplock.logger.error("[Skiplock] ERROR: Invalid CRON '#{self.cron}' for Job #{self.job_class}") if Skiplock.logger
|
54
64
|
self.delete
|
55
65
|
end
|
56
66
|
elsif purge_completion
|
@@ -62,14 +72,16 @@ module Skiplock
|
|
62
72
|
end
|
63
73
|
self
|
64
74
|
rescue Exception => e
|
65
|
-
Skiplock.logger
|
66
|
-
|
67
|
-
|
75
|
+
if Skiplock.logger
|
76
|
+
Skiplock.logger.error(e.to_s)
|
77
|
+
Skiplock.logger.error(e.backtrace.join("\n"))
|
78
|
+
File.write("tmp/skiplock/#{self.id}", yaml)
|
79
|
+
end
|
68
80
|
nil
|
69
81
|
end
|
70
82
|
|
71
83
|
def execute(purge_completion: true, max_retries: 20)
|
72
|
-
Skiplock.logger.info
|
84
|
+
Skiplock.logger.info("[Skiplock] Performing #{self.job_class} (#{self.id}) from queue '#{self.queue_name || 'default'}'...") if Skiplock.logger
|
73
85
|
self.data ||= {}
|
74
86
|
self.exception_executions ||= {}
|
75
87
|
job_data = self.attributes.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions', 'exception_executions').merge('job_id' => self.id, 'enqueued_at' => self.updated_at, 'arguments' => (self.data['arguments'] || []))
|
@@ -81,20 +93,22 @@ module Skiplock
|
|
81
93
|
activejob.perform_now
|
82
94
|
rescue Exception => ex
|
83
95
|
end
|
84
|
-
if
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
+
if Skiplock.logger
|
97
|
+
if ex || self.exception_executions.key?('activejob_retry')
|
98
|
+
Skiplock.logger.error("[Skiplock] Job #{self.job_class} (#{self.id}) was interrupted by an exception#{ ' (rescued and retried by ActiveJob)' if self.exception_executions.key?('activejob_retry') }")
|
99
|
+
if ex
|
100
|
+
Skiplock.logger.error(ex.to_s)
|
101
|
+
Skiplock.logger.error(ex.backtrace.join("\n"))
|
102
|
+
end
|
103
|
+
else
|
104
|
+
end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
105
|
+
job_name = self.job_class
|
106
|
+
if self.job_class == 'Skiplock::Extension::ProxyJob'
|
107
|
+
target, method_name = ::YAML.load(self.data['arguments'].first)
|
108
|
+
job_name = "'#{target.name}.#{method_name}'"
|
109
|
+
end
|
110
|
+
Skiplock.logger.info "[Skiplock] Performed #{job_name} (#{self.id}) from queue '#{self.queue_name || 'default'}' in #{end_time - start_time} seconds"
|
96
111
|
end
|
97
|
-
Skiplock.logger.info "[Skiplock] Performed #{job_name} (#{self.id}) from queue '#{self.queue_name || 'default'}' in #{end_time - start_time} seconds"
|
98
112
|
end
|
99
113
|
self.dispose(ex, purge_completion: purge_completion, max_retries: max_retries)
|
100
114
|
end
|
data/lib/skiplock/manager.rb
CHANGED
@@ -9,18 +9,14 @@ module Skiplock
|
|
9
9
|
@config[:hostname] = `hostname -f`.strip
|
10
10
|
configure
|
11
11
|
Module.__send__(:include, Skiplock::Extension) if @config[:extensions] == true
|
12
|
-
if (caller.any?{ |l| l =~ %r{/rack/} } &&
|
12
|
+
if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
|
13
13
|
cleanup_workers
|
14
14
|
@worker = create_worker
|
15
|
-
@
|
16
|
-
at_exit
|
17
|
-
@worker.shutdown
|
18
|
-
@thread.join(@config[:graceful_shutdown])
|
19
|
-
@worker.delete
|
20
|
-
end
|
15
|
+
@worker.start(**@config)
|
16
|
+
at_exit { @worker.shutdown }
|
21
17
|
end
|
22
18
|
rescue Exception => ex
|
23
|
-
@logger.error(ex.
|
19
|
+
@logger.error(ex.to_s)
|
24
20
|
@logger.error(ex.backtrace.join("\n"))
|
25
21
|
end
|
26
22
|
|
@@ -40,18 +36,15 @@ module Skiplock
|
|
40
36
|
fork do
|
41
37
|
sleep 1
|
42
38
|
worker = create_worker(master: false)
|
43
|
-
|
39
|
+
worker.start(worker_num: n + 1, **@config)
|
44
40
|
loop do
|
45
41
|
sleep 0.5
|
46
42
|
break if @shutdown || Process.ppid != @parent_id
|
47
43
|
end
|
48
44
|
worker.shutdown
|
49
|
-
thread.join(@config[:graceful_shutdown])
|
50
|
-
worker.delete
|
51
|
-
exit
|
52
45
|
end
|
53
46
|
end
|
54
|
-
@
|
47
|
+
@worker.start(**@config)
|
55
48
|
loop do
|
56
49
|
sleep 0.5
|
57
50
|
break if @shutdown
|
@@ -59,8 +52,6 @@ module Skiplock
|
|
59
52
|
@logger.info "[Skiplock] Terminating signal... Waiting for jobs to finish (up to #{@config[:graceful_shutdown]} seconds)..." if @config[:graceful_shutdown]
|
60
53
|
Process.waitall
|
61
54
|
@worker.shutdown
|
62
|
-
@thread.join(@config[:graceful_shutdown])
|
63
|
-
@worker.delete
|
64
55
|
@logger.info "[Skiplock] Shutdown completed."
|
65
56
|
end
|
66
57
|
|
data/lib/skiplock/version.rb
CHANGED
data/lib/skiplock/worker.rb
CHANGED
@@ -2,116 +2,120 @@ module Skiplock
|
|
2
2
|
class Worker < ActiveRecord::Base
|
3
3
|
self.implicit_order_column = 'created_at'
|
4
4
|
|
5
|
-
def
|
5
|
+
def start(worker_num: 0, **config)
|
6
6
|
@config = config
|
7
|
-
@worker_num = worker_num
|
8
7
|
@queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
|
9
8
|
@next_schedule_at = Time.now.to_f
|
10
|
-
@executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads], max_threads: @config[:max_threads], max_queue: @config[:max_threads], idletime: 60, auto_terminate: true, fallback_policy: :discard)
|
9
|
+
@executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads], idletime: 60, auto_terminate: true, fallback_policy: :discard)
|
11
10
|
@running = true
|
12
|
-
Process.setproctitle("skiplock-#{self.master ? 'master[0]' : 'worker[' +
|
13
|
-
|
14
|
-
|
15
|
-
@connection.exec_query('LISTEN "skiplock::jobs"')
|
16
|
-
if self.master
|
17
|
-
Dir.mkdir('tmp/skiplock') unless Dir.exist?('tmp/skiplock')
|
18
|
-
check_sync_errors
|
19
|
-
Cron.setup
|
20
|
-
end
|
21
|
-
error = false
|
22
|
-
timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
23
|
-
while @running
|
24
|
-
Rails.application.reloader.wrap do
|
25
|
-
begin
|
26
|
-
if error
|
27
|
-
unless @connection.active?
|
28
|
-
@connection.reconnect!
|
29
|
-
sleep(0.5)
|
30
|
-
@connection.exec_query('LISTEN "skiplock::jobs"')
|
31
|
-
@next_schedule_at = Time.now.to_f
|
32
|
-
end
|
33
|
-
check_sync_errors if self.master
|
34
|
-
error = false
|
35
|
-
end
|
36
|
-
job_notifications = []
|
37
|
-
@connection.raw_connection.wait_for_notify(0.1) do |channel, pid, payload|
|
38
|
-
job_notifications << payload if payload
|
39
|
-
loop do
|
40
|
-
payload = @connection.raw_connection.notifies
|
41
|
-
break unless @running && payload
|
42
|
-
job_notifications << payload[:extra]
|
43
|
-
end
|
44
|
-
job_notifications.each do |n|
|
45
|
-
op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
|
46
|
-
next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
|
47
|
-
if scheduled_at.to_f <= Time.now.to_f
|
48
|
-
@next_schedule_at = Time.now.to_f
|
49
|
-
elsif scheduled_at.to_f < @next_schedule_at
|
50
|
-
@next_schedule_at = scheduled_at.to_f
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
if Time.now.to_f >= @next_schedule_at && @executor.remaining_capacity > 0
|
55
|
-
job = dispatch_job
|
56
|
-
if job.is_a?(Job)
|
57
|
-
@executor.post(job, @config[:purge_completion], @config[:max_retries]) do |job, purge_completion, max_retries|
|
58
|
-
job.execute(purge_completion: purge_completion, max_retries: max_retries)
|
59
|
-
end
|
60
|
-
else
|
61
|
-
@next_schedule_at = job
|
62
|
-
end
|
63
|
-
end
|
64
|
-
if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
|
65
|
-
self.touch
|
66
|
-
timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
67
|
-
end
|
68
|
-
rescue Exception => ex
|
69
|
-
# most likely error with database connection
|
70
|
-
Skiplock.logger.error(ex.name)
|
71
|
-
Skiplock.logger.error(ex.backtrace.join("\n"))
|
72
|
-
Skiplock.on_errors.each { |p| p.call(ex, @last_exception) }
|
73
|
-
error = true
|
74
|
-
t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
75
|
-
while @running
|
76
|
-
sleep(0.5)
|
77
|
-
break if Process.clock_gettime(Process::CLOCK_MONOTONIC) - t > 5
|
78
|
-
end
|
79
|
-
@last_exception = ex
|
80
|
-
end
|
81
|
-
sleep(0.2)
|
82
|
-
end
|
83
|
-
end
|
84
|
-
@connection.exec_query('UNLISTEN *')
|
85
|
-
@executor.shutdown
|
86
|
-
@executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
|
11
|
+
Process.setproctitle("skiplock-#{self.master ? 'master[0]' : 'worker[' + worker_num.to_s + ']'}") if @config[:standalone]
|
12
|
+
@executor.post do
|
13
|
+
Rails.application.reloader.wrap { run }
|
87
14
|
end
|
88
15
|
end
|
89
16
|
|
90
17
|
def shutdown
|
91
18
|
@running = false
|
19
|
+
@executor.shutdown
|
20
|
+
@executor.kill unless @executor.wait_for_termination(@config[:graceful_shutdown])
|
21
|
+
self.delete
|
92
22
|
end
|
93
23
|
|
94
24
|
private
|
95
25
|
|
96
26
|
def check_sync_errors
|
97
|
-
# get
|
27
|
+
# get executed jobs that could not sync with database
|
98
28
|
Dir.glob('tmp/skiplock/*').each do |f|
|
99
29
|
job_from_db = Job.find_by(id: File.basename(f), running: true)
|
100
30
|
disposed = true
|
101
31
|
if job_from_db
|
102
32
|
job, ex = YAML.load_file(f) rescue nil
|
103
|
-
disposed = job.dispose(ex, purge_completion: @config[:purge_completion], max_retries: @config[:max_retries])
|
33
|
+
disposed = job.dispose(ex, purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) if job
|
104
34
|
end
|
105
35
|
File.delete(f) if disposed
|
106
36
|
end
|
107
37
|
end
|
108
38
|
|
109
|
-
def
|
39
|
+
def get_next_available_job
|
110
40
|
@connection.transaction do
|
111
41
|
job = Job.find_by_sql("SELECT id, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
|
112
|
-
|
113
|
-
|
42
|
+
if job && job.scheduled_at.to_f <= Time.now.to_f
|
43
|
+
job = Job.find_by_sql("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{job.id}' RETURNING *").first
|
44
|
+
end
|
45
|
+
job
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def run
|
50
|
+
@connection = self.class.connection
|
51
|
+
@connection.exec_query('LISTEN "skiplock::jobs"')
|
52
|
+
if self.master
|
53
|
+
Dir.mkdir('tmp/skiplock') unless Dir.exist?('tmp/skiplock')
|
54
|
+
check_sync_errors
|
55
|
+
Cron.setup
|
56
|
+
end
|
57
|
+
error = false
|
58
|
+
timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
59
|
+
while @running
|
60
|
+
Rails.application.reloader.wrap do
|
61
|
+
begin
|
62
|
+
if error
|
63
|
+
unless @connection.active?
|
64
|
+
@connection.reconnect!
|
65
|
+
sleep(0.5)
|
66
|
+
@connection.exec_query('LISTEN "skiplock::jobs"')
|
67
|
+
@next_schedule_at = Time.now.to_f
|
68
|
+
end
|
69
|
+
check_sync_errors if self.master
|
70
|
+
error = false
|
71
|
+
end
|
72
|
+
if Time.now.to_f >= @next_schedule_at && @executor.remaining_capacity > 0
|
73
|
+
job = get_next_available_job
|
74
|
+
if job.try(:running)
|
75
|
+
@executor.post do
|
76
|
+
Rails.application.executor.wrap do
|
77
|
+
ActiveSupport::Dependencies.interlock.permit_concurrent_loads { job.execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
|
78
|
+
end
|
79
|
+
end
|
80
|
+
else
|
81
|
+
@next_schedule_at = (job ? job.scheduled_at.to_f : Float::INFINITY)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
job_notifications = []
|
85
|
+
@connection.raw_connection.wait_for_notify(0.4) do |channel, pid, payload|
|
86
|
+
job_notifications << payload if payload
|
87
|
+
loop do
|
88
|
+
payload = @connection.raw_connection.notifies
|
89
|
+
break unless @running && payload
|
90
|
+
job_notifications << payload[:extra]
|
91
|
+
end
|
92
|
+
job_notifications.each do |n|
|
93
|
+
op, id, worker_id, job_class, queue_name, running, expired_at, finished_at, scheduled_at = n.split(',')
|
94
|
+
next if op == 'DELETE' || running == 'true' || expired_at.to_f > 0 || finished_at.to_f > 0
|
95
|
+
@next_schedule_at = scheduled_at.to_f if scheduled_at.to_f < @next_schedule_at
|
96
|
+
end
|
97
|
+
end
|
98
|
+
if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
|
99
|
+
self.touch
|
100
|
+
timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
101
|
+
end
|
102
|
+
rescue Exception => ex
|
103
|
+
# most likely error with database connection
|
104
|
+
Skiplock.logger.error(ex.to_s)
|
105
|
+
Skiplock.logger.error(ex.backtrace.join("\n"))
|
106
|
+
Skiplock.on_errors.each { |p| p.call(ex, @last_exception) }
|
107
|
+
error = true
|
108
|
+
t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
109
|
+
while @running
|
110
|
+
sleep(0.5)
|
111
|
+
break if Process.clock_gettime(Process::CLOCK_MONOTONIC) - t > 5
|
112
|
+
end
|
113
|
+
@last_exception = ex
|
114
|
+
end
|
115
|
+
sleep(0.3)
|
116
|
+
end
|
114
117
|
end
|
118
|
+
@connection.exec_query('UNLISTEN *')
|
115
119
|
end
|
116
120
|
end
|
117
121
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: skiplock
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tin Vo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-09-
|
11
|
+
date: 2021-09-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activejob
|