skiplock 1.0.21 → 1.0.22
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/skiplock/job.rb +24 -21
- data/lib/skiplock/manager.rb +18 -15
- data/lib/skiplock/version.rb +1 -1
- data/lib/skiplock/worker.rb +19 -15
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3b99466d7f1848f9dcd750a4dceb6a39d7124707e941c74b6c75f0b3a57944ff
|
4
|
+
data.tar.gz: 3f3ed1c9327d41e675d5c170590b52fe32f1facc5b57667dba513e5324e6ee20
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '0129296507b0479ba28a6e4e0bce810340f0ff2559004da1385cf02b6d6023f7be5cb1543f3fb24cebdeb579d6be6ed1e5d7d72d43245de5374926607b23d7df'
|
7
|
+
data.tar.gz: 163faf212d06b6d7f8045eb347ccab77de5b16bfe77fe8780a5d73176ce3c5804cc25cbca8f962d92bc5dfccc1ada223d03fc0b443d0df48125a12bf4ec7ddeb
|
data/lib/skiplock/job.rb
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
module Skiplock
|
2
2
|
class Job < ActiveRecord::Base
|
3
3
|
self.implicit_order_column = 'updated_at'
|
4
|
-
|
4
|
+
attribute :activejob_error
|
5
|
+
attribute :exception
|
6
|
+
attribute :max_retries, :integer
|
7
|
+
attribute :purge, :boolean
|
5
8
|
belongs_to :worker, inverse_of: :jobs, required: false
|
6
9
|
|
7
10
|
def self.dispatch(purge_completion: true, max_retries: 20)
|
@@ -40,7 +43,7 @@ module Skiplock
|
|
40
43
|
Dir.glob('tmp/skiplock/*').each do |f|
|
41
44
|
disposed = true
|
42
45
|
if self.exists?(id: File.basename(f), running: true)
|
43
|
-
job =
|
46
|
+
job = YAML.load_file(f) rescue nil
|
44
47
|
disposed = job.dispose if job.is_a?(Skiplock::Job)
|
45
48
|
end
|
46
49
|
(File.delete(f) rescue nil) if disposed
|
@@ -54,15 +57,15 @@ module Skiplock
|
|
54
57
|
end
|
55
58
|
|
56
59
|
def dispose
|
57
|
-
return unless
|
58
|
-
|
60
|
+
return unless self.max_retries
|
61
|
+
yaml = self.to_yaml
|
59
62
|
purging = false
|
60
63
|
self.running = false
|
61
64
|
self.worker_id = nil
|
62
65
|
self.updated_at = Time.now > self.updated_at ? Time.now : self.updated_at + 1 # in case of clock drifting
|
63
|
-
if
|
64
|
-
self.exception_executions["[#{
|
65
|
-
if (self.executions.to_i >=
|
66
|
+
if self.exception
|
67
|
+
self.exception_executions["[#{self.exception.class.name}]"] = self.exception_executions["[#{self.exception.class.name}]"].to_i + 1 unless self.data.key?('activejob_error')
|
68
|
+
if (self.executions.to_i >= self.max_retries + 1) || self.data.key?('activejob_error') || self.exception.is_a?(Skiplock::Extension::ProxyError)
|
66
69
|
self.expired_at = Time.now
|
67
70
|
else
|
68
71
|
self.scheduled_at = Time.now + (5 * 2**self.executions.to_i)
|
@@ -76,7 +79,7 @@ module Skiplock
|
|
76
79
|
next_cron_at = Cron.next_schedule_at(self.cron)
|
77
80
|
if next_cron_at
|
78
81
|
# update job to record completions counter before resetting finished_at to nil
|
79
|
-
self.update_columns(self.attributes.slice(*self.changes.keys))
|
82
|
+
self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
|
80
83
|
self.finished_at = nil
|
81
84
|
self.executions = nil
|
82
85
|
self.exception_executions = nil
|
@@ -86,13 +89,13 @@ module Skiplock
|
|
86
89
|
Skiplock.logger.error("[Skiplock] ERROR: Invalid CRON '#{self.cron}' for Job #{self.job_class}") if Skiplock.logger
|
87
90
|
purging = true
|
88
91
|
end
|
89
|
-
elsif
|
92
|
+
elsif self.purge == true
|
90
93
|
purging = true
|
91
94
|
end
|
92
95
|
end
|
93
|
-
purging ? self.delete : self.update_columns(self.attributes.slice(*self.changes.keys))
|
96
|
+
purging ? self.delete : self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
|
94
97
|
rescue Exception => e
|
95
|
-
File.
|
98
|
+
File.write("tmp/skiplock/#{self.id}", yaml) rescue nil
|
96
99
|
if Skiplock.logger
|
97
100
|
Skiplock.logger.error(e.to_s)
|
98
101
|
Skiplock.logger.error(e.backtrace.join("\n"))
|
@@ -109,9 +112,9 @@ module Skiplock
|
|
109
112
|
self.data.delete('result')
|
110
113
|
self.exception_executions ||= {}
|
111
114
|
self.activejob_error = nil
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
+
self.max_retries = (self.data['options'].key?('max_retries') ? self.data['options']['max_retries'].to_i : max_retries) rescue max_retries
|
116
|
+
self.max_retries = 20 if self.max_retries < 0 || self.max_retries > 20
|
117
|
+
self.purge = (self.data['options'].key?('purge') ? self.data['options']['purge'] : purge_completion) rescue purge_completion
|
115
118
|
job_data = self.attributes.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions', 'exception_executions').merge('job_id' => self.id, 'enqueued_at' => self.updated_at, 'arguments' => (self.data['arguments'] || []))
|
116
119
|
self.executions = self.executions.to_i + 1
|
117
120
|
Thread.current[:skiplock_job] = self
|
@@ -119,15 +122,15 @@ module Skiplock
|
|
119
122
|
begin
|
120
123
|
self.data['result'] = ActiveJob::Base.execute(job_data)
|
121
124
|
rescue Exception => ex
|
122
|
-
|
123
|
-
Skiplock.on_errors.each { |p| p.call(
|
125
|
+
self.exception = ex
|
126
|
+
Skiplock.on_errors.each { |p| p.call(ex) }
|
124
127
|
end
|
125
128
|
if Skiplock.logger
|
126
|
-
if
|
129
|
+
if self.exception || self.activejob_error
|
127
130
|
Skiplock.logger.error("[Skiplock] Job #{self.job_class} (#{self.id}) was interrupted by an exception#{ ' (rescued and retried by ActiveJob)' if self.activejob_error }")
|
128
|
-
if
|
129
|
-
Skiplock.logger.error(
|
130
|
-
Skiplock.logger.error(
|
131
|
+
if self.exception
|
132
|
+
Skiplock.logger.error(self.exception.to_s)
|
133
|
+
Skiplock.logger.error(self.exception.backtrace.join("\n"))
|
131
134
|
end
|
132
135
|
else
|
133
136
|
end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
@@ -139,7 +142,7 @@ module Skiplock
|
|
139
142
|
Skiplock.logger.info "[Skiplock] Performed #{job_name} (#{self.id}) from queue '#{self.queue_name || 'default'}' in #{end_time - start_time} seconds"
|
140
143
|
end
|
141
144
|
end
|
142
|
-
|
145
|
+
self.exception || self.activejob_error || self.data['result']
|
143
146
|
ensure
|
144
147
|
self.finished_at ||= Time.now if self.data.key?('result') && !self.activejob_error
|
145
148
|
self.dispose
|
data/lib/skiplock/manager.rb
CHANGED
@@ -4,12 +4,18 @@ module Skiplock
|
|
4
4
|
@config = Skiplock::DEFAULT_CONFIG.dup
|
5
5
|
@config.merge!(YAML.load_file('config/skiplock.yml')) rescue nil
|
6
6
|
@config.symbolize_keys!
|
7
|
+
Rails.application.eager_load! if Rails.env.development?
|
8
|
+
if @config[:extensions] == true
|
9
|
+
Module.__send__(:include, Skiplock::Extension)
|
10
|
+
elsif @config[:extensions].is_a?(Array)
|
11
|
+
@config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
|
12
|
+
end
|
7
13
|
async if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
|
8
14
|
end
|
9
15
|
|
10
16
|
def async
|
11
|
-
configure
|
12
17
|
setup_logger
|
18
|
+
configure
|
13
19
|
Worker.cleanup(@hostname)
|
14
20
|
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
|
15
21
|
@worker.start(**@config)
|
@@ -21,22 +27,20 @@ module Skiplock
|
|
21
27
|
|
22
28
|
def standalone(**options)
|
23
29
|
@config.merge!(options)
|
24
|
-
configure
|
25
|
-
setup_logger
|
26
|
-
Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
|
27
|
-
Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
|
28
|
-
@config[:workers] = 1 if @config[:workers] <= 0
|
29
30
|
@config[:standalone] = true
|
31
|
+
@config[:workers] = 1 if @config[:workers] <= 0
|
32
|
+
setup_logger
|
33
|
+
configure
|
30
34
|
banner
|
31
|
-
Worker.cleanup(@hostname)
|
32
|
-
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
|
33
35
|
@parent_id = Process.pid
|
34
36
|
@shutdown = false
|
35
37
|
Signal.trap('INT') { @shutdown = true }
|
36
38
|
Signal.trap('TERM') { @shutdown = true }
|
37
39
|
Signal.trap('HUP') { setup_logger }
|
40
|
+
Worker.cleanup(@hostname)
|
41
|
+
@worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
|
42
|
+
ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
|
38
43
|
(@config[:workers] - 1).times do |n|
|
39
|
-
sleep 0.2
|
40
44
|
fork do
|
41
45
|
sleep 1
|
42
46
|
worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
|
@@ -48,6 +52,7 @@ module Skiplock
|
|
48
52
|
worker.shutdown
|
49
53
|
end
|
50
54
|
end
|
55
|
+
ActiveRecord::Base.establish_connection if @config[:workers] > 1
|
51
56
|
@worker.start(**@config)
|
52
57
|
loop do
|
53
58
|
sleep 0.5
|
@@ -107,12 +112,6 @@ module Skiplock
|
|
107
112
|
raise "Unable to detect any known exception notification library. Please define custom 'on_error' event callbacks and change to 'custom' notification in 'config/skiplock.yml'"
|
108
113
|
end
|
109
114
|
end
|
110
|
-
Rails.application.eager_load! if Rails.env.development?
|
111
|
-
if @config[:extensions] == true
|
112
|
-
Module.__send__(:include, Skiplock::Extension)
|
113
|
-
elsif @config[:extensions].is_a?(Array)
|
114
|
-
@config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
|
115
|
-
end
|
116
115
|
case @config[:notification]
|
117
116
|
when 'airbrake'
|
118
117
|
raise 'airbrake gem not found' unless defined?(Airbrake)
|
@@ -144,6 +143,10 @@ module Skiplock
|
|
144
143
|
@logger.extend(ActiveSupport::Logger.broadcast(::Logger.new(File.join(Rails.root, 'log', @config[:logfile].to_s), 'daily')))
|
145
144
|
ActiveJob::Base.logger = nil
|
146
145
|
end
|
146
|
+
if @config[:standalone]
|
147
|
+
Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
|
148
|
+
Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
|
149
|
+
end
|
147
150
|
rescue Exception => ex
|
148
151
|
@logger.error "Exception with logger: #{ex.to_s}"
|
149
152
|
@logger.error ex.backtrace.join("\n")
|
data/lib/skiplock/version.rb
CHANGED
data/lib/skiplock/worker.rb
CHANGED
@@ -32,37 +32,39 @@ module Skiplock
|
|
32
32
|
@queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
|
33
33
|
@running = true
|
34
34
|
@executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :discard)
|
35
|
-
if self.master
|
36
|
-
Job.flush
|
37
|
-
Cron.setup
|
38
|
-
end
|
39
35
|
@executor.post { run }
|
40
36
|
Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
|
41
37
|
end
|
42
38
|
|
43
39
|
private
|
44
40
|
|
45
|
-
def reloader_post
|
46
|
-
Rails.application.reloader.wrap { @executor.post { Rails.application.executor.wrap { yield } } } if block_given?
|
47
|
-
end
|
48
|
-
|
49
41
|
def run
|
50
42
|
sleep 3
|
51
43
|
Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
|
44
|
+
connection = nil
|
52
45
|
error = false
|
46
|
+
listen = false
|
53
47
|
next_schedule_at = Time.now.to_f
|
54
48
|
pg_exception_timestamp = nil
|
55
49
|
timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
56
|
-
|
57
|
-
|
58
|
-
while @running
|
50
|
+
while @running
|
51
|
+
Rails.application.reloader.wrap do
|
59
52
|
begin
|
53
|
+
unless listen
|
54
|
+
connection = self.class.connection
|
55
|
+
connection.exec_query('LISTEN "skiplock::jobs"')
|
56
|
+
if self.master
|
57
|
+
Job.flush
|
58
|
+
Cron.setup
|
59
|
+
end
|
60
|
+
listen = true
|
61
|
+
end
|
60
62
|
if error
|
61
63
|
unless connection.active?
|
62
64
|
connection.reconnect!
|
63
65
|
sleep(0.5)
|
64
66
|
connection.exec_query('LISTEN "skiplock::jobs"')
|
65
|
-
|
67
|
+
Job.flush if self.master
|
66
68
|
pg_exception_timestamp = nil
|
67
69
|
next_schedule_at = Time.now.to_f
|
68
70
|
end
|
@@ -75,7 +77,9 @@ module Skiplock
|
|
75
77
|
result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
|
76
78
|
end
|
77
79
|
if result && result['running']
|
78
|
-
|
80
|
+
@executor.post do
|
81
|
+
Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
|
82
|
+
end
|
79
83
|
else
|
80
84
|
next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
|
81
85
|
end
|
@@ -103,7 +107,7 @@ module Skiplock
|
|
103
107
|
Skiplock.logger.error(ex.backtrace.join("\n"))
|
104
108
|
report_exception = true
|
105
109
|
# if error is with database connection then only report if it persists longer than 1 minute
|
106
|
-
if ex.is_a?(::PG::ConnectionBad)
|
110
|
+
if ex.is_a?(::PG::ConnectionBad) ||ex.is_a?(::PG::UnableToSend) || ex.message.include?('Bad file descriptor')
|
107
111
|
report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
|
108
112
|
pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
109
113
|
end
|
@@ -113,8 +117,8 @@ module Skiplock
|
|
113
117
|
end
|
114
118
|
sleep(0.3)
|
115
119
|
end
|
116
|
-
connection.exec_query('UNLISTEN *')
|
117
120
|
end
|
121
|
+
connection.exec_query('UNLISTEN *')
|
118
122
|
end
|
119
123
|
|
120
124
|
def wait(timeout)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: skiplock
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.22
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tin Vo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-09-
|
11
|
+
date: 2021-09-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activejob
|