skiplock 1.0.21 → 1.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 69e771cf5508d4a5a648dab43eb33b980370e317a1bf370ceaff62ecee4592e3
4
- data.tar.gz: d1df71a515d871073b8d039f1bcbc3fb00b398806ed9f3bd82dc0169737e2f52
3
+ metadata.gz: 3b99466d7f1848f9dcd750a4dceb6a39d7124707e941c74b6c75f0b3a57944ff
4
+ data.tar.gz: 3f3ed1c9327d41e675d5c170590b52fe32f1facc5b57667dba513e5324e6ee20
5
5
  SHA512:
6
- metadata.gz: cca4a86fc4c31e4054a920b32740f294e3f54e9de65fe3a0c31b43258fee6e6f1a790ae1a95f79d9b719119936fedac953afd695ef5e7ce9e990b7f29fa1c397
7
- data.tar.gz: 6bdc5a38218b052a825ab2b5f5ef2c73a2c9a6a2d981d8aed7cc61244f8806fdad06876278aa320bafb956c9dc2f15b705d87a2f171cbb2bc3805bfada640ce4
6
+ metadata.gz: '0129296507b0479ba28a6e4e0bce810340f0ff2559004da1385cf02b6d6023f7be5cb1543f3fb24cebdeb579d6be6ed1e5d7d72d43245de5374926607b23d7df'
7
+ data.tar.gz: 163faf212d06b6d7f8045eb347ccab77de5b16bfe77fe8780a5d73176ce3c5804cc25cbca8f962d92bc5dfccc1ada223d03fc0b443d0df48125a12bf4ec7ddeb
data/lib/skiplock/job.rb CHANGED
@@ -1,7 +1,10 @@
1
1
  module Skiplock
2
2
  class Job < ActiveRecord::Base
3
3
  self.implicit_order_column = 'updated_at'
4
- attr_accessor :activejob_error
4
+ attribute :activejob_error
5
+ attribute :exception
6
+ attribute :max_retries, :integer
7
+ attribute :purge, :boolean
5
8
  belongs_to :worker, inverse_of: :jobs, required: false
6
9
 
7
10
  def self.dispatch(purge_completion: true, max_retries: 20)
@@ -40,7 +43,7 @@ module Skiplock
40
43
  Dir.glob('tmp/skiplock/*').each do |f|
41
44
  disposed = true
42
45
  if self.exists?(id: File.basename(f), running: true)
43
- job = Marshal.load(File.binread(f)) rescue nil
46
+ job = YAML.load_file(f) rescue nil
44
47
  disposed = job.dispose if job.is_a?(Skiplock::Job)
45
48
  end
46
49
  (File.delete(f) rescue nil) if disposed
@@ -54,15 +57,15 @@ module Skiplock
54
57
  end
55
58
 
56
59
  def dispose
57
- return unless @max_retries
58
- dump = Marshal.dump(self)
60
+ return unless self.max_retries
61
+ yaml = self.to_yaml
59
62
  purging = false
60
63
  self.running = false
61
64
  self.worker_id = nil
62
65
  self.updated_at = Time.now > self.updated_at ? Time.now : self.updated_at + 1 # in case of clock drifting
63
- if @exception
64
- self.exception_executions["[#{@exception.class.name}]"] = self.exception_executions["[#{@exception.class.name}]"].to_i + 1 unless self.data.key?('activejob_error')
65
- if (self.executions.to_i >= @max_retries + 1) || self.data.key?('activejob_error') || @exception.is_a?(Skiplock::Extension::ProxyError)
66
+ if self.exception
67
+ self.exception_executions["[#{self.exception.class.name}]"] = self.exception_executions["[#{self.exception.class.name}]"].to_i + 1 unless self.data.key?('activejob_error')
68
+ if (self.executions.to_i >= self.max_retries + 1) || self.data.key?('activejob_error') || self.exception.is_a?(Skiplock::Extension::ProxyError)
66
69
  self.expired_at = Time.now
67
70
  else
68
71
  self.scheduled_at = Time.now + (5 * 2**self.executions.to_i)
@@ -76,7 +79,7 @@ module Skiplock
76
79
  next_cron_at = Cron.next_schedule_at(self.cron)
77
80
  if next_cron_at
78
81
  # update job to record completions counter before resetting finished_at to nil
79
- self.update_columns(self.attributes.slice(*self.changes.keys))
82
+ self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
80
83
  self.finished_at = nil
81
84
  self.executions = nil
82
85
  self.exception_executions = nil
@@ -86,13 +89,13 @@ module Skiplock
86
89
  Skiplock.logger.error("[Skiplock] ERROR: Invalid CRON '#{self.cron}' for Job #{self.job_class}") if Skiplock.logger
87
90
  purging = true
88
91
  end
89
- elsif @purge == true
92
+ elsif self.purge == true
90
93
  purging = true
91
94
  end
92
95
  end
93
- purging ? self.delete : self.update_columns(self.attributes.slice(*self.changes.keys))
96
+ purging ? self.delete : self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
94
97
  rescue Exception => e
95
- File.binwrite("tmp/skiplock/#{self.id}", dump) rescue nil
98
+ File.write("tmp/skiplock/#{self.id}", yaml) rescue nil
96
99
  if Skiplock.logger
97
100
  Skiplock.logger.error(e.to_s)
98
101
  Skiplock.logger.error(e.backtrace.join("\n"))
@@ -109,9 +112,9 @@ module Skiplock
109
112
  self.data.delete('result')
110
113
  self.exception_executions ||= {}
111
114
  self.activejob_error = nil
112
- @max_retries = (self.data['options'].key?('max_retries') ? self.data['options']['max_retries'].to_i : max_retries) rescue max_retries
113
- @max_retries = 20 if @max_retries < 0 || @max_retries > 20
114
- @purge = (self.data['options'].key?('purge') ? self.data['options']['purge'] : purge_completion) rescue purge_completion
115
+ self.max_retries = (self.data['options'].key?('max_retries') ? self.data['options']['max_retries'].to_i : max_retries) rescue max_retries
116
+ self.max_retries = 20 if self.max_retries < 0 || self.max_retries > 20
117
+ self.purge = (self.data['options'].key?('purge') ? self.data['options']['purge'] : purge_completion) rescue purge_completion
115
118
  job_data = self.attributes.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions', 'exception_executions').merge('job_id' => self.id, 'enqueued_at' => self.updated_at, 'arguments' => (self.data['arguments'] || []))
116
119
  self.executions = self.executions.to_i + 1
117
120
  Thread.current[:skiplock_job] = self
@@ -119,15 +122,15 @@ module Skiplock
119
122
  begin
120
123
  self.data['result'] = ActiveJob::Base.execute(job_data)
121
124
  rescue Exception => ex
122
- @exception = ex
123
- Skiplock.on_errors.each { |p| p.call(@exception) }
125
+ self.exception = ex
126
+ Skiplock.on_errors.each { |p| p.call(ex) }
124
127
  end
125
128
  if Skiplock.logger
126
- if @exception || self.activejob_error
129
+ if self.exception || self.activejob_error
127
130
  Skiplock.logger.error("[Skiplock] Job #{self.job_class} (#{self.id}) was interrupted by an exception#{ ' (rescued and retried by ActiveJob)' if self.activejob_error }")
128
- if @exception
129
- Skiplock.logger.error(@exception.to_s)
130
- Skiplock.logger.error(@exception.backtrace.join("\n"))
131
+ if self.exception
132
+ Skiplock.logger.error(self.exception.to_s)
133
+ Skiplock.logger.error(self.exception.backtrace.join("\n"))
131
134
  end
132
135
  else
133
136
  end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -139,7 +142,7 @@ module Skiplock
139
142
  Skiplock.logger.info "[Skiplock] Performed #{job_name} (#{self.id}) from queue '#{self.queue_name || 'default'}' in #{end_time - start_time} seconds"
140
143
  end
141
144
  end
142
- @exception || self.activejob_error || self.data['result']
145
+ self.exception || self.activejob_error || self.data['result']
143
146
  ensure
144
147
  self.finished_at ||= Time.now if self.data.key?('result') && !self.activejob_error
145
148
  self.dispose
@@ -4,12 +4,18 @@ module Skiplock
4
4
  @config = Skiplock::DEFAULT_CONFIG.dup
5
5
  @config.merge!(YAML.load_file('config/skiplock.yml')) rescue nil
6
6
  @config.symbolize_keys!
7
+ Rails.application.eager_load! if Rails.env.development?
8
+ if @config[:extensions] == true
9
+ Module.__send__(:include, Skiplock::Extension)
10
+ elsif @config[:extensions].is_a?(Array)
11
+ @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
12
+ end
7
13
  async if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
8
14
  end
9
15
 
10
16
  def async
11
- configure
12
17
  setup_logger
18
+ configure
13
19
  Worker.cleanup(@hostname)
14
20
  @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
15
21
  @worker.start(**@config)
@@ -21,22 +27,20 @@ module Skiplock
21
27
 
22
28
  def standalone(**options)
23
29
  @config.merge!(options)
24
- configure
25
- setup_logger
26
- Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
27
- Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
28
- @config[:workers] = 1 if @config[:workers] <= 0
29
30
  @config[:standalone] = true
31
+ @config[:workers] = 1 if @config[:workers] <= 0
32
+ setup_logger
33
+ configure
30
34
  banner
31
- Worker.cleanup(@hostname)
32
- @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
33
35
  @parent_id = Process.pid
34
36
  @shutdown = false
35
37
  Signal.trap('INT') { @shutdown = true }
36
38
  Signal.trap('TERM') { @shutdown = true }
37
39
  Signal.trap('HUP') { setup_logger }
40
+ Worker.cleanup(@hostname)
41
+ @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
42
+ ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
38
43
  (@config[:workers] - 1).times do |n|
39
- sleep 0.2
40
44
  fork do
41
45
  sleep 1
42
46
  worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
@@ -48,6 +52,7 @@ module Skiplock
48
52
  worker.shutdown
49
53
  end
50
54
  end
55
+ ActiveRecord::Base.establish_connection if @config[:workers] > 1
51
56
  @worker.start(**@config)
52
57
  loop do
53
58
  sleep 0.5
@@ -107,12 +112,6 @@ module Skiplock
107
112
  raise "Unable to detect any known exception notification library. Please define custom 'on_error' event callbacks and change to 'custom' notification in 'config/skiplock.yml'"
108
113
  end
109
114
  end
110
- Rails.application.eager_load! if Rails.env.development?
111
- if @config[:extensions] == true
112
- Module.__send__(:include, Skiplock::Extension)
113
- elsif @config[:extensions].is_a?(Array)
114
- @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
115
- end
116
115
  case @config[:notification]
117
116
  when 'airbrake'
118
117
  raise 'airbrake gem not found' unless defined?(Airbrake)
@@ -144,6 +143,10 @@ module Skiplock
144
143
  @logger.extend(ActiveSupport::Logger.broadcast(::Logger.new(File.join(Rails.root, 'log', @config[:logfile].to_s), 'daily')))
145
144
  ActiveJob::Base.logger = nil
146
145
  end
146
+ if @config[:standalone]
147
+ Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
148
+ Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
149
+ end
147
150
  rescue Exception => ex
148
151
  @logger.error "Exception with logger: #{ex.to_s}"
149
152
  @logger.error ex.backtrace.join("\n")
@@ -1,4 +1,4 @@
1
1
  module Skiplock
2
- VERSION = Version = '1.0.21'
2
+ VERSION = Version = '1.0.22'
3
3
  end
4
4
 
@@ -32,37 +32,39 @@ module Skiplock
32
32
  @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
33
33
  @running = true
34
34
  @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :discard)
35
- if self.master
36
- Job.flush
37
- Cron.setup
38
- end
39
35
  @executor.post { run }
40
36
  Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
41
37
  end
42
38
 
43
39
  private
44
40
 
45
- def reloader_post
46
- Rails.application.reloader.wrap { @executor.post { Rails.application.executor.wrap { yield } } } if block_given?
47
- end
48
-
49
41
  def run
50
42
  sleep 3
51
43
  Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
44
+ connection = nil
52
45
  error = false
46
+ listen = false
53
47
  next_schedule_at = Time.now.to_f
54
48
  pg_exception_timestamp = nil
55
49
  timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
56
- ActiveRecord::Base.connection_pool.with_connection do |connection|
57
- connection.exec_query('LISTEN "skiplock::jobs"')
58
- while @running
50
+ while @running
51
+ Rails.application.reloader.wrap do
59
52
  begin
53
+ unless listen
54
+ connection = self.class.connection
55
+ connection.exec_query('LISTEN "skiplock::jobs"')
56
+ if self.master
57
+ Job.flush
58
+ Cron.setup
59
+ end
60
+ listen = true
61
+ end
60
62
  if error
61
63
  unless connection.active?
62
64
  connection.reconnect!
63
65
  sleep(0.5)
64
66
  connection.exec_query('LISTEN "skiplock::jobs"')
65
- reloader_post { Job.flush } if self.master
67
+ Job.flush if self.master
66
68
  pg_exception_timestamp = nil
67
69
  next_schedule_at = Time.now.to_f
68
70
  end
@@ -75,7 +77,9 @@ module Skiplock
75
77
  result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
76
78
  end
77
79
  if result && result['running']
78
- reloader_post { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
80
+ @executor.post do
81
+ Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
82
+ end
79
83
  else
80
84
  next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
81
85
  end
@@ -103,7 +107,7 @@ module Skiplock
103
107
  Skiplock.logger.error(ex.backtrace.join("\n"))
104
108
  report_exception = true
105
109
  # if error is with database connection then only report if it persists longer than 1 minute
106
- if ex.is_a?(::PG::ConnectionBad)
110
+ if ex.is_a?(::PG::ConnectionBad) ||ex.is_a?(::PG::UnableToSend) || ex.message.include?('Bad file descriptor')
107
111
  report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
108
112
  pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
109
113
  end
@@ -113,8 +117,8 @@ module Skiplock
113
117
  end
114
118
  sleep(0.3)
115
119
  end
116
- connection.exec_query('UNLISTEN *')
117
120
  end
121
+ connection.exec_query('UNLISTEN *')
118
122
  end
119
123
 
120
124
  def wait(timeout)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: skiplock
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.21
4
+ version: 1.0.22
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tin Vo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-09-16 00:00:00.000000000 Z
11
+ date: 2021-09-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activejob