skiplock 1.0.21 → 1.0.22

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 69e771cf5508d4a5a648dab43eb33b980370e317a1bf370ceaff62ecee4592e3
4
- data.tar.gz: d1df71a515d871073b8d039f1bcbc3fb00b398806ed9f3bd82dc0169737e2f52
3
+ metadata.gz: 3b99466d7f1848f9dcd750a4dceb6a39d7124707e941c74b6c75f0b3a57944ff
4
+ data.tar.gz: 3f3ed1c9327d41e675d5c170590b52fe32f1facc5b57667dba513e5324e6ee20
5
5
  SHA512:
6
- metadata.gz: cca4a86fc4c31e4054a920b32740f294e3f54e9de65fe3a0c31b43258fee6e6f1a790ae1a95f79d9b719119936fedac953afd695ef5e7ce9e990b7f29fa1c397
7
- data.tar.gz: 6bdc5a38218b052a825ab2b5f5ef2c73a2c9a6a2d981d8aed7cc61244f8806fdad06876278aa320bafb956c9dc2f15b705d87a2f171cbb2bc3805bfada640ce4
6
+ metadata.gz: '0129296507b0479ba28a6e4e0bce810340f0ff2559004da1385cf02b6d6023f7be5cb1543f3fb24cebdeb579d6be6ed1e5d7d72d43245de5374926607b23d7df'
7
+ data.tar.gz: 163faf212d06b6d7f8045eb347ccab77de5b16bfe77fe8780a5d73176ce3c5804cc25cbca8f962d92bc5dfccc1ada223d03fc0b443d0df48125a12bf4ec7ddeb
data/lib/skiplock/job.rb CHANGED
@@ -1,7 +1,10 @@
1
1
  module Skiplock
2
2
  class Job < ActiveRecord::Base
3
3
  self.implicit_order_column = 'updated_at'
4
- attr_accessor :activejob_error
4
+ attribute :activejob_error
5
+ attribute :exception
6
+ attribute :max_retries, :integer
7
+ attribute :purge, :boolean
5
8
  belongs_to :worker, inverse_of: :jobs, required: false
6
9
 
7
10
  def self.dispatch(purge_completion: true, max_retries: 20)
@@ -40,7 +43,7 @@ module Skiplock
40
43
  Dir.glob('tmp/skiplock/*').each do |f|
41
44
  disposed = true
42
45
  if self.exists?(id: File.basename(f), running: true)
43
- job = Marshal.load(File.binread(f)) rescue nil
46
+ job = YAML.load_file(f) rescue nil
44
47
  disposed = job.dispose if job.is_a?(Skiplock::Job)
45
48
  end
46
49
  (File.delete(f) rescue nil) if disposed
@@ -54,15 +57,15 @@ module Skiplock
54
57
  end
55
58
 
56
59
  def dispose
57
- return unless @max_retries
58
- dump = Marshal.dump(self)
60
+ return unless self.max_retries
61
+ yaml = self.to_yaml
59
62
  purging = false
60
63
  self.running = false
61
64
  self.worker_id = nil
62
65
  self.updated_at = Time.now > self.updated_at ? Time.now : self.updated_at + 1 # in case of clock drifting
63
- if @exception
64
- self.exception_executions["[#{@exception.class.name}]"] = self.exception_executions["[#{@exception.class.name}]"].to_i + 1 unless self.data.key?('activejob_error')
65
- if (self.executions.to_i >= @max_retries + 1) || self.data.key?('activejob_error') || @exception.is_a?(Skiplock::Extension::ProxyError)
66
+ if self.exception
67
+ self.exception_executions["[#{self.exception.class.name}]"] = self.exception_executions["[#{self.exception.class.name}]"].to_i + 1 unless self.data.key?('activejob_error')
68
+ if (self.executions.to_i >= self.max_retries + 1) || self.data.key?('activejob_error') || self.exception.is_a?(Skiplock::Extension::ProxyError)
66
69
  self.expired_at = Time.now
67
70
  else
68
71
  self.scheduled_at = Time.now + (5 * 2**self.executions.to_i)
@@ -76,7 +79,7 @@ module Skiplock
76
79
  next_cron_at = Cron.next_schedule_at(self.cron)
77
80
  if next_cron_at
78
81
  # update job to record completions counter before resetting finished_at to nil
79
- self.update_columns(self.attributes.slice(*self.changes.keys))
82
+ self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
80
83
  self.finished_at = nil
81
84
  self.executions = nil
82
85
  self.exception_executions = nil
@@ -86,13 +89,13 @@ module Skiplock
86
89
  Skiplock.logger.error("[Skiplock] ERROR: Invalid CRON '#{self.cron}' for Job #{self.job_class}") if Skiplock.logger
87
90
  purging = true
88
91
  end
89
- elsif @purge == true
92
+ elsif self.purge == true
90
93
  purging = true
91
94
  end
92
95
  end
93
- purging ? self.delete : self.update_columns(self.attributes.slice(*self.changes.keys))
96
+ purging ? self.delete : self.update_columns(self.attributes.slice(*(self.changes.keys & self.class.column_names)))
94
97
  rescue Exception => e
95
- File.binwrite("tmp/skiplock/#{self.id}", dump) rescue nil
98
+ File.write("tmp/skiplock/#{self.id}", yaml) rescue nil
96
99
  if Skiplock.logger
97
100
  Skiplock.logger.error(e.to_s)
98
101
  Skiplock.logger.error(e.backtrace.join("\n"))
@@ -109,9 +112,9 @@ module Skiplock
109
112
  self.data.delete('result')
110
113
  self.exception_executions ||= {}
111
114
  self.activejob_error = nil
112
- @max_retries = (self.data['options'].key?('max_retries') ? self.data['options']['max_retries'].to_i : max_retries) rescue max_retries
113
- @max_retries = 20 if @max_retries < 0 || @max_retries > 20
114
- @purge = (self.data['options'].key?('purge') ? self.data['options']['purge'] : purge_completion) rescue purge_completion
115
+ self.max_retries = (self.data['options'].key?('max_retries') ? self.data['options']['max_retries'].to_i : max_retries) rescue max_retries
116
+ self.max_retries = 20 if self.max_retries < 0 || self.max_retries > 20
117
+ self.purge = (self.data['options'].key?('purge') ? self.data['options']['purge'] : purge_completion) rescue purge_completion
115
118
  job_data = self.attributes.slice('job_class', 'queue_name', 'locale', 'timezone', 'priority', 'executions', 'exception_executions').merge('job_id' => self.id, 'enqueued_at' => self.updated_at, 'arguments' => (self.data['arguments'] || []))
116
119
  self.executions = self.executions.to_i + 1
117
120
  Thread.current[:skiplock_job] = self
@@ -119,15 +122,15 @@ module Skiplock
119
122
  begin
120
123
  self.data['result'] = ActiveJob::Base.execute(job_data)
121
124
  rescue Exception => ex
122
- @exception = ex
123
- Skiplock.on_errors.each { |p| p.call(@exception) }
125
+ self.exception = ex
126
+ Skiplock.on_errors.each { |p| p.call(ex) }
124
127
  end
125
128
  if Skiplock.logger
126
- if @exception || self.activejob_error
129
+ if self.exception || self.activejob_error
127
130
  Skiplock.logger.error("[Skiplock] Job #{self.job_class} (#{self.id}) was interrupted by an exception#{ ' (rescued and retried by ActiveJob)' if self.activejob_error }")
128
- if @exception
129
- Skiplock.logger.error(@exception.to_s)
130
- Skiplock.logger.error(@exception.backtrace.join("\n"))
131
+ if self.exception
132
+ Skiplock.logger.error(self.exception.to_s)
133
+ Skiplock.logger.error(self.exception.backtrace.join("\n"))
131
134
  end
132
135
  else
133
136
  end_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -139,7 +142,7 @@ module Skiplock
139
142
  Skiplock.logger.info "[Skiplock] Performed #{job_name} (#{self.id}) from queue '#{self.queue_name || 'default'}' in #{end_time - start_time} seconds"
140
143
  end
141
144
  end
142
- @exception || self.activejob_error || self.data['result']
145
+ self.exception || self.activejob_error || self.data['result']
143
146
  ensure
144
147
  self.finished_at ||= Time.now if self.data.key?('result') && !self.activejob_error
145
148
  self.dispose
@@ -4,12 +4,18 @@ module Skiplock
4
4
  @config = Skiplock::DEFAULT_CONFIG.dup
5
5
  @config.merge!(YAML.load_file('config/skiplock.yml')) rescue nil
6
6
  @config.symbolize_keys!
7
+ Rails.application.eager_load! if Rails.env.development?
8
+ if @config[:extensions] == true
9
+ Module.__send__(:include, Skiplock::Extension)
10
+ elsif @config[:extensions].is_a?(Array)
11
+ @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
12
+ end
7
13
  async if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
8
14
  end
9
15
 
10
16
  def async
11
- configure
12
17
  setup_logger
18
+ configure
13
19
  Worker.cleanup(@hostname)
14
20
  @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
15
21
  @worker.start(**@config)
@@ -21,22 +27,20 @@ module Skiplock
21
27
 
22
28
  def standalone(**options)
23
29
  @config.merge!(options)
24
- configure
25
- setup_logger
26
- Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
27
- Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
28
- @config[:workers] = 1 if @config[:workers] <= 0
29
30
  @config[:standalone] = true
31
+ @config[:workers] = 1 if @config[:workers] <= 0
32
+ setup_logger
33
+ configure
30
34
  banner
31
- Worker.cleanup(@hostname)
32
- @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
33
35
  @parent_id = Process.pid
34
36
  @shutdown = false
35
37
  Signal.trap('INT') { @shutdown = true }
36
38
  Signal.trap('TERM') { @shutdown = true }
37
39
  Signal.trap('HUP') { setup_logger }
40
+ Worker.cleanup(@hostname)
41
+ @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
42
+ ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
38
43
  (@config[:workers] - 1).times do |n|
39
- sleep 0.2
40
44
  fork do
41
45
  sleep 1
42
46
  worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
@@ -48,6 +52,7 @@ module Skiplock
48
52
  worker.shutdown
49
53
  end
50
54
  end
55
+ ActiveRecord::Base.establish_connection if @config[:workers] > 1
51
56
  @worker.start(**@config)
52
57
  loop do
53
58
  sleep 0.5
@@ -107,12 +112,6 @@ module Skiplock
107
112
  raise "Unable to detect any known exception notification library. Please define custom 'on_error' event callbacks and change to 'custom' notification in 'config/skiplock.yml'"
108
113
  end
109
114
  end
110
- Rails.application.eager_load! if Rails.env.development?
111
- if @config[:extensions] == true
112
- Module.__send__(:include, Skiplock::Extension)
113
- elsif @config[:extensions].is_a?(Array)
114
- @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
115
- end
116
115
  case @config[:notification]
117
116
  when 'airbrake'
118
117
  raise 'airbrake gem not found' unless defined?(Airbrake)
@@ -144,6 +143,10 @@ module Skiplock
144
143
  @logger.extend(ActiveSupport::Logger.broadcast(::Logger.new(File.join(Rails.root, 'log', @config[:logfile].to_s), 'daily')))
145
144
  ActiveJob::Base.logger = nil
146
145
  end
146
+ if @config[:standalone]
147
+ Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
148
+ Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
149
+ end
147
150
  rescue Exception => ex
148
151
  @logger.error "Exception with logger: #{ex.to_s}"
149
152
  @logger.error ex.backtrace.join("\n")
@@ -1,4 +1,4 @@
1
1
  module Skiplock
2
- VERSION = Version = '1.0.21'
2
+ VERSION = Version = '1.0.22'
3
3
  end
4
4
 
@@ -32,37 +32,39 @@ module Skiplock
32
32
  @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
33
33
  @running = true
34
34
  @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :discard)
35
- if self.master
36
- Job.flush
37
- Cron.setup
38
- end
39
35
  @executor.post { run }
40
36
  Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
41
37
  end
42
38
 
43
39
  private
44
40
 
45
- def reloader_post
46
- Rails.application.reloader.wrap { @executor.post { Rails.application.executor.wrap { yield } } } if block_given?
47
- end
48
-
49
41
  def run
50
42
  sleep 3
51
43
  Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
44
+ connection = nil
52
45
  error = false
46
+ listen = false
53
47
  next_schedule_at = Time.now.to_f
54
48
  pg_exception_timestamp = nil
55
49
  timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
56
- ActiveRecord::Base.connection_pool.with_connection do |connection|
57
- connection.exec_query('LISTEN "skiplock::jobs"')
58
- while @running
50
+ while @running
51
+ Rails.application.reloader.wrap do
59
52
  begin
53
+ unless listen
54
+ connection = self.class.connection
55
+ connection.exec_query('LISTEN "skiplock::jobs"')
56
+ if self.master
57
+ Job.flush
58
+ Cron.setup
59
+ end
60
+ listen = true
61
+ end
60
62
  if error
61
63
  unless connection.active?
62
64
  connection.reconnect!
63
65
  sleep(0.5)
64
66
  connection.exec_query('LISTEN "skiplock::jobs"')
65
- reloader_post { Job.flush } if self.master
67
+ Job.flush if self.master
66
68
  pg_exception_timestamp = nil
67
69
  next_schedule_at = Time.now.to_f
68
70
  end
@@ -75,7 +77,9 @@ module Skiplock
75
77
  result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
76
78
  end
77
79
  if result && result['running']
78
- reloader_post { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
80
+ @executor.post do
81
+ Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
82
+ end
79
83
  else
80
84
  next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
81
85
  end
@@ -103,7 +107,7 @@ module Skiplock
103
107
  Skiplock.logger.error(ex.backtrace.join("\n"))
104
108
  report_exception = true
105
109
  # if error is with database connection then only report if it persists longer than 1 minute
106
- if ex.is_a?(::PG::ConnectionBad)
110
+ if ex.is_a?(::PG::ConnectionBad) ||ex.is_a?(::PG::UnableToSend) || ex.message.include?('Bad file descriptor')
107
111
  report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
108
112
  pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
109
113
  end
@@ -113,8 +117,8 @@ module Skiplock
113
117
  end
114
118
  sleep(0.3)
115
119
  end
116
- connection.exec_query('UNLISTEN *')
117
120
  end
121
+ connection.exec_query('UNLISTEN *')
118
122
  end
119
123
 
120
124
  def wait(timeout)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: skiplock
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.21
4
+ version: 1.0.22
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tin Vo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-09-16 00:00:00.000000000 Z
11
+ date: 2021-09-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activejob