skiplock 1.0.22 → 1.0.23

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3b99466d7f1848f9dcd750a4dceb6a39d7124707e941c74b6c75f0b3a57944ff
4
- data.tar.gz: 3f3ed1c9327d41e675d5c170590b52fe32f1facc5b57667dba513e5324e6ee20
3
+ metadata.gz: 275e1ddd1a59648e86378889e2e2cc6a4e22943a3fb4211209f20a674744b1b3
4
+ data.tar.gz: 7327f5893f36cc4b05163cfd0df47992344b2ca6b97c2ef10ca4f00b219a9ac9
5
5
  SHA512:
6
- metadata.gz: '0129296507b0479ba28a6e4e0bce810340f0ff2559004da1385cf02b6d6023f7be5cb1543f3fb24cebdeb579d6be6ed1e5d7d72d43245de5374926607b23d7df'
7
- data.tar.gz: 163faf212d06b6d7f8045eb347ccab77de5b16bfe77fe8780a5d73176ce3c5804cc25cbca8f962d92bc5dfccc1ada223d03fc0b443d0df48125a12bf4ec7ddeb
6
+ metadata.gz: e9b8afd563ec4ff46838e73ae031a75a69142ee392a05d0e5b612008153b10a6a62a931e22643e817722d0017548573f9cafed6ba943f41da3cf8d0872d76481
7
+ data.tar.gz: 8b04173649d9e4a21edb9d47e23d37d5c8d0b08ba8658f46c0fd8341270abf1148a2131e3f57251421caf024d5ce8d9bfb35a648ee9a904f999d6b21bf0fe5ec
data/README.md CHANGED
@@ -85,6 +85,7 @@ The library is quite small compared to other PostgreSQL job queues (eg. *delay_j
85
85
  Usage: skiplock [options]
86
86
  -e, --environment STRING Rails environment
87
87
  -l, --logfile STRING Log filename
88
+ -L, --loglevel STRING Log level (debug, info, warn, error, fatal, unknown)
88
89
  -s, --graceful-shutdown NUM Number of seconds to wait for graceful shutdown
89
90
  -r, --max-retries NUM Number of maxixum retries
90
91
  -t, --max-threads NUM Number of maximum threads
@@ -10,7 +10,7 @@ module Skiplock
10
10
  elsif @config[:extensions].is_a?(Array)
11
11
  @config[:extensions].each { |n| n.constantize.__send__(:extend, Skiplock::Extension) if n.safe_constantize }
12
12
  end
13
- async if (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0)
13
+ (caller.any?{ |l| l =~ %r{/rack/} } && @config[:workers] == 0) ? async : Cron.setup
14
14
  end
15
15
 
16
16
  def async
@@ -18,6 +18,7 @@ module Skiplock
18
18
  configure
19
19
  Worker.cleanup(@hostname)
20
20
  @worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname)
21
+ Cron.setup if @worker.master
21
22
  @worker.start(**@config)
22
23
  at_exit { @worker.shutdown }
23
24
  rescue Exception => ex
@@ -42,7 +43,8 @@ module Skiplock
42
43
  ActiveRecord::Base.connection.disconnect! if @config[:workers] > 1
43
44
  (@config[:workers] - 1).times do |n|
44
45
  fork do
45
- sleep 1
46
+ sleep(0.25*n + 1)
47
+ ActiveRecord::Base.establish_connection
46
48
  worker = Worker.generate(capacity: @config[:max_threads], hostname: @hostname, master: false)
47
49
  worker.start(worker_num: n + 1, **@config)
48
50
  loop do
@@ -90,7 +92,7 @@ module Skiplock
90
92
  end
91
93
 
92
94
  def configure
93
- @hostname = Socket.gethostname
95
+ @hostname = Socket.ip_address_list.reject(&:ipv4_loopback?).reject(&:ipv6?).map(&:ip_address).join('|')
94
96
  @config.transform_values! {|v| v.is_a?(String) ? v.downcase : v}
95
97
  @config[:graceful_shutdown] = 300 if @config[:graceful_shutdown] > 300
96
98
  @config[:graceful_shutdown] = nil if @config[:graceful_shutdown] < 0
@@ -145,6 +147,7 @@ module Skiplock
145
147
  end
146
148
  if @config[:standalone]
147
149
  Rails.logger.reopen('/dev/null') rescue Rails.logger.reopen('NUL') # supports Windows NUL device
150
+ Rails.logger.level = @logger.level
148
151
  Rails.logger.extend(ActiveSupport::Logger.broadcast(@logger))
149
152
  end
150
153
  rescue Exception => ex
@@ -1,4 +1,4 @@
1
1
  module Skiplock
2
- VERSION = Version = '1.0.22'
2
+ VERSION = Version = '1.0.23'
3
3
  end
4
4
 
@@ -29,66 +29,67 @@ module Skiplock
29
29
  def start(worker_num: 0, **config)
30
30
  @num = worker_num
31
31
  @config = config
32
+ @pg_config = ActiveRecord::Base.connection.raw_connection.conninfo_hash.compact
32
33
  @queues_order_query = @config[:queues].map { |q,v| "WHEN queue_name = '#{q}' THEN #{v}" }.join(' ') if @config[:queues].is_a?(Hash) && @config[:queues].count > 0
33
34
  @running = true
34
- @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :discard)
35
+ @executor = Concurrent::ThreadPoolExecutor.new(min_threads: @config[:min_threads] + 1, max_threads: @config[:max_threads] + 1, max_queue: @config[:max_threads] + 1, idletime: 60, auto_terminate: true, fallback_policy: :abort)
35
36
  @executor.post { run }
36
- Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]") if @config[:standalone]
37
+ if @config[:standalone]
38
+ Process.setproctitle("skiplock: #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2} [#{Rails.application.class.name.deconstantize.downcase}:#{Rails.env}]")
39
+ ActiveRecord::Base.connection.throw_away!
40
+ end
37
41
  end
38
42
 
39
43
  private
40
44
 
45
+ def establish_connection
46
+ map = ::PG::TypeMapByOid.new
47
+ map.add_coder(::PG::TextDecoder::Boolean.new(oid: 16, name: 'bool'))
48
+ map.add_coder(::PG::TextDecoder::Integer.new(oid: 20, name: 'int8'))
49
+ map.add_coder(::PG::TextDecoder::Integer.new(oid: 21, name: 'int2'))
50
+ map.add_coder(::PG::TextDecoder::Integer.new(oid: 23, name: 'int4'))
51
+ map.add_coder(::PG::TextDecoder::TimestampUtc.new(oid: 1114, name: 'timestamp'))
52
+ map.add_coder(::PG::TextDecoder::String.new(oid: 2950, name: 'uuid'))
53
+ map.add_coder(::PG::TextDecoder::JSON.new(oid: 3802, name: 'jsonb'))
54
+ @connection = ::PG.connect(@pg_config)
55
+ @connection.type_map_for_results = map
56
+ @connection.exec('LISTEN "skiplock::jobs"').clear
57
+ end
58
+
41
59
  def run
42
60
  sleep 3
43
61
  Skiplock.logger.info "[Skiplock] Starting in #{@config[:standalone] ? 'standalone' : 'async'} mode (PID: #{self.pid}) with #{@config[:max_threads]} max threads as #{self.master ? 'master' : 'cluster'} worker#{(' ' + @num.to_s) if @num > 0 && @config[:workers] > 2}..."
44
- connection = nil
45
- error = false
46
- listen = false
47
62
  next_schedule_at = Time.now.to_f
48
63
  pg_exception_timestamp = nil
49
64
  timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
50
65
  while @running
51
66
  Rails.application.reloader.wrap do
52
67
  begin
53
- unless listen
54
- connection = self.class.connection
55
- connection.exec_query('LISTEN "skiplock::jobs"')
56
- if self.master
57
- Job.flush
58
- Cron.setup
59
- end
60
- listen = true
61
- end
62
- if error
63
- unless connection.active?
64
- connection.reconnect!
65
- sleep(0.5)
66
- connection.exec_query('LISTEN "skiplock::jobs"')
67
- Job.flush if self.master
68
- pg_exception_timestamp = nil
69
- next_schedule_at = Time.now.to_f
70
- end
71
- error = false
68
+ if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
69
+ establish_connection
70
+ @executor.post { Rails.application.executor.wrap { Job.flush } } if self.master
71
+ pg_exception_timestamp = nil
72
+ next_schedule_at = Time.now.to_f
72
73
  end
73
74
  if Time.now.to_f >= next_schedule_at && @executor.remaining_capacity > 1 # reserves 1 slot in queue for Job.flush in case of pg_connection error
74
75
  result = nil
75
- connection.transaction do
76
- result = connection.select_all("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1").first
77
- result = connection.select_all("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *").first if result && result['scheduled_at'].to_f <= Time.now.to_f
76
+ @connection.transaction do |conn|
77
+ conn.exec("SELECT id, running, scheduled_at FROM skiplock.jobs WHERE running = FALSE AND expired_at IS NULL AND finished_at IS NULL ORDER BY scheduled_at ASC NULLS FIRST,#{@queues_order_query ? ' CASE ' + @queues_order_query + ' ELSE NULL END ASC NULLS LAST,' : ''} priority ASC NULLS LAST, created_at ASC FOR UPDATE SKIP LOCKED LIMIT 1") do |r|
78
+ result = r.first
79
+ conn.exec("UPDATE skiplock.jobs SET running = TRUE, worker_id = '#{self.id}', updated_at = NOW() WHERE id = '#{result['id']}' RETURNING *") { |r| result = r.first } if result && result['scheduled_at'].to_f <= Time.now.to_f
80
+ end
78
81
  end
79
82
  if result && result['running']
80
- @executor.post do
81
- Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) }
82
- end
83
+ @executor.post { Rails.application.executor.wrap { Job.instantiate(result).execute(purge_completion: @config[:purge_completion], max_retries: @config[:max_retries]) } }
83
84
  else
84
85
  next_schedule_at = (result ? result['scheduled_at'].to_f : Float::INFINITY)
85
86
  end
86
87
  end
87
88
  job_notifications = []
88
- connection.raw_connection.wait_for_notify(0.2) do |channel, pid, payload|
89
+ @connection.wait_for_notify(0.2) do |channel, pid, payload|
89
90
  job_notifications << payload if payload
90
91
  loop do
91
- payload = connection.raw_connection.notifies
92
+ payload = @connection.notifies
92
93
  break unless @running && payload
93
94
  job_notifications << payload[:extra]
94
95
  end
@@ -99,29 +100,31 @@ module Skiplock
99
100
  end
100
101
  end
101
102
  if Process.clock_gettime(Process::CLOCK_MONOTONIC) - timestamp > 60
102
- connection.exec_query("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'")
103
+ @connection.exec("UPDATE skiplock.workers SET updated_at = NOW() WHERE id = '#{self.id}'").clear
103
104
  timestamp = Process.clock_gettime(Process::CLOCK_MONOTONIC)
104
105
  end
105
106
  rescue Exception => ex
106
- Skiplock.logger.error(ex.to_s)
107
- Skiplock.logger.error(ex.backtrace.join("\n"))
108
107
  report_exception = true
109
108
  # if error is with database connection then only report if it persists longer than 1 minute
110
- if ex.is_a?(::PG::ConnectionBad) ||ex.is_a?(::PG::UnableToSend) || ex.message.include?('Bad file descriptor')
109
+ if @connection.nil? || @connection.status != ::PG::CONNECTION_OK
111
110
  report_exception = false if pg_exception_timestamp.nil? || Process.clock_gettime(Process::CLOCK_MONOTONIC) - pg_exception_timestamp <= 60
112
111
  pg_exception_timestamp ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
113
112
  end
114
- Skiplock.on_errors.each { |p| p.call(ex) } if report_exception
115
- error = true
113
+ if report_exception
114
+ Skiplock.logger.error(ex.to_s)
115
+ Skiplock.logger.error(ex.backtrace.join("\n"))
116
+ Skiplock.on_errors.each { |p| p.call(ex) }
117
+ end
116
118
  wait(5)
117
119
  end
118
120
  sleep(0.3)
119
121
  end
120
122
  end
121
- connection.exec_query('UNLISTEN *')
123
+ ensure
124
+ @connection.close if @connection && !@connection.finished?
122
125
  end
123
126
 
124
- def wait(timeout)
127
+ def wait(timeout = 1)
125
128
  t = Process.clock_gettime(Process::CLOCK_MONOTONIC)
126
129
  while @running
127
130
  sleep(0.5)
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: skiplock
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.22
4
+ version: 1.0.23
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tin Vo