resque-durable 1.2.0 → 3.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/lib/resque/durable/background_heartbeat.rb +82 -0
- data/lib/resque/durable/guid.rb +3 -16
- data/lib/resque/durable/queue_audit.rb +30 -10
- data/lib/resque/durable.rb +34 -3
- metadata +39 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: '05487046606654ab0ce5b6f7ba068282a592fedb7acc5a601c212e6bba39ef17'
|
4
|
+
data.tar.gz: ed369c7e932fd3e9a68179a01d7fa7a898396f3a3415618769ca731bf16ae3dc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7ef77a5685754435423ca5997b46e4747e2d5cf1330d1056c47b40591ccac0e54156bdcad356831d1de867b2ddf6134513994f0cf0d151d2229aac0e10b423ba
|
7
|
+
data.tar.gz: 458f943f604c15ba58bad0805e22fbcc69cfd6b2559e09f33aca78869356af0633635f92cd47718ab7392c3c43063dd92400cf233bc642bd88e24be8284f1514
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'thread'
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Durable
|
5
|
+
|
6
|
+
# Creates a background thread to regularly heartbeat the queue audit.
|
7
|
+
class BackgroundHeartbeat
|
8
|
+
|
9
|
+
def initialize(queue_audit, interval)
|
10
|
+
@queue_audit = queue_audit
|
11
|
+
@last_timeout = nil
|
12
|
+
@interval = interval
|
13
|
+
@mutex = Mutex.new
|
14
|
+
@stop = false
|
15
|
+
@thread = nil
|
16
|
+
end
|
17
|
+
|
18
|
+
class << self
|
19
|
+
# only a separate method for easy stubbing
|
20
|
+
def exit_now!
|
21
|
+
abort
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def with_heartbeat
|
26
|
+
start!
|
27
|
+
yield
|
28
|
+
ensure
|
29
|
+
stop_and_wait!
|
30
|
+
end
|
31
|
+
|
32
|
+
def heartbeat!
|
33
|
+
@last_timeout ||= @queue_audit.timeout_at
|
34
|
+
@last_timeout = @queue_audit.optimistic_heartbeat!(@last_timeout)
|
35
|
+
rescue StandardError => e
|
36
|
+
@queue_audit.logger.error("Exception in BackgroundHeartbeat thread: #{e.class.name}: #{e.message}")
|
37
|
+
self.class.exit_now!
|
38
|
+
ensure
|
39
|
+
ActiveRecord::Base.clear_active_connections!
|
40
|
+
end
|
41
|
+
|
42
|
+
def start!
|
43
|
+
raise "Thread is already running!" if @thread
|
44
|
+
@stop = false
|
45
|
+
|
46
|
+
# Perform immediately to reduce heartbeat race condition opportunities
|
47
|
+
heartbeat!
|
48
|
+
|
49
|
+
@thread = Thread.new do
|
50
|
+
while !@stop
|
51
|
+
heartbeat!
|
52
|
+
|
53
|
+
@mutex.synchronize do
|
54
|
+
@mutex.sleep(@interval)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def stop_and_wait!
|
61
|
+
return unless @thread
|
62
|
+
# Prevent deadlock if called by the `heartbeat` thread, which can't wait for itself to die.
|
63
|
+
return signal_stop! if @thread == Thread.current
|
64
|
+
while @thread.alive?
|
65
|
+
signal_stop!
|
66
|
+
sleep 0.01
|
67
|
+
end
|
68
|
+
@thread.join
|
69
|
+
@thread = nil
|
70
|
+
end
|
71
|
+
|
72
|
+
# Signal the `heartbeat` thread to stop looping immediately. Safe to be call from any thread.
|
73
|
+
def signal_stop!
|
74
|
+
return unless @thread
|
75
|
+
@mutex.synchronize do
|
76
|
+
@stop = true
|
77
|
+
@thread.wakeup
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/resque/durable/guid.rb
CHANGED
@@ -1,24 +1,11 @@
|
|
1
|
+
require 'uuidtools'
|
2
|
+
|
1
3
|
module Resque
|
2
4
|
module Durable
|
3
5
|
module GUID
|
4
|
-
|
5
6
|
def self.generate
|
6
|
-
|
7
|
-
Process.pid,
|
8
|
-
Time.now.to_i,
|
9
|
-
increment_counter
|
10
|
-
].join('/')
|
11
|
-
end
|
12
|
-
|
13
|
-
def self.hostname
|
14
|
-
@hostname ||= `hostname`.chomp
|
7
|
+
UUIDTools::UUID.random_create.to_s
|
15
8
|
end
|
16
|
-
|
17
|
-
def self.increment_counter
|
18
|
-
@counter ||= 0
|
19
|
-
@counter += 1
|
20
|
-
end
|
21
|
-
|
22
9
|
end
|
23
10
|
end
|
24
11
|
end
|
@@ -1,9 +1,11 @@
|
|
1
1
|
require 'active_record'
|
2
|
-
require 'active_support/core_ext
|
2
|
+
require 'active_support/core_ext'
|
3
3
|
|
4
4
|
module Resque
|
5
5
|
module Durable
|
6
6
|
class QueueAudit < ActiveRecord::Base
|
7
|
+
JobCollision = Class.new(StandardError)
|
8
|
+
|
7
9
|
self.table_name = :durable_queue_audits
|
8
10
|
# id
|
9
11
|
# enqueued_id
|
@@ -19,19 +21,18 @@ module Resque
|
|
19
21
|
|
20
22
|
validates_length_of :payload_before_type_cast, :in => 1..5000
|
21
23
|
|
22
|
-
validates_inclusion_of :duration, :in => 1.minute..3.hours
|
24
|
+
validates_inclusion_of :duration, :in => 1.minute.to_i..3.hours.to_i
|
23
25
|
|
24
|
-
scope :older_than,
|
25
|
-
{ :conditions => [ 'created_at < ?', date ] }
|
26
|
-
}
|
26
|
+
scope :older_than, ->(date) { where('created_at < ?', date) }
|
27
27
|
|
28
|
-
scope :failed,
|
29
|
-
|
28
|
+
scope :failed, -> {
|
29
|
+
where(completed_at: nil)
|
30
|
+
.where('timeout_at < ?', Time.now.utc)
|
31
|
+
.order('timeout_at asc')
|
32
|
+
.limit(500)
|
30
33
|
}
|
31
34
|
|
32
|
-
scope :complete,
|
33
|
-
{ :conditions => 'completed_at is not null' }
|
34
|
-
}
|
35
|
+
scope :complete, -> { where('completed_at is not null') }
|
35
36
|
|
36
37
|
module Recovery
|
37
38
|
|
@@ -89,6 +90,18 @@ module Resque
|
|
89
90
|
update_attribute(:timeout_at, Time.now.utc + duration)
|
90
91
|
end
|
91
92
|
|
93
|
+
# Bumps the `timeout_at` column, but raises a `JobCollision` exception if
|
94
|
+
# another process has changed the value, indicating we may have multiple
|
95
|
+
# workers processing the same job.
|
96
|
+
def optimistic_heartbeat!(last_timeout_at)
|
97
|
+
next_timeout_at = Time.now.utc + duration
|
98
|
+
nrows = self.class.
|
99
|
+
where(id: id, timeout_at: last_timeout_at).
|
100
|
+
update_all(timeout_at: next_timeout_at)
|
101
|
+
raise JobCollision.new unless nrows == 1
|
102
|
+
next_timeout_at
|
103
|
+
end
|
104
|
+
|
92
105
|
def fail!
|
93
106
|
update_attribute(:timeout_at, Time.now.utc)
|
94
107
|
end
|
@@ -118,6 +131,13 @@ module Resque
|
|
118
131
|
(enqueue_count ** 3).minutes
|
119
132
|
end
|
120
133
|
|
134
|
+
def reset_backoff!(timeout_at = Time.now.utc)
|
135
|
+
# Set timeout_at = Time.now and enqueue_count = 1 so
|
136
|
+
# the job can be picked up by the Durable Monitor asap.
|
137
|
+
self.timeout_at = timeout_at
|
138
|
+
self.enqueue_count = 1
|
139
|
+
save!
|
140
|
+
end
|
121
141
|
end
|
122
142
|
end
|
123
143
|
end
|
data/lib/resque/durable.rb
CHANGED
@@ -3,11 +3,20 @@ module Resque
|
|
3
3
|
autoload :GUID, 'resque/durable/guid'
|
4
4
|
autoload :Monitor, 'resque/durable/monitor'
|
5
5
|
autoload :QueueAudit, 'resque/durable/queue_audit'
|
6
|
+
autoload :BackgroundHeartbeat, 'resque/durable/background_heartbeat'
|
6
7
|
|
7
8
|
def self.extended(base)
|
9
|
+
# The duration since the last heartbeat that the monitor will wait before
|
10
|
+
# re-enqueing the job.
|
8
11
|
base.cattr_accessor :job_timeout
|
9
12
|
base.job_timeout = 10.minutes
|
10
13
|
|
14
|
+
# How frequently a background thread will optimistically heartbeat the
|
15
|
+
# QueueAudit. Value must be smaller than job_timeout. Currently opt-in.
|
16
|
+
#
|
17
|
+
# Recommended value: `15.seconds`
|
18
|
+
base.cattr_accessor :background_heartbeat_interval
|
19
|
+
|
11
20
|
base.cattr_accessor :auditor
|
12
21
|
base.auditor = QueueAudit
|
13
22
|
end
|
@@ -46,13 +55,35 @@ module Resque
|
|
46
55
|
|
47
56
|
def around_perform_manage_audit(*args)
|
48
57
|
if a = audit(args)
|
49
|
-
a.heartbeat!
|
50
58
|
return if a.complete?
|
51
|
-
|
52
|
-
|
59
|
+
if background_heartbeat_interval
|
60
|
+
raise "background_heartbeat_interval (#{background_heartbeat_interval.inspect}) be smaller than job_timeout (#{job_timeout.inspect})" if background_heartbeat_interval >= job_timeout
|
61
|
+
BackgroundHeartbeat.new(audit(args), background_heartbeat_interval).with_heartbeat do
|
62
|
+
yield
|
63
|
+
end
|
64
|
+
else
|
65
|
+
a.heartbeat!
|
66
|
+
yield
|
67
|
+
end
|
68
|
+
|
69
|
+
if requeue_immediately
|
70
|
+
a.reset_backoff!
|
71
|
+
else
|
72
|
+
a.complete!
|
73
|
+
end
|
53
74
|
else
|
54
75
|
yield
|
55
76
|
end
|
77
|
+
ensure
|
78
|
+
@requeue_immediately = false
|
79
|
+
end
|
80
|
+
|
81
|
+
def requeue_immediately
|
82
|
+
@requeue_immediately
|
83
|
+
end
|
84
|
+
|
85
|
+
def requeue_immediately!
|
86
|
+
@requeue_immediately = true
|
56
87
|
end
|
57
88
|
|
58
89
|
def build_audit(args)
|
metadata
CHANGED
@@ -1,18 +1,46 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-durable
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 3.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eric Chapweske
|
8
8
|
- Ben Osheroff
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2022-07-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activerecord
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '4.2'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '4.2'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: resque
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '1.27'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '1.27'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: uuidtools
|
16
44
|
requirement: !ruby/object:Gem::Requirement
|
17
45
|
requirements:
|
18
46
|
- - ">="
|
@@ -25,13 +53,14 @@ dependencies:
|
|
25
53
|
- - ">="
|
26
54
|
- !ruby/object:Gem::Version
|
27
55
|
version: '0'
|
28
|
-
description:
|
29
|
-
email:
|
56
|
+
description:
|
57
|
+
email:
|
30
58
|
executables: []
|
31
59
|
extensions: []
|
32
60
|
extra_rdoc_files: []
|
33
61
|
files:
|
34
62
|
- lib/resque/durable.rb
|
63
|
+
- lib/resque/durable/background_heartbeat.rb
|
35
64
|
- lib/resque/durable/guid.rb
|
36
65
|
- lib/resque/durable/monitor.rb
|
37
66
|
- lib/resque/durable/queue_audit.rb
|
@@ -39,7 +68,7 @@ homepage: https://github.com/zendesk/resque-durable
|
|
39
68
|
licenses:
|
40
69
|
- MIT
|
41
70
|
metadata: {}
|
42
|
-
post_install_message:
|
71
|
+
post_install_message:
|
43
72
|
rdoc_options: []
|
44
73
|
require_paths:
|
45
74
|
- lib
|
@@ -47,16 +76,16 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
47
76
|
requirements:
|
48
77
|
- - ">="
|
49
78
|
- !ruby/object:Gem::Version
|
50
|
-
version: '
|
79
|
+
version: '2.4'
|
51
80
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
52
81
|
requirements:
|
53
82
|
- - ">="
|
54
83
|
- !ruby/object:Gem::Version
|
55
84
|
version: '0'
|
56
85
|
requirements: []
|
57
|
-
rubyforge_project:
|
58
|
-
rubygems_version: 2.
|
59
|
-
signing_key:
|
86
|
+
rubyforge_project:
|
87
|
+
rubygems_version: 2.7.6.2
|
88
|
+
signing_key:
|
60
89
|
specification_version: 4
|
61
90
|
summary: Resque queue backed by database audits, with automatic retry
|
62
91
|
test_files: []
|