daemon_runner 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/Gemfile +2 -0
- data/README.md +23 -0
- data/daemon_runner.gemspec +1 -0
- data/examples/example1.rb +10 -2
- data/examples/example_semaphore.rb +11 -0
- data/examples/example_semaphore_release.rb +16 -0
- data/examples/run_consul.rb +7 -0
- data/lib/daemon_runner.rb +2 -0
- data/lib/daemon_runner/client.rb +36 -9
- data/lib/daemon_runner/retry_errors.rb +24 -0
- data/lib/daemon_runner/semaphore.rb +284 -0
- data/lib/daemon_runner/session.rb +27 -4
- metadata +21 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 53c26321dab0ca599634cdf709c5fbed7f0d7567
|
4
|
+
data.tar.gz: 054a65b2f31eac817e24fa25d9ad068e516efa1a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cca748f13e715c7170f77f6ccf83e91c0121e6352bb029f0bbf910d57f4d6d2a2f8dfbf7170d3d381f8aff3bc52ebbd7d685acd6036fe85ff94c781d08d7ffd1
|
7
|
+
data.tar.gz: 6b13b67488eb4704057eed3d715d456c80876aadad2fc2cb5dd60694da6f88d85eeefbbcedf5dca87f04b631f9c94ff19e0c2c1e41bb7bb180c37b5bae4f6795
|
data/.gitignore
CHANGED
data/Gemfile
CHANGED
@@ -3,6 +3,8 @@ source 'https://rubygems.org'
|
|
3
3
|
# Specify your gem's dependencies in daemon_runner.gemspec
|
4
4
|
gemspec
|
5
5
|
|
6
|
+
gem 'diplomat', github: 'WeAreFarmGeek/diplomat', ref: 'e64b0dec3b3616ded40bb64139a388dfc1a68232'
|
7
|
+
|
6
8
|
group :development do
|
7
9
|
gem 'bundler', '~> 1.7'
|
8
10
|
gem 'thor-scmversion', '= 1.7.0'
|
data/README.md
CHANGED
@@ -148,6 +148,29 @@ def schedule
|
|
148
148
|
end
|
149
149
|
```
|
150
150
|
|
151
|
+
### Retries
|
152
|
+
Simple interface to retry requests that are known to fails sometimes. To add a retry wrap the code like this:
|
153
|
+
|
154
|
+
```
|
155
|
+
DaemonRunner::RetryErrors.retry do
|
156
|
+
my_not_so_good_network_service_that_fails_sometimes
|
157
|
+
end
|
158
|
+
```
|
159
|
+
|
160
|
+
* `options` - Options hash to pass to `retry` (**optional**)
|
161
|
+
* :retries - Number of times to retry an exception (**optional**, _default_: 3)
|
162
|
+
* :exceptions - Array of exceptions to catch and retry (**optional**, _default_: `[Faraday::ClientError]`)
|
163
|
+
|
164
|
+
### Locking
|
165
|
+
Locking can be done either via an exclusive lock or a semaphore lock. The major difference is that with a semaphore lock you can define how many nodes can obtain the lock.
|
166
|
+
|
167
|
+
#### Exclusive Lock
|
168
|
+
**TBD**
|
169
|
+
|
170
|
+
#### Semaphore Lock
|
171
|
+
For an example of how to implement semaphore locking take a look at the [example](/examples/example_semaphore.rb)
|
172
|
+
|
173
|
+
|
151
174
|
## Development
|
152
175
|
|
153
176
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/daemon_runner.gemspec
CHANGED
@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
|
|
22
22
|
spec.add_dependency "mixlib-shellout", "~> 2.2"
|
23
23
|
spec.add_dependency "diplomat", "~> 1.0"
|
24
24
|
spec.add_dependency "rufus-scheduler", "~> 3.2"
|
25
|
+
spec.add_dependency "retryable", "~> 2.0"
|
25
26
|
|
26
27
|
spec.add_development_dependency "bundler", "~> 1.12"
|
27
28
|
spec.add_development_dependency "rake", "~> 10.0"
|
data/examples/example1.rb
CHANGED
@@ -37,12 +37,17 @@ class MyService
|
|
37
37
|
end
|
38
38
|
|
39
39
|
def run!(args)
|
40
|
+
sleep 10
|
40
41
|
name = args[0]
|
41
42
|
reason = args[1]
|
42
43
|
puts name
|
43
44
|
puts reason
|
44
45
|
name
|
45
46
|
end
|
47
|
+
|
48
|
+
def task_id
|
49
|
+
'long-running-task'
|
50
|
+
end
|
46
51
|
end
|
47
52
|
end
|
48
53
|
end
|
@@ -51,10 +56,13 @@ end
|
|
51
56
|
class MyService
|
52
57
|
class Tasks
|
53
58
|
class Quiz
|
54
|
-
def
|
55
|
-
|
59
|
+
def initialize
|
60
|
+
@task_id = 'long-running-task'
|
61
|
+
@schedule = [:interval, '30s']
|
56
62
|
end
|
63
|
+
|
57
64
|
def foo!(args)
|
65
|
+
sleep 10
|
58
66
|
puts 'Firing error'
|
59
67
|
sargs
|
60
68
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require_relative '../lib/daemon_runner'
|
4
|
+
require 'dev/consul'
|
5
|
+
|
6
|
+
@service = 'myreleaseservice'
|
7
|
+
@lock_count = 3
|
8
|
+
@locked = false
|
9
|
+
@lock_time = 10
|
10
|
+
|
11
|
+
DaemonRunner::Semaphore.lock(@service, @lock_count) do
|
12
|
+
@lock_time.downto(0).each do |i|
|
13
|
+
puts "Releasing lock in #{i} seconds"
|
14
|
+
sleep 1
|
15
|
+
end
|
16
|
+
end
|
data/lib/daemon_runner.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
require_relative 'daemon_runner/version'
|
2
2
|
require_relative 'daemon_runner/logger'
|
3
|
+
require_relative 'daemon_runner/retry_errors'
|
3
4
|
require_relative 'daemon_runner/client'
|
4
5
|
require_relative 'daemon_runner/shell_out'
|
5
6
|
require_relative 'daemon_runner/session'
|
7
|
+
require_relative 'daemon_runner/semaphore'
|
data/lib/daemon_runner/client.rb
CHANGED
@@ -17,15 +17,22 @@ module DaemonRunner
|
|
17
17
|
# @param [RuntimeError] error the body of the error
|
18
18
|
def scheduler.on_error(job, error)
|
19
19
|
error_sleep_time = job[:error_sleep_time]
|
20
|
+
on_error_release_lock = job[:on_error_release_lock]
|
20
21
|
logger = job[:logger]
|
21
22
|
task_id = job[:task_id]
|
22
23
|
|
24
|
+
mutex = @mutexes[task_id]
|
25
|
+
|
23
26
|
logger.error "#{task_id}: #{error}"
|
27
|
+
logger.debug "#{task_id}: #{error.backtrace.join("\n")}"
|
24
28
|
logger.debug "#{task_id}: Suspending #{task_id} for #{error_sleep_time} seconds"
|
25
|
-
|
29
|
+
|
30
|
+
# Unlock the job mutex if the job owns it and on_error_release_lock is true
|
31
|
+
mutex.unlock if on_error_release_lock && mutex.owned?
|
32
|
+
|
26
33
|
sleep error_sleep_time
|
34
|
+
|
27
35
|
logger.debug "#{task_id}: Resuming #{task_id}"
|
28
|
-
job.resume
|
29
36
|
end
|
30
37
|
end
|
31
38
|
|
@@ -58,6 +65,16 @@ module DaemonRunner
|
|
58
65
|
[:interval, loop_sleep_time]
|
59
66
|
end
|
60
67
|
|
68
|
+
# @return [Boolean] Whether to release a mutex lock if an error occurs in a task.
|
69
|
+
def on_error_release_lock
|
70
|
+
return @on_error_release_lock unless @on_error_release_lock.nil?
|
71
|
+
@on_error_release_lock = if options[:on_error_release_lock].nil?
|
72
|
+
true
|
73
|
+
else
|
74
|
+
options[:on_error_release_lock]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
61
78
|
# @return [Fixnum] Number of seconds to sleep between loop interactions.
|
62
79
|
def loop_sleep_time
|
63
80
|
return @loop_sleep_time unless @loop_sleep_time.nil?
|
@@ -123,11 +140,14 @@ module DaemonRunner
|
|
123
140
|
|
124
141
|
out[:method] = task[1]
|
125
142
|
|
126
|
-
|
127
|
-
out[:instance].
|
143
|
+
if out[:instance].instance_variable_defined?(:@task_id)
|
144
|
+
out[:task_id] = out[:instance].instance_variable_get(:@task_id)
|
145
|
+
elsif out[:instance].respond_to?(:task_id)
|
146
|
+
out[:task_id] = out[:instance].send(:task_id).to_s
|
128
147
|
else
|
129
|
-
"#{out[:class_name]}.#{out[:method]}"
|
148
|
+
out[:task_id] = "#{out[:class_name]}.#{out[:method]}"
|
130
149
|
end
|
150
|
+
|
131
151
|
raise ArgumentError, 'Invalid task id' if out[:task_id].nil? || out[:task_id].empty?
|
132
152
|
|
133
153
|
out[:args] = task[2..-1].flatten
|
@@ -140,10 +160,12 @@ module DaemonRunner
|
|
140
160
|
def parse_schedule(instance)
|
141
161
|
valid_types = [:in, :at, :every, :interval, :cron]
|
142
162
|
out = {}
|
143
|
-
|
144
|
-
instance.
|
163
|
+
if instance.instance_variable_defined?(:@schedule)
|
164
|
+
task_schedule = instance.instance_variable_get(:@schedule)
|
165
|
+
elsif instance.respond_to?(:schedule)
|
166
|
+
task_schedule = instance.send(:schedule)
|
145
167
|
else
|
146
|
-
schedule
|
168
|
+
task_schedule = schedule
|
147
169
|
end
|
148
170
|
|
149
171
|
raise ArgumentError, 'Malformed schedule definition, should be [TYPE, DURATION]' if task_schedule.length < 2
|
@@ -151,6 +173,7 @@ module DaemonRunner
|
|
151
173
|
|
152
174
|
out[:type] = task_schedule[0].to_sym
|
153
175
|
out[:schedule] = task_schedule[1]
|
176
|
+
out[:extra_opts] = task_schedule[2] if task_schedule.length > 2
|
154
177
|
out
|
155
178
|
end
|
156
179
|
|
@@ -171,12 +194,16 @@ module DaemonRunner
|
|
171
194
|
schedule_log_line += " with schedule: #{schedule[:schedule]}"
|
172
195
|
logger.debug schedule_log_line
|
173
196
|
|
174
|
-
|
197
|
+
opts = { :overlap => false, :job => true, :mutex => task_id }
|
198
|
+
opts.merge!(schedule[:extra_opts]) if schedule.key?(:extra_opts)
|
199
|
+
|
200
|
+
scheduler.send(schedule[:type], schedule[:schedule], opts) do |job|
|
175
201
|
log_line = "#{task_id}: Running #{class_name}.#{method}"
|
176
202
|
log_line += "(#{args})" unless args.empty?
|
177
203
|
logger.debug log_line
|
178
204
|
|
179
205
|
job[:error_sleep_time] = error_sleep_time
|
206
|
+
job[:on_error_release_lock] = on_error_release_lock
|
180
207
|
job[:logger] = logger
|
181
208
|
job[:task_id] = task_id
|
182
209
|
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'retryable'
|
2
|
+
|
3
|
+
module DaemonRunner
|
4
|
+
#
|
5
|
+
# Retry Errors
|
6
|
+
#
|
7
|
+
class RetryErrors
|
8
|
+
extend Logger
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def retry(retries: 3, exceptions: [Faraday::ClientError], &block)
|
12
|
+
properties = {
|
13
|
+
on: exceptions,
|
14
|
+
sleep: lambda { |c| 2**c * 0.3 },
|
15
|
+
tries: retries
|
16
|
+
}
|
17
|
+
Retryable.retryable(properties) do |retries, exception|
|
18
|
+
logger.warn "try #{retries} failed with exception: #{exception}" if retries > 0
|
19
|
+
block.call
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,284 @@
|
|
1
|
+
module DaemonRunner
|
2
|
+
#
|
3
|
+
# Manage semaphore locks with Consul
|
4
|
+
#
|
5
|
+
class Semaphore
|
6
|
+
include Logger
|
7
|
+
|
8
|
+
class << self
|
9
|
+
include Logger
|
10
|
+
|
11
|
+
# Acquire a lock with the current session
|
12
|
+
#
|
13
|
+
# @param limit [Integer] The number of nodes that can request the lock
|
14
|
+
# @see #initialize for extra `options`
|
15
|
+
# @return [DaemonRunner::Semaphore] instance of the semaphore class
|
16
|
+
#
|
17
|
+
def lock(name, limit = 3, **options)
|
18
|
+
options.merge!(name: name)
|
19
|
+
semaphore = Semaphore.new(options)
|
20
|
+
semaphore.lock
|
21
|
+
if block_given?
|
22
|
+
lock_thr = semaphore.renew
|
23
|
+
yield
|
24
|
+
end
|
25
|
+
semaphore
|
26
|
+
rescue Exception => e
|
27
|
+
logger.error e
|
28
|
+
logger.debug e.backtrace.join("\n")
|
29
|
+
raise
|
30
|
+
ensure
|
31
|
+
lock_thr.kill
|
32
|
+
semaphore.release
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
# The Consul session
|
37
|
+
attr_reader :session
|
38
|
+
|
39
|
+
# The current state of the semaphore
|
40
|
+
attr_reader :state
|
41
|
+
|
42
|
+
# The current semaphore members
|
43
|
+
attr_reader :members
|
44
|
+
|
45
|
+
# The current lock modify index
|
46
|
+
attr_reader :lock_modify_index
|
47
|
+
|
48
|
+
# The lock content
|
49
|
+
attr_reader :lock_content
|
50
|
+
|
51
|
+
# The Consul key prefix
|
52
|
+
attr_reader :prefix
|
53
|
+
|
54
|
+
# The number of nodes that can obtain a semaphore lock
|
55
|
+
attr_reader :limit
|
56
|
+
|
57
|
+
# @param name [String] The name of the session, it is also used in the `prefix`
|
58
|
+
# @param prefix [String|NilClass] The Consul Kv prefix
|
59
|
+
# @param lock [String|NilClass] The path to the lock file
|
60
|
+
def initialize(name:, prefix: nil, lock: nil, limit: 3)
|
61
|
+
create_session(name)
|
62
|
+
@prefix = prefix.nil? ? "service/#{name}/lock/" : prefix
|
63
|
+
@prefix += '/' unless @prefix.end_with?('/')
|
64
|
+
@lock = lock.nil? ? "#{@prefix}.lock" : lock
|
65
|
+
@lock_modify_index = nil
|
66
|
+
@lock_content = nil
|
67
|
+
@limit = set_limit(limit)
|
68
|
+
@reset = false
|
69
|
+
end
|
70
|
+
|
71
|
+
# Obtain a lock with the current session
|
72
|
+
#
|
73
|
+
# @return [Boolean] `true` if the lock was obtained
|
74
|
+
#
|
75
|
+
def lock
|
76
|
+
contender_key
|
77
|
+
semaphore_state
|
78
|
+
try_lock
|
79
|
+
end
|
80
|
+
|
81
|
+
# Renew lock watching for changes
|
82
|
+
# @return [Thread] Thread running a blocking call maintaining the lock state
|
83
|
+
#
|
84
|
+
def renew
|
85
|
+
thr = Thread.new do
|
86
|
+
loop do
|
87
|
+
if renew?
|
88
|
+
semaphore_state
|
89
|
+
try_lock
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
thr
|
94
|
+
end
|
95
|
+
|
96
|
+
# Release a lock with the current session
|
97
|
+
#
|
98
|
+
# @return [Boolean] `true` if the lock was released
|
99
|
+
#
|
100
|
+
def release
|
101
|
+
semaphore_state
|
102
|
+
try_release
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
def create_session(name)
|
107
|
+
::DaemonRunner::RetryErrors.retry(exceptions: [DaemonRunner::Session::CreateSessionError]) do
|
108
|
+
@session = Session.start(name, behavior: 'delete')
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def set_limit(new_limit)
|
113
|
+
if lock_exists?
|
114
|
+
if new_limit.to_i != @limit.to_i
|
115
|
+
logger.warn 'Limit in lockfile and @limit do not match using limit from lockfile'
|
116
|
+
end
|
117
|
+
@limit = lock_content['Limit']
|
118
|
+
else
|
119
|
+
@limit = new_limit
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
# Create a contender key
|
124
|
+
def contender_key(value = 'none')
|
125
|
+
if value.nil? || value.empty?
|
126
|
+
raise ArgumentError, 'Value cannot be empty or nil'
|
127
|
+
end
|
128
|
+
key = "#{prefix}/#{session.id}"
|
129
|
+
::DaemonRunner::RetryErrors.retry do
|
130
|
+
@contender_key = Diplomat::Lock.acquire(key, session.id, value)
|
131
|
+
end
|
132
|
+
@contender_key
|
133
|
+
end
|
134
|
+
|
135
|
+
# Get the current semaphore state by fetching all
|
136
|
+
# conterder keys and the lock key
|
137
|
+
def semaphore_state
|
138
|
+
options = { decode_values: true, recurse: true }
|
139
|
+
@state = Diplomat::Kv.get(prefix, options, :return)
|
140
|
+
decode_semaphore_state unless state.empty?
|
141
|
+
state
|
142
|
+
end
|
143
|
+
|
144
|
+
def try_lock
|
145
|
+
prune_members
|
146
|
+
do_update = add_self_to_holders
|
147
|
+
@reset = false
|
148
|
+
if do_update
|
149
|
+
format_holders
|
150
|
+
@locked = write_lock
|
151
|
+
end
|
152
|
+
log_lock_state
|
153
|
+
end
|
154
|
+
|
155
|
+
def try_release
|
156
|
+
do_update = remove_self_from_holders
|
157
|
+
if do_update
|
158
|
+
format_holders
|
159
|
+
@locked = !write_lock
|
160
|
+
end
|
161
|
+
DaemonRunner::Session.release(prefix)
|
162
|
+
session.destroy!
|
163
|
+
log_release_state
|
164
|
+
end
|
165
|
+
|
166
|
+
# Write a new lock file if the number of contenders is less than `limit`
|
167
|
+
# @return [Boolean] `true` if the lock was written succesfully
|
168
|
+
def write_lock
|
169
|
+
index = lock_modify_index.nil? ? 0 : lock_modify_index
|
170
|
+
value = generate_lockfile
|
171
|
+
return true if value == true
|
172
|
+
Diplomat::Kv.put(@lock, value, cas: index)
|
173
|
+
end
|
174
|
+
|
175
|
+
# Start a blocking query on the prefix, if there are changes
|
176
|
+
# we need to try to obtain the lock again.
|
177
|
+
#
|
178
|
+
# @return [Boolean] `true` if there are changes,
|
179
|
+
# `false` if the request has timed out
|
180
|
+
def renew?
|
181
|
+
logger.debug("Watching Consul #{prefix} for changes")
|
182
|
+
options = { recurse: true }
|
183
|
+
changes = Diplomat::Kv.get(prefix, options, :wait, :wait)
|
184
|
+
logger.info("Changes on #{prefix} detected") if changes
|
185
|
+
changes
|
186
|
+
rescue StandardError => e
|
187
|
+
logger.error(e)
|
188
|
+
end
|
189
|
+
|
190
|
+
private
|
191
|
+
|
192
|
+
# Decode raw response from Consul
|
193
|
+
# Set `@lock_modify_index`, `@lock_content`, and `@members`
|
194
|
+
# @returns [Array] List of members
|
195
|
+
def decode_semaphore_state
|
196
|
+
lock_key = state.find { |k| k['Key'] == @lock }
|
197
|
+
member_keys = state.delete_if { |k| k['Key'] == @lock }
|
198
|
+
member_keys.map! { |k| k['Key'] }
|
199
|
+
|
200
|
+
unless lock_key.nil?
|
201
|
+
@lock_modify_index = lock_key['ModifyIndex']
|
202
|
+
@lock_content = JSON.parse(lock_key['Value'])
|
203
|
+
end
|
204
|
+
@members = member_keys.map { |k| k.split('/')[-1] }
|
205
|
+
end
|
206
|
+
|
207
|
+
# Returns current state of lockfile
|
208
|
+
def lock_exists?
|
209
|
+
(!lock_modify_index.nil? && !lock_content.nil?) && !@reset
|
210
|
+
end
|
211
|
+
|
212
|
+
# Get the active members from the lock file, removing any _dead_ members.
|
213
|
+
# This is accomplished by using the contenders keys(`@members`) to get the
|
214
|
+
# list of all alive members. So we can easily remove any nodes that don't
|
215
|
+
# appear in that list.
|
216
|
+
def prune_members
|
217
|
+
@holders = if lock_exists?
|
218
|
+
holders = lock_content['Holders']
|
219
|
+
return @holders = [] if holders.nil?
|
220
|
+
holders = holders.keys
|
221
|
+
holders & members
|
222
|
+
else
|
223
|
+
[]
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
# Add our session.id to the holders list if holders is less than limit
|
228
|
+
def add_self_to_holders
|
229
|
+
@holders.uniq!
|
230
|
+
@reset = true if @holders.length == 0
|
231
|
+
return true if @holders.include? session.id
|
232
|
+
if @holders.length < limit
|
233
|
+
@holders << session.id
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
237
|
+
# Remove our session.id from the holders list
|
238
|
+
def remove_self_from_holders
|
239
|
+
return unless lock_exists?
|
240
|
+
@holders = lock_content['Holders']
|
241
|
+
@holders = @holders.keys
|
242
|
+
@holders.delete(session.id)
|
243
|
+
@holders
|
244
|
+
end
|
245
|
+
|
246
|
+
# Format the list of holders for the lock file
|
247
|
+
def format_holders
|
248
|
+
@holders.uniq!
|
249
|
+
@holders.sort!
|
250
|
+
holders = {}
|
251
|
+
logger.debug "Holders are: #{@holders.join(',')}"
|
252
|
+
@holders.map { |m| holders[m] = true }
|
253
|
+
@holders = holders
|
254
|
+
end
|
255
|
+
|
256
|
+
# Generate JSON formatted lockfile content, only if the content has changed
|
257
|
+
def generate_lockfile
|
258
|
+
if lock_exists? && lock_content['Holders'] == @holders
|
259
|
+
logger.info 'Holders are unchanged, not updating'
|
260
|
+
return true
|
261
|
+
end
|
262
|
+
lockfile_format = {
|
263
|
+
'Limit' => limit,
|
264
|
+
'Holders' => @holders
|
265
|
+
}
|
266
|
+
JSON.generate(lockfile_format)
|
267
|
+
end
|
268
|
+
|
269
|
+
def log_lock_state
|
270
|
+
log_lock(locked: @locked)
|
271
|
+
end
|
272
|
+
|
273
|
+
def log_release_state
|
274
|
+
log_lock(locked: !@locked, end_string: 'released')
|
275
|
+
end
|
276
|
+
|
277
|
+
def log_lock(locked: true, end_string: 'obtained')
|
278
|
+
msg = 'Lock %{text} %{end_string}'
|
279
|
+
text = locked == true ? 'succesfully' : 'could not be'
|
280
|
+
msg = msg % { text: text, end_string: end_string }
|
281
|
+
logger.info msg
|
282
|
+
end
|
283
|
+
end
|
284
|
+
end
|
@@ -5,16 +5,21 @@ module DaemonRunner
|
|
5
5
|
# Manage distributed locks with Consul
|
6
6
|
#
|
7
7
|
class Session
|
8
|
+
class SessionError < RuntimeError; end
|
9
|
+
class CreateSessionError < SessionError; end
|
10
|
+
|
8
11
|
include Logger
|
9
12
|
|
10
13
|
class << self
|
11
14
|
attr_reader :session
|
12
15
|
|
13
16
|
def start(name, **options)
|
14
|
-
@session
|
17
|
+
@session = Session.new(name, options).renew!
|
18
|
+
raise CreateSessionError, 'Failed to create session' if @session == false
|
19
|
+
@session.verify_session
|
20
|
+
@session
|
15
21
|
end
|
16
22
|
|
17
|
-
|
18
23
|
# Acquire a lock with the current session, or initialize a new session
|
19
24
|
#
|
20
25
|
# @param path [String] A path in the Consul key-value space to lock
|
@@ -22,7 +27,7 @@ module DaemonRunner
|
|
22
27
|
# @return [Boolean] `true` if the lock was acquired
|
23
28
|
#
|
24
29
|
def lock(path)
|
25
|
-
Diplomat::Lock.
|
30
|
+
Diplomat::Lock.wait_to_acquire(path, session.id)
|
26
31
|
end
|
27
32
|
|
28
33
|
# Release a lock held by the current session
|
@@ -107,6 +112,19 @@ module DaemonRunner
|
|
107
112
|
Diplomat::Session.destroy(id)
|
108
113
|
end
|
109
114
|
|
115
|
+
# Verify wheather the session exists after a period of time
|
116
|
+
def verify_session(wait_time = 2)
|
117
|
+
logger.info(" - Wait until Consul session #{id} exists")
|
118
|
+
wait_time.times do
|
119
|
+
exists = session_exist?
|
120
|
+
raise CreateSessionError, 'Error creating session' unless exists
|
121
|
+
sleep 1
|
122
|
+
end
|
123
|
+
logger.info(" - Found Consul session #{id}")
|
124
|
+
rescue CreateSessionError
|
125
|
+
init
|
126
|
+
end
|
127
|
+
|
110
128
|
private
|
111
129
|
|
112
130
|
# Initialize a session and store it's ID
|
@@ -118,8 +136,13 @@ module DaemonRunner
|
|
118
136
|
:LockDelay => "#{delay}s",
|
119
137
|
:Behavior => behavior
|
120
138
|
)
|
121
|
-
|
122
139
|
logger.info(" - Initialized a Consul session #{id}")
|
123
140
|
end
|
141
|
+
|
142
|
+
# Does the session exist
|
143
|
+
def session_exist?
|
144
|
+
sessions = Diplomat::Session.list
|
145
|
+
sessions.any? { |s| s['ID'] == id }
|
146
|
+
end
|
124
147
|
end
|
125
148
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: daemon_runner
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Andrew Thompson
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-11-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logging
|
@@ -66,6 +66,20 @@ dependencies:
|
|
66
66
|
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '3.2'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: retryable
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '2.0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '2.0'
|
69
83
|
- !ruby/object:Gem::Dependency
|
70
84
|
name: bundler
|
71
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -156,9 +170,14 @@ files:
|
|
156
170
|
- bin/setup
|
157
171
|
- daemon_runner.gemspec
|
158
172
|
- examples/example1.rb
|
173
|
+
- examples/example_semaphore.rb
|
174
|
+
- examples/example_semaphore_release.rb
|
175
|
+
- examples/run_consul.rb
|
159
176
|
- lib/daemon_runner.rb
|
160
177
|
- lib/daemon_runner/client.rb
|
161
178
|
- lib/daemon_runner/logger.rb
|
179
|
+
- lib/daemon_runner/retry_errors.rb
|
180
|
+
- lib/daemon_runner/semaphore.rb
|
162
181
|
- lib/daemon_runner/session.rb
|
163
182
|
- lib/daemon_runner/shell_out.rb
|
164
183
|
- lib/daemon_runner/version.rb
|