zk 1.4.2 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.dotfiles/ctags_paths +1 -0
- data/.dotfiles/rspec-logging +2 -2
- data/.gitignore +1 -0
- data/Gemfile +9 -3
- data/Guardfile +36 -0
- data/README.markdown +21 -18
- data/RELEASES.markdown +10 -0
- data/Rakefile +1 -1
- data/lib/zk.rb +28 -21
- data/lib/zk/client/threaded.rb +107 -17
- data/lib/zk/client/unixisms.rb +1 -41
- data/lib/zk/core_ext.rb +28 -0
- data/lib/zk/election.rb +14 -3
- data/lib/zk/event_handler.rb +36 -37
- data/lib/zk/event_handler_subscription/actor.rb +37 -2
- data/lib/zk/event_handler_subscription/base.rb +9 -0
- data/lib/zk/exceptions.rb +5 -0
- data/lib/zk/fork_hook.rb +112 -0
- data/lib/zk/install_fork_hooks.rb +37 -0
- data/lib/zk/locker/exclusive_locker.rb +14 -10
- data/lib/zk/locker/locker_base.rb +43 -26
- data/lib/zk/locker/shared_locker.rb +9 -5
- data/lib/zk/logging.rb +29 -7
- data/lib/zk/node_deletion_watcher.rb +167 -0
- data/lib/zk/pool.rb +14 -4
- data/lib/zk/subscription.rb +15 -34
- data/lib/zk/threaded_callback.rb +113 -29
- data/lib/zk/threadpool.rb +136 -40
- data/lib/zk/version.rb +1 -1
- data/spec/logging_progress_bar_formatter.rb +12 -0
- data/spec/shared/client_contexts.rb +13 -1
- data/spec/shared/client_examples.rb +3 -1
- data/spec/spec_helper.rb +28 -3
- data/spec/support/client_forker.rb +49 -8
- data/spec/support/latch.rb +1 -19
- data/spec/support/logging.rb +26 -10
- data/spec/support/wait_watchers.rb +2 -2
- data/spec/zk/00_forked_client_integration_spec.rb +1 -1
- data/spec/zk/client_spec.rb +11 -2
- data/spec/zk/election_spec.rb +21 -7
- data/spec/zk/locker_spec.rb +42 -22
- data/spec/zk/node_deletion_watcher_spec.rb +69 -0
- data/spec/zk/pool_spec.rb +32 -18
- data/spec/zk/threaded_callback_spec.rb +78 -0
- data/spec/zk/threadpool_spec.rb +52 -0
- data/spec/zk/watch_spec.rb +4 -0
- data/zk.gemspec +2 -1
- metadata +36 -10
- data/spec/support/logging_progress_bar_formatter.rb +0 -14
data/lib/zk/pool.rb
CHANGED
@@ -91,6 +91,13 @@ module ZK
|
|
91
91
|
end
|
92
92
|
end
|
93
93
|
|
94
|
+
# @private
|
95
|
+
def wait_until_closed
|
96
|
+
@mutex.synchronize do
|
97
|
+
@checkin_cond.wait_until { @state == :closed }
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
94
101
|
# yields next available connection to the block
|
95
102
|
#
|
96
103
|
# raises PoolIsShuttingDownException immediately if close_all! has been
|
@@ -139,7 +146,7 @@ module ZK
|
|
139
146
|
end
|
140
147
|
|
141
148
|
def assert_open!
|
142
|
-
raise Exceptions::PoolIsShuttingDownException unless open?
|
149
|
+
raise Exceptions::PoolIsShuttingDownException, "pool is shutting down" unless open?
|
143
150
|
end
|
144
151
|
|
145
152
|
end # Base
|
@@ -195,7 +202,7 @@ module ZK
|
|
195
202
|
|
196
203
|
@pool << connection
|
197
204
|
|
198
|
-
@checkin_cond.
|
205
|
+
@checkin_cond.broadcast
|
199
206
|
end
|
200
207
|
end
|
201
208
|
|
@@ -206,7 +213,8 @@ module ZK
|
|
206
213
|
|
207
214
|
def checkout(blocking=true)
|
208
215
|
raise ArgumentError, "checkout does not take a block, use .with_connection" if block_given?
|
209
|
-
@mutex.
|
216
|
+
@mutex.lock
|
217
|
+
begin
|
210
218
|
while true
|
211
219
|
assert_open!
|
212
220
|
|
@@ -236,7 +244,9 @@ module ZK
|
|
236
244
|
else
|
237
245
|
return false
|
238
246
|
end
|
239
|
-
end # while
|
247
|
+
end # while
|
248
|
+
ensure
|
249
|
+
@mutex.unlock rescue nil
|
240
250
|
end
|
241
251
|
end
|
242
252
|
|
data/lib/zk/subscription.rb
CHANGED
@@ -18,9 +18,9 @@ module ZK
|
|
18
18
|
|
19
19
|
def initialize(parent, block)
|
20
20
|
raise ArgumentError, "block must repsond_to?(:call)" unless block.respond_to?(:call)
|
21
|
-
@parent
|
21
|
+
@parent = parent
|
22
22
|
@callable = block
|
23
|
-
|
23
|
+
@mutex = Monitor.new
|
24
24
|
end
|
25
25
|
|
26
26
|
def unregistered?
|
@@ -29,11 +29,20 @@ module ZK
|
|
29
29
|
|
30
30
|
# calls unregister on parent, then sets parent to nil
|
31
31
|
def unregister
|
32
|
-
|
33
|
-
|
34
|
-
|
32
|
+
obj = nil
|
33
|
+
|
34
|
+
synchronize do
|
35
|
+
return false unless @parent
|
36
|
+
obj, @parent = @parent, nil
|
37
|
+
end
|
38
|
+
|
39
|
+
obj.unregister(self)
|
40
|
+
end
|
41
|
+
|
42
|
+
# an alias for unregister
|
43
|
+
def unsubscribe
|
44
|
+
unregister
|
35
45
|
end
|
36
|
-
alias unsubscribe unregister
|
37
46
|
|
38
47
|
# @private
|
39
48
|
def call(*args)
|
@@ -50,34 +59,6 @@ module ZK
|
|
50
59
|
@mutex.synchronize { yield }
|
51
60
|
end
|
52
61
|
end
|
53
|
-
|
54
|
-
module ActorStyle
|
55
|
-
extend Concern
|
56
|
-
|
57
|
-
included do
|
58
|
-
alias_method_chain :unsubscribe, :threaded_callback
|
59
|
-
alias_method_chain :callable, :threaded_callback_wrapper
|
60
|
-
alias_method_chain :reopen_after_fork!, :threaded_refresh
|
61
|
-
end
|
62
|
-
|
63
|
-
def unsubscribe_with_threaded_callback
|
64
|
-
synchronize do
|
65
|
-
@threaded_callback && @threaded_callback.shutdown
|
66
|
-
unsubscribe_without_threaded_callback
|
67
|
-
end
|
68
|
-
end
|
69
|
-
|
70
|
-
def reopen_after_fork_with_threaded_refresh!
|
71
|
-
reopen_after_fork_without_threaded_refresh!
|
72
|
-
@threaded_callback = ThreadedCallback.new(@callable)
|
73
|
-
end
|
74
|
-
|
75
|
-
def callable_with_threaded_callback_wrapper(*args)
|
76
|
-
synchronize do
|
77
|
-
@threaded_callback ||= ThreadedCallback.new(@callable)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
81
62
|
end
|
82
63
|
end
|
83
64
|
|
data/lib/zk/threaded_callback.rb
CHANGED
@@ -5,37 +5,58 @@ module ZK
|
|
5
5
|
# for background processing.
|
6
6
|
class ThreadedCallback
|
7
7
|
include ZK::Logging
|
8
|
+
include ZK::Exceptions
|
8
9
|
|
9
10
|
attr_reader :callback
|
10
11
|
|
11
12
|
def initialize(callback=nil, &blk)
|
12
13
|
@callback = callback || blk
|
13
|
-
@mutex = Monitor.new
|
14
14
|
|
15
|
-
@
|
16
|
-
|
17
|
-
reopen_after_fork!
|
18
|
-
end
|
15
|
+
@state = :paused
|
16
|
+
reopen_after_fork!
|
19
17
|
end
|
20
18
|
|
21
19
|
def running?
|
22
|
-
@mutex.synchronize { @running }
|
20
|
+
@mutex.synchronize { @state == :running }
|
21
|
+
end
|
22
|
+
|
23
|
+
# @private
|
24
|
+
def alive?
|
25
|
+
@thread && @thread.alive?
|
23
26
|
end
|
24
27
|
|
25
28
|
# how long to wait on thread shutdown before we return
|
26
|
-
def shutdown(timeout=
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
29
|
+
def shutdown(timeout=5)
|
30
|
+
# logger.debug { "#{self.class}##{__method__}" }
|
31
|
+
|
32
|
+
@mutex.lock
|
33
|
+
begin
|
34
|
+
return true if @state == :shutdown
|
35
|
+
|
36
|
+
@state = :shutdown
|
37
|
+
@cond.broadcast
|
38
|
+
ensure
|
39
|
+
@mutex.unlock rescue nil
|
34
40
|
end
|
41
|
+
|
42
|
+
return true unless @thread
|
43
|
+
|
44
|
+
unless @thread.join(timeout) == @thread
|
45
|
+
logger.error { "#{self.class} timed out waiting for dispatch thread, callback: #{callback.inspect}" }
|
46
|
+
return false
|
47
|
+
end
|
48
|
+
|
49
|
+
true
|
35
50
|
end
|
36
51
|
|
37
52
|
def call(*args)
|
38
|
-
@
|
53
|
+
@mutex.lock
|
54
|
+
begin
|
55
|
+
@array << args
|
56
|
+
@cond.broadcast
|
57
|
+
ensure
|
58
|
+
@mutex.unlock rescue nil
|
59
|
+
end
|
39
60
|
end
|
40
61
|
|
41
62
|
# called after a fork to replace a dead delivery thread
|
@@ -44,27 +65,90 @@ module ZK
|
|
44
65
|
#
|
45
66
|
# @private
|
46
67
|
def reopen_after_fork!
|
47
|
-
|
48
|
-
|
49
|
-
@
|
50
|
-
|
51
|
-
|
68
|
+
# logger.debug { "#{self.class}##{__method__}" }
|
69
|
+
|
70
|
+
unless @state == :paused
|
71
|
+
raise InvalidStateError, "state should have been :paused, not: #{@state.inspect}"
|
72
|
+
end
|
73
|
+
|
74
|
+
if @thread
|
75
|
+
raise InvalidStateError, "WTF?! did you fork in a callback? my thread was alive!" if @thread.alive?
|
76
|
+
@thread = nil
|
77
|
+
end
|
78
|
+
|
79
|
+
@mutex = Mutex.new
|
80
|
+
@cond = ConditionVariable.new
|
81
|
+
@array = []
|
82
|
+
resume_after_fork_in_parent
|
83
|
+
end
|
84
|
+
|
85
|
+
# shuts down the event delivery thread, but keeps the queue so we can continue
|
86
|
+
# delivering queued events when {#resume_after_fork_in_parent} is called
|
87
|
+
def pause_before_fork_in_parent
|
88
|
+
@mutex.lock
|
89
|
+
begin
|
90
|
+
raise InvalidStateError, "@state was not :running, @state: #{@state.inspect}" if @state != :running
|
91
|
+
return if @state == :paused
|
92
|
+
|
93
|
+
@state = :paused
|
94
|
+
@cond.broadcast
|
95
|
+
ensure
|
96
|
+
@mutex.unlock rescue nil
|
97
|
+
end
|
98
|
+
|
99
|
+
return unless @thread
|
100
|
+
|
101
|
+
@thread.join
|
102
|
+
@thread = nil
|
103
|
+
end
|
104
|
+
|
105
|
+
def resume_after_fork_in_parent
|
106
|
+
@mutex.lock
|
107
|
+
begin
|
108
|
+
raise InvalidStateError, "@state was not :paused, @state: #{@state.inspect}" if @state != :paused
|
109
|
+
raise InvalidStateError, "@thread was not nil! #{@thread.inspect}" if @thread
|
110
|
+
|
111
|
+
@state = :running
|
112
|
+
# logger.debug { "#{self.class}##{__method__} spawning dispatch thread" }
|
113
|
+
spawn_dispatch_thread
|
114
|
+
ensure
|
115
|
+
@mutex.unlock rescue nil
|
116
|
+
end
|
52
117
|
end
|
53
118
|
|
54
119
|
protected
|
120
|
+
# intentionally *not* synchronized
|
55
121
|
def spawn_dispatch_thread
|
56
|
-
Thread.new
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
122
|
+
@thread = Thread.new(&method(:dispatch_thread_body))
|
123
|
+
end
|
124
|
+
|
125
|
+
def dispatch_thread_body
|
126
|
+
Thread.current.abort_on_exception = true
|
127
|
+
while true
|
128
|
+
args = nil
|
129
|
+
|
130
|
+
@mutex.lock
|
131
|
+
begin
|
132
|
+
@cond.wait(@mutex) while @array.empty? and @state == :running
|
133
|
+
|
134
|
+
if @state != :running
|
135
|
+
# logger.warn { "ThreadedCallback, state is #{@state.inspect}, returning" }
|
136
|
+
return
|
65
137
|
end
|
138
|
+
|
139
|
+
args = @array.shift
|
140
|
+
ensure
|
141
|
+
@mutex.unlock rescue nil
|
142
|
+
end
|
143
|
+
|
144
|
+
begin
|
145
|
+
callback.call(*args)
|
146
|
+
rescue Exception => e
|
147
|
+
logger.error { e.to_std_format }
|
66
148
|
end
|
67
149
|
end
|
150
|
+
# ensure
|
151
|
+
# logger.debug { "#{self.class}##{__method__} returning" }
|
68
152
|
end
|
69
153
|
end
|
70
154
|
end
|
data/lib/zk/threadpool.rb
CHANGED
@@ -2,6 +2,7 @@ module ZK
|
|
2
2
|
# a simple threadpool for running blocks of code off the main thread
|
3
3
|
class Threadpool
|
4
4
|
include Logging
|
5
|
+
include Exceptions
|
5
6
|
|
6
7
|
DEFAULT_SIZE = 5
|
7
8
|
|
@@ -18,15 +19,28 @@ module ZK
|
|
18
19
|
@size = size || self.class.default_size
|
19
20
|
|
20
21
|
@threadpool = []
|
21
|
-
@
|
22
|
+
@state = :new
|
23
|
+
@queue = []
|
22
24
|
|
23
|
-
@mutex =
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@cond = ConditionVariable.new
|
24
27
|
|
25
28
|
@error_callbacks = []
|
26
29
|
|
27
30
|
start!
|
28
31
|
end
|
29
32
|
|
33
|
+
# are all of our threads alive?
|
34
|
+
# returns false if there are no running threads
|
35
|
+
def alive?
|
36
|
+
@mutex.lock
|
37
|
+
begin
|
38
|
+
!@threadpool.empty? and @threadpool.all?(&:alive?)
|
39
|
+
ensure
|
40
|
+
@mutex.unlock rescue nil
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
30
44
|
# Queue an operation to be run on an internal threadpool. You may either
|
31
45
|
# provide an object that responds_to?(:call) or pass a block. There is no
|
32
46
|
# mechanism for retrieving the result of the operation, it is purely
|
@@ -36,45 +50,103 @@ module ZK
|
|
36
50
|
def defer(callable=nil, &blk)
|
37
51
|
callable ||= blk
|
38
52
|
|
39
|
-
# XXX(slyphon): do we care if the threadpool is not running?
|
40
|
-
# raise Exceptions::ThreadpoolIsNotRunningException unless running?
|
41
53
|
raise ArgumentError, "Argument to Threadpool#defer must respond_to?(:call)" unless callable.respond_to?(:call)
|
42
54
|
|
43
|
-
@
|
55
|
+
@mutex.lock
|
56
|
+
begin
|
57
|
+
@queue << callable
|
58
|
+
@cond.broadcast
|
59
|
+
ensure
|
60
|
+
@mutex.unlock rescue nil
|
61
|
+
end
|
62
|
+
|
44
63
|
nil
|
45
64
|
end
|
46
65
|
|
47
66
|
def running?
|
48
|
-
@mutex.
|
67
|
+
@mutex.lock
|
68
|
+
begin
|
69
|
+
@state == :running
|
70
|
+
ensure
|
71
|
+
@mutex.unlock rescue nil
|
72
|
+
end
|
49
73
|
end
|
50
74
|
|
51
75
|
# returns true if the current thread is one of the threadpool threads
|
52
76
|
def on_threadpool?
|
53
|
-
tp =
|
54
|
-
|
77
|
+
tp = nil
|
78
|
+
|
79
|
+
@mutex.synchronize do
|
80
|
+
return false unless @threadpool # you can't dup nil
|
81
|
+
tp = @threadpool.dup
|
82
|
+
end
|
83
|
+
|
84
|
+
tp.respond_to?(:include?) and tp.include?(Thread.current)
|
55
85
|
end
|
56
86
|
|
57
87
|
# starts the threadpool if not already running
|
58
88
|
def start!
|
59
89
|
@mutex.synchronize do
|
60
|
-
return false if @running
|
61
|
-
@
|
90
|
+
return false if @state == :running
|
91
|
+
@state = :running
|
62
92
|
spawn_threadpool
|
63
93
|
end
|
94
|
+
|
64
95
|
true
|
65
96
|
end
|
66
97
|
|
67
98
|
# like the start! method, but checks for dead threads in the threadpool
|
68
99
|
# (which will happen after a fork())
|
100
|
+
#
|
101
|
+
# This will reset the state of the pool and any blocks registered will be
|
102
|
+
# lost
|
103
|
+
#
|
104
|
+
#
|
69
105
|
# @private
|
70
106
|
def reopen_after_fork!
|
71
|
-
|
72
|
-
@
|
73
|
-
|
107
|
+
# ok, we know that only the child process calls this, right?
|
108
|
+
return false unless (@state == :running) or (@state == :paused)
|
109
|
+
logger.debug { "#{self.class}##{__method__}" }
|
110
|
+
|
111
|
+
@state = :running
|
112
|
+
@mutex = Mutex.new
|
113
|
+
@cond = ConditionVariable.new
|
114
|
+
@queue = []
|
74
115
|
prune_dead_threads
|
75
116
|
spawn_threadpool
|
76
117
|
end
|
77
118
|
|
119
|
+
# @private
|
120
|
+
def pause_before_fork_in_parent
|
121
|
+
threads = nil
|
122
|
+
|
123
|
+
@mutex.lock
|
124
|
+
begin
|
125
|
+
raise InvalidStateError, "invalid state, expected to be :running, was #{@state.inspect}" if @state != :running
|
126
|
+
return false if @state == :paused
|
127
|
+
@state = :paused
|
128
|
+
@cond.broadcast # wake threads, let them die
|
129
|
+
threads = @threadpool.slice!(0, @threadpool.length)
|
130
|
+
ensure
|
131
|
+
@mutex.unlock rescue nil
|
132
|
+
end
|
133
|
+
|
134
|
+
join_all(threads)
|
135
|
+
true
|
136
|
+
end
|
137
|
+
|
138
|
+
# @private
|
139
|
+
def resume_after_fork_in_parent
|
140
|
+
@mutex.lock
|
141
|
+
begin
|
142
|
+
raise InvalidStateError, "expected :paused, was #{@state.inspect}" if @state != :paused
|
143
|
+
ensure
|
144
|
+
@mutex.unlock rescue nil
|
145
|
+
end
|
146
|
+
|
147
|
+
start!
|
148
|
+
end
|
149
|
+
|
78
150
|
# register a block to be called back with unhandled exceptions that occur
|
79
151
|
# in the threadpool.
|
80
152
|
#
|
@@ -95,14 +167,27 @@ module ZK
|
|
95
167
|
# the default timeout is 2 seconds per thread
|
96
168
|
#
|
97
169
|
def shutdown(timeout=2)
|
98
|
-
|
99
|
-
return unless @running
|
100
|
-
@running = false
|
101
|
-
@threadqueue.clear
|
102
|
-
@size.times { @threadqueue << KILL_TOKEN }
|
170
|
+
threads = nil
|
103
171
|
|
172
|
+
@mutex.lock
|
173
|
+
begin
|
174
|
+
return false if @state == :shutdown
|
175
|
+
@state = :shutdown
|
176
|
+
|
177
|
+
@queue.clear
|
104
178
|
threads, @threadpool = @threadpool, []
|
179
|
+
@cond.broadcast
|
180
|
+
ensure
|
181
|
+
@mutex.unlock rescue nil
|
182
|
+
end
|
183
|
+
|
184
|
+
join_all(threads)
|
185
|
+
|
186
|
+
nil
|
187
|
+
end
|
105
188
|
|
189
|
+
private
|
190
|
+
def join_all(threads, timeout=nil)
|
106
191
|
while th = threads.shift
|
107
192
|
begin
|
108
193
|
th.join(timeout)
|
@@ -111,14 +196,8 @@ module ZK
|
|
111
196
|
logger.error { e.to_std_format }
|
112
197
|
end
|
113
198
|
end
|
114
|
-
|
115
|
-
@threadqueue = ::Queue.new
|
116
199
|
end
|
117
|
-
|
118
|
-
nil
|
119
|
-
end
|
120
|
-
|
121
|
-
private
|
200
|
+
|
122
201
|
def dispatch_to_error_handler(e)
|
123
202
|
# make a copy that will be free from thread manipulation
|
124
203
|
# and doesn't require holding the lock
|
@@ -150,7 +229,8 @@ module ZK
|
|
150
229
|
end
|
151
230
|
|
152
231
|
def prune_dead_threads
|
153
|
-
@mutex.
|
232
|
+
@mutex.lock
|
233
|
+
begin
|
154
234
|
threads, @threadpool = @threadpool, []
|
155
235
|
return if threads.empty?
|
156
236
|
|
@@ -164,25 +244,41 @@ module ZK
|
|
164
244
|
logger.error { e.to_std_format }
|
165
245
|
end
|
166
246
|
end
|
247
|
+
ensure
|
248
|
+
@mutex.unlock rescue nil
|
167
249
|
end
|
168
250
|
end
|
169
251
|
|
170
|
-
def spawn_threadpool
|
171
|
-
@
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
252
|
+
def spawn_threadpool
|
253
|
+
until @threadpool.size >= @size.to_i
|
254
|
+
@threadpool << Thread.new(&method(:worker_thread_body))
|
255
|
+
end
|
256
|
+
# logger.debug { "spawn threadpool complete" }
|
257
|
+
end
|
258
|
+
|
259
|
+
def worker_thread_body
|
260
|
+
while true
|
261
|
+
op = nil
|
262
|
+
|
263
|
+
@mutex.lock
|
264
|
+
begin
|
265
|
+
return if @state != :running
|
266
|
+
|
267
|
+
unless op = @queue.shift
|
268
|
+
@cond.wait(@mutex) if @queue.empty? and (@state == :running)
|
183
269
|
end
|
270
|
+
ensure
|
271
|
+
@mutex.unlock rescue nil
|
272
|
+
end
|
273
|
+
|
274
|
+
next unless op
|
184
275
|
|
185
|
-
|
276
|
+
# logger.debug { "got #{op.inspect} in thread" }
|
277
|
+
|
278
|
+
begin
|
279
|
+
op.call if op
|
280
|
+
rescue Exception => e
|
281
|
+
dispatch_to_error_handler(e)
|
186
282
|
end
|
187
283
|
end
|
188
284
|
end
|