zk 1.4.2 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/.dotfiles/ctags_paths +1 -0
  2. data/.dotfiles/rspec-logging +2 -2
  3. data/.gitignore +1 -0
  4. data/Gemfile +9 -3
  5. data/Guardfile +36 -0
  6. data/README.markdown +21 -18
  7. data/RELEASES.markdown +10 -0
  8. data/Rakefile +1 -1
  9. data/lib/zk.rb +28 -21
  10. data/lib/zk/client/threaded.rb +107 -17
  11. data/lib/zk/client/unixisms.rb +1 -41
  12. data/lib/zk/core_ext.rb +28 -0
  13. data/lib/zk/election.rb +14 -3
  14. data/lib/zk/event_handler.rb +36 -37
  15. data/lib/zk/event_handler_subscription/actor.rb +37 -2
  16. data/lib/zk/event_handler_subscription/base.rb +9 -0
  17. data/lib/zk/exceptions.rb +5 -0
  18. data/lib/zk/fork_hook.rb +112 -0
  19. data/lib/zk/install_fork_hooks.rb +37 -0
  20. data/lib/zk/locker/exclusive_locker.rb +14 -10
  21. data/lib/zk/locker/locker_base.rb +43 -26
  22. data/lib/zk/locker/shared_locker.rb +9 -5
  23. data/lib/zk/logging.rb +29 -7
  24. data/lib/zk/node_deletion_watcher.rb +167 -0
  25. data/lib/zk/pool.rb +14 -4
  26. data/lib/zk/subscription.rb +15 -34
  27. data/lib/zk/threaded_callback.rb +113 -29
  28. data/lib/zk/threadpool.rb +136 -40
  29. data/lib/zk/version.rb +1 -1
  30. data/spec/logging_progress_bar_formatter.rb +12 -0
  31. data/spec/shared/client_contexts.rb +13 -1
  32. data/spec/shared/client_examples.rb +3 -1
  33. data/spec/spec_helper.rb +28 -3
  34. data/spec/support/client_forker.rb +49 -8
  35. data/spec/support/latch.rb +1 -19
  36. data/spec/support/logging.rb +26 -10
  37. data/spec/support/wait_watchers.rb +2 -2
  38. data/spec/zk/00_forked_client_integration_spec.rb +1 -1
  39. data/spec/zk/client_spec.rb +11 -2
  40. data/spec/zk/election_spec.rb +21 -7
  41. data/spec/zk/locker_spec.rb +42 -22
  42. data/spec/zk/node_deletion_watcher_spec.rb +69 -0
  43. data/spec/zk/pool_spec.rb +32 -18
  44. data/spec/zk/threaded_callback_spec.rb +78 -0
  45. data/spec/zk/threadpool_spec.rb +52 -0
  46. data/spec/zk/watch_spec.rb +4 -0
  47. data/zk.gemspec +2 -1
  48. metadata +36 -10
  49. data/spec/support/logging_progress_bar_formatter.rb +0 -14
@@ -91,6 +91,13 @@ module ZK
91
91
  end
92
92
  end
93
93
 
94
+ # @private
95
+ def wait_until_closed
96
+ @mutex.synchronize do
97
+ @checkin_cond.wait_until { @state == :closed }
98
+ end
99
+ end
100
+
94
101
  # yields next available connection to the block
95
102
  #
96
103
  # raises PoolIsShuttingDownException immediately if close_all! has been
@@ -139,7 +146,7 @@ module ZK
139
146
  end
140
147
 
141
148
  def assert_open!
142
- raise Exceptions::PoolIsShuttingDownException unless open?
149
+ raise Exceptions::PoolIsShuttingDownException, "pool is shutting down" unless open?
143
150
  end
144
151
 
145
152
  end # Base
@@ -195,7 +202,7 @@ module ZK
195
202
 
196
203
  @pool << connection
197
204
 
198
- @checkin_cond.signal
205
+ @checkin_cond.broadcast
199
206
  end
200
207
  end
201
208
 
@@ -206,7 +213,8 @@ module ZK
206
213
 
207
214
  def checkout(blocking=true)
208
215
  raise ArgumentError, "checkout does not take a block, use .with_connection" if block_given?
209
- @mutex.synchronize do
216
+ @mutex.lock
217
+ begin
210
218
  while true
211
219
  assert_open!
212
220
 
@@ -236,7 +244,9 @@ module ZK
236
244
  else
237
245
  return false
238
246
  end
239
- end # while
247
+ end # while
248
+ ensure
249
+ @mutex.unlock rescue nil
240
250
  end
241
251
  end
242
252
 
@@ -18,9 +18,9 @@ module ZK
18
18
 
19
19
  def initialize(parent, block)
20
20
  raise ArgumentError, "block must repsond_to?(:call)" unless block.respond_to?(:call)
21
- @parent = parent
21
+ @parent = parent
22
22
  @callable = block
23
- reopen_after_fork!
23
+ @mutex = Monitor.new
24
24
  end
25
25
 
26
26
  def unregistered?
@@ -29,11 +29,20 @@ module ZK
29
29
 
30
30
  # calls unregister on parent, then sets parent to nil
31
31
  def unregister
32
- return false unless @parent
33
- @parent.unregister(self)
34
- @parent = nil
32
+ obj = nil
33
+
34
+ synchronize do
35
+ return false unless @parent
36
+ obj, @parent = @parent, nil
37
+ end
38
+
39
+ obj.unregister(self)
40
+ end
41
+
42
+ # an alias for unregister
43
+ def unsubscribe
44
+ unregister
35
45
  end
36
- alias unsubscribe unregister
37
46
 
38
47
  # @private
39
48
  def call(*args)
@@ -50,34 +59,6 @@ module ZK
50
59
  @mutex.synchronize { yield }
51
60
  end
52
61
  end
53
-
54
- module ActorStyle
55
- extend Concern
56
-
57
- included do
58
- alias_method_chain :unsubscribe, :threaded_callback
59
- alias_method_chain :callable, :threaded_callback_wrapper
60
- alias_method_chain :reopen_after_fork!, :threaded_refresh
61
- end
62
-
63
- def unsubscribe_with_threaded_callback
64
- synchronize do
65
- @threaded_callback && @threaded_callback.shutdown
66
- unsubscribe_without_threaded_callback
67
- end
68
- end
69
-
70
- def reopen_after_fork_with_threaded_refresh!
71
- reopen_after_fork_without_threaded_refresh!
72
- @threaded_callback = ThreadedCallback.new(@callable)
73
- end
74
-
75
- def callable_with_threaded_callback_wrapper(*args)
76
- synchronize do
77
- @threaded_callback ||= ThreadedCallback.new(@callable)
78
- end
79
- end
80
- end
81
62
  end
82
63
  end
83
64
 
@@ -5,37 +5,58 @@ module ZK
5
5
  # for background processing.
6
6
  class ThreadedCallback
7
7
  include ZK::Logging
8
+ include ZK::Exceptions
8
9
 
9
10
  attr_reader :callback
10
11
 
11
12
  def initialize(callback=nil, &blk)
12
13
  @callback = callback || blk
13
- @mutex = Monitor.new
14
14
 
15
- @mutex.synchronize do
16
- @running = true
17
- reopen_after_fork!
18
- end
15
+ @state = :paused
16
+ reopen_after_fork!
19
17
  end
20
18
 
21
19
  def running?
22
- @mutex.synchronize { @running }
20
+ @mutex.synchronize { @state == :running }
21
+ end
22
+
23
+ # @private
24
+ def alive?
25
+ @thread && @thread.alive?
23
26
  end
24
27
 
25
28
  # how long to wait on thread shutdown before we return
26
- def shutdown(timeout=2)
27
- @mutex.synchronize do
28
- @running = false
29
- @queue.push(KILL_TOKEN)
30
- return unless @thread
31
- unless @thread.join(2)
32
- logger.error { "#{self.class} timed out waiting for dispatch thread, callback: #{callback.inspect}" }
33
- end
29
+ def shutdown(timeout=5)
30
+ # logger.debug { "#{self.class}##{__method__}" }
31
+
32
+ @mutex.lock
33
+ begin
34
+ return true if @state == :shutdown
35
+
36
+ @state = :shutdown
37
+ @cond.broadcast
38
+ ensure
39
+ @mutex.unlock rescue nil
34
40
  end
41
+
42
+ return true unless @thread
43
+
44
+ unless @thread.join(timeout) == @thread
45
+ logger.error { "#{self.class} timed out waiting for dispatch thread, callback: #{callback.inspect}" }
46
+ return false
47
+ end
48
+
49
+ true
35
50
  end
36
51
 
37
52
  def call(*args)
38
- @queue.push(args)
53
+ @mutex.lock
54
+ begin
55
+ @array << args
56
+ @cond.broadcast
57
+ ensure
58
+ @mutex.unlock rescue nil
59
+ end
39
60
  end
40
61
 
41
62
  # called after a fork to replace a dead delivery thread
@@ -44,27 +65,90 @@ module ZK
44
65
  #
45
66
  # @private
46
67
  def reopen_after_fork!
47
- return unless @running
48
- return if @thread and @thread.alive?
49
- @mutex = Monitor.new
50
- @queue = Queue.new
51
- @thread = spawn_dispatch_thread()
68
+ # logger.debug { "#{self.class}##{__method__}" }
69
+
70
+ unless @state == :paused
71
+ raise InvalidStateError, "state should have been :paused, not: #{@state.inspect}"
72
+ end
73
+
74
+ if @thread
75
+ raise InvalidStateError, "WTF?! did you fork in a callback? my thread was alive!" if @thread.alive?
76
+ @thread = nil
77
+ end
78
+
79
+ @mutex = Mutex.new
80
+ @cond = ConditionVariable.new
81
+ @array = []
82
+ resume_after_fork_in_parent
83
+ end
84
+
85
+ # shuts down the event delivery thread, but keeps the queue so we can continue
86
+ # delivering queued events when {#resume_after_fork_in_parent} is called
87
+ def pause_before_fork_in_parent
88
+ @mutex.lock
89
+ begin
90
+ raise InvalidStateError, "@state was not :running, @state: #{@state.inspect}" if @state != :running
91
+ return if @state == :paused
92
+
93
+ @state = :paused
94
+ @cond.broadcast
95
+ ensure
96
+ @mutex.unlock rescue nil
97
+ end
98
+
99
+ return unless @thread
100
+
101
+ @thread.join
102
+ @thread = nil
103
+ end
104
+
105
+ def resume_after_fork_in_parent
106
+ @mutex.lock
107
+ begin
108
+ raise InvalidStateError, "@state was not :paused, @state: #{@state.inspect}" if @state != :paused
109
+ raise InvalidStateError, "@thread was not nil! #{@thread.inspect}" if @thread
110
+
111
+ @state = :running
112
+ # logger.debug { "#{self.class}##{__method__} spawning dispatch thread" }
113
+ spawn_dispatch_thread
114
+ ensure
115
+ @mutex.unlock rescue nil
116
+ end
52
117
  end
53
118
 
54
119
  protected
120
+ # intentionally *not* synchronized
55
121
  def spawn_dispatch_thread
56
- Thread.new do
57
- while running?
58
- args = @queue.pop
59
- break if args == KILL_TOKEN
60
- begin
61
- callback.call(*args)
62
- rescue Exception => e
63
- logger.error { "error caught in handler for path: #{path.inspect}, interests: #{interests.inspect}" }
64
- logger.error { e.to_std_format }
122
+ @thread = Thread.new(&method(:dispatch_thread_body))
123
+ end
124
+
125
+ def dispatch_thread_body
126
+ Thread.current.abort_on_exception = true
127
+ while true
128
+ args = nil
129
+
130
+ @mutex.lock
131
+ begin
132
+ @cond.wait(@mutex) while @array.empty? and @state == :running
133
+
134
+ if @state != :running
135
+ # logger.warn { "ThreadedCallback, state is #{@state.inspect}, returning" }
136
+ return
65
137
  end
138
+
139
+ args = @array.shift
140
+ ensure
141
+ @mutex.unlock rescue nil
142
+ end
143
+
144
+ begin
145
+ callback.call(*args)
146
+ rescue Exception => e
147
+ logger.error { e.to_std_format }
66
148
  end
67
149
  end
150
+ # ensure
151
+ # logger.debug { "#{self.class}##{__method__} returning" }
68
152
  end
69
153
  end
70
154
  end
@@ -2,6 +2,7 @@ module ZK
2
2
  # a simple threadpool for running blocks of code off the main thread
3
3
  class Threadpool
4
4
  include Logging
5
+ include Exceptions
5
6
 
6
7
  DEFAULT_SIZE = 5
7
8
 
@@ -18,15 +19,28 @@ module ZK
18
19
  @size = size || self.class.default_size
19
20
 
20
21
  @threadpool = []
21
- @threadqueue = ::Queue.new
22
+ @state = :new
23
+ @queue = []
22
24
 
23
- @mutex = Monitor.new
25
+ @mutex = Mutex.new
26
+ @cond = ConditionVariable.new
24
27
 
25
28
  @error_callbacks = []
26
29
 
27
30
  start!
28
31
  end
29
32
 
33
+ # are all of our threads alive?
34
+ # returns false if there are no running threads
35
+ def alive?
36
+ @mutex.lock
37
+ begin
38
+ !@threadpool.empty? and @threadpool.all?(&:alive?)
39
+ ensure
40
+ @mutex.unlock rescue nil
41
+ end
42
+ end
43
+
30
44
  # Queue an operation to be run on an internal threadpool. You may either
31
45
  # provide an object that responds_to?(:call) or pass a block. There is no
32
46
  # mechanism for retrieving the result of the operation, it is purely
@@ -36,45 +50,103 @@ module ZK
36
50
  def defer(callable=nil, &blk)
37
51
  callable ||= blk
38
52
 
39
- # XXX(slyphon): do we care if the threadpool is not running?
40
- # raise Exceptions::ThreadpoolIsNotRunningException unless running?
41
53
  raise ArgumentError, "Argument to Threadpool#defer must respond_to?(:call)" unless callable.respond_to?(:call)
42
54
 
43
- @threadqueue << callable
55
+ @mutex.lock
56
+ begin
57
+ @queue << callable
58
+ @cond.broadcast
59
+ ensure
60
+ @mutex.unlock rescue nil
61
+ end
62
+
44
63
  nil
45
64
  end
46
65
 
47
66
  def running?
48
- @mutex.synchronize { @running }
67
+ @mutex.lock
68
+ begin
69
+ @state == :running
70
+ ensure
71
+ @mutex.unlock rescue nil
72
+ end
49
73
  end
50
74
 
51
75
  # returns true if the current thread is one of the threadpool threads
52
76
  def on_threadpool?
53
- tp = @mutex.synchronize { @threadpool.dup }
54
- tp and tp.respond_to?(:include?) and tp.include?(Thread.current)
77
+ tp = nil
78
+
79
+ @mutex.synchronize do
80
+ return false unless @threadpool # you can't dup nil
81
+ tp = @threadpool.dup
82
+ end
83
+
84
+ tp.respond_to?(:include?) and tp.include?(Thread.current)
55
85
  end
56
86
 
57
87
  # starts the threadpool if not already running
58
88
  def start!
59
89
  @mutex.synchronize do
60
- return false if @running
61
- @running = true
90
+ return false if @state == :running
91
+ @state = :running
62
92
  spawn_threadpool
63
93
  end
94
+
64
95
  true
65
96
  end
66
97
 
67
98
  # like the start! method, but checks for dead threads in the threadpool
68
99
  # (which will happen after a fork())
100
+ #
101
+ # This will reset the state of the pool and any blocks registered will be
102
+ # lost
103
+ #
104
+ #
69
105
  # @private
70
106
  def reopen_after_fork!
71
- return false unless @running
72
- @mutex = Monitor.new
73
- @threadqueue = Queue.new
107
+ # ok, we know that only the child process calls this, right?
108
+ return false unless (@state == :running) or (@state == :paused)
109
+ logger.debug { "#{self.class}##{__method__}" }
110
+
111
+ @state = :running
112
+ @mutex = Mutex.new
113
+ @cond = ConditionVariable.new
114
+ @queue = []
74
115
  prune_dead_threads
75
116
  spawn_threadpool
76
117
  end
77
118
 
119
+ # @private
120
+ def pause_before_fork_in_parent
121
+ threads = nil
122
+
123
+ @mutex.lock
124
+ begin
125
+ raise InvalidStateError, "invalid state, expected to be :running, was #{@state.inspect}" if @state != :running
126
+ return false if @state == :paused
127
+ @state = :paused
128
+ @cond.broadcast # wake threads, let them die
129
+ threads = @threadpool.slice!(0, @threadpool.length)
130
+ ensure
131
+ @mutex.unlock rescue nil
132
+ end
133
+
134
+ join_all(threads)
135
+ true
136
+ end
137
+
138
+ # @private
139
+ def resume_after_fork_in_parent
140
+ @mutex.lock
141
+ begin
142
+ raise InvalidStateError, "expected :paused, was #{@state.inspect}" if @state != :paused
143
+ ensure
144
+ @mutex.unlock rescue nil
145
+ end
146
+
147
+ start!
148
+ end
149
+
78
150
  # register a block to be called back with unhandled exceptions that occur
79
151
  # in the threadpool.
80
152
  #
@@ -95,14 +167,27 @@ module ZK
95
167
  # the default timeout is 2 seconds per thread
96
168
  #
97
169
  def shutdown(timeout=2)
98
- @mutex.synchronize do
99
- return unless @running
100
- @running = false
101
- @threadqueue.clear
102
- @size.times { @threadqueue << KILL_TOKEN }
170
+ threads = nil
103
171
 
172
+ @mutex.lock
173
+ begin
174
+ return false if @state == :shutdown
175
+ @state = :shutdown
176
+
177
+ @queue.clear
104
178
  threads, @threadpool = @threadpool, []
179
+ @cond.broadcast
180
+ ensure
181
+ @mutex.unlock rescue nil
182
+ end
183
+
184
+ join_all(threads)
185
+
186
+ nil
187
+ end
105
188
 
189
+ private
190
+ def join_all(threads, timeout=nil)
106
191
  while th = threads.shift
107
192
  begin
108
193
  th.join(timeout)
@@ -111,14 +196,8 @@ module ZK
111
196
  logger.error { e.to_std_format }
112
197
  end
113
198
  end
114
-
115
- @threadqueue = ::Queue.new
116
199
  end
117
-
118
- nil
119
- end
120
-
121
- private
200
+
122
201
  def dispatch_to_error_handler(e)
123
202
  # make a copy that will be free from thread manipulation
124
203
  # and doesn't require holding the lock
@@ -150,7 +229,8 @@ module ZK
150
229
  end
151
230
 
152
231
  def prune_dead_threads
153
- @mutex.synchronize do
232
+ @mutex.lock
233
+ begin
154
234
  threads, @threadpool = @threadpool, []
155
235
  return if threads.empty?
156
236
 
@@ -164,25 +244,41 @@ module ZK
164
244
  logger.error { e.to_std_format }
165
245
  end
166
246
  end
247
+ ensure
248
+ @mutex.unlock rescue nil
167
249
  end
168
250
  end
169
251
 
170
- def spawn_threadpool #:nodoc:
171
- @mutex.synchronize do
172
- until @threadpool.size >= @size.to_i
173
- thread = Thread.new do
174
- while @running
175
- begin
176
- op = @threadqueue.pop
177
- break if op == KILL_TOKEN
178
- op.call
179
- rescue Exception => e
180
- dispatch_to_error_handler(e)
181
- end
182
- end
252
+ def spawn_threadpool
253
+ until @threadpool.size >= @size.to_i
254
+ @threadpool << Thread.new(&method(:worker_thread_body))
255
+ end
256
+ # logger.debug { "spawn threadpool complete" }
257
+ end
258
+
259
+ def worker_thread_body
260
+ while true
261
+ op = nil
262
+
263
+ @mutex.lock
264
+ begin
265
+ return if @state != :running
266
+
267
+ unless op = @queue.shift
268
+ @cond.wait(@mutex) if @queue.empty? and (@state == :running)
183
269
  end
270
+ ensure
271
+ @mutex.unlock rescue nil
272
+ end
273
+
274
+ next unless op
184
275
 
185
- @threadpool << thread
276
+ # logger.debug { "got #{op.inspect} in thread" }
277
+
278
+ begin
279
+ op.call if op
280
+ rescue Exception => e
281
+ dispatch_to_error_handler(e)
186
282
  end
187
283
  end
188
284
  end