zk 1.4.2 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. data/.dotfiles/ctags_paths +1 -0
  2. data/.dotfiles/rspec-logging +2 -2
  3. data/.gitignore +1 -0
  4. data/Gemfile +9 -3
  5. data/Guardfile +36 -0
  6. data/README.markdown +21 -18
  7. data/RELEASES.markdown +10 -0
  8. data/Rakefile +1 -1
  9. data/lib/zk.rb +28 -21
  10. data/lib/zk/client/threaded.rb +107 -17
  11. data/lib/zk/client/unixisms.rb +1 -41
  12. data/lib/zk/core_ext.rb +28 -0
  13. data/lib/zk/election.rb +14 -3
  14. data/lib/zk/event_handler.rb +36 -37
  15. data/lib/zk/event_handler_subscription/actor.rb +37 -2
  16. data/lib/zk/event_handler_subscription/base.rb +9 -0
  17. data/lib/zk/exceptions.rb +5 -0
  18. data/lib/zk/fork_hook.rb +112 -0
  19. data/lib/zk/install_fork_hooks.rb +37 -0
  20. data/lib/zk/locker/exclusive_locker.rb +14 -10
  21. data/lib/zk/locker/locker_base.rb +43 -26
  22. data/lib/zk/locker/shared_locker.rb +9 -5
  23. data/lib/zk/logging.rb +29 -7
  24. data/lib/zk/node_deletion_watcher.rb +167 -0
  25. data/lib/zk/pool.rb +14 -4
  26. data/lib/zk/subscription.rb +15 -34
  27. data/lib/zk/threaded_callback.rb +113 -29
  28. data/lib/zk/threadpool.rb +136 -40
  29. data/lib/zk/version.rb +1 -1
  30. data/spec/logging_progress_bar_formatter.rb +12 -0
  31. data/spec/shared/client_contexts.rb +13 -1
  32. data/spec/shared/client_examples.rb +3 -1
  33. data/spec/spec_helper.rb +28 -3
  34. data/spec/support/client_forker.rb +49 -8
  35. data/spec/support/latch.rb +1 -19
  36. data/spec/support/logging.rb +26 -10
  37. data/spec/support/wait_watchers.rb +2 -2
  38. data/spec/zk/00_forked_client_integration_spec.rb +1 -1
  39. data/spec/zk/client_spec.rb +11 -2
  40. data/spec/zk/election_spec.rb +21 -7
  41. data/spec/zk/locker_spec.rb +42 -22
  42. data/spec/zk/node_deletion_watcher_spec.rb +69 -0
  43. data/spec/zk/pool_spec.rb +32 -18
  44. data/spec/zk/threaded_callback_spec.rb +78 -0
  45. data/spec/zk/threadpool_spec.rb +52 -0
  46. data/spec/zk/watch_spec.rb +4 -0
  47. data/zk.gemspec +2 -1
  48. metadata +36 -10
  49. data/spec/support/logging_progress_bar_formatter.rb +0 -14
@@ -1,3 +1,3 @@
1
1
  module ZK
2
- VERSION = "1.4.2"
2
+ VERSION = "1.5.0"
3
3
  end
@@ -0,0 +1,12 @@
1
+ require 'rspec/core/formatters/progress_formatter'
2
+
3
+ # essentially a monkey-patch to the ProgressBarFormatter, outputs
4
+ # '== #{example_proxy.description} ==' in the logs before each test. makes it
5
+ # easier to match up tests with the SQL they produce
6
+ class LoggingProgressBarFormatter < RSpec::Core::Formatters::ProgressFormatter
7
+ def example_started(example)
8
+ ::Logging.logger['spec'].write(yellow("\n=====<([ #{example.full_description} ])>=====\n"))
9
+ super
10
+ end
11
+ end
12
+
@@ -8,22 +8,34 @@ shared_context 'threaded client connection' do
8
8
  include_context 'connection opts'
9
9
 
10
10
  before do
11
+ # logger.debug { "threaded client connection - begin before hook" }
12
+
11
13
  @connection_string = "localhost:#{ZK.test_port}"
12
14
  @base_path = '/zktests'
13
15
  @zk = ZK::Client::Threaded.new(*connection_args).tap { |z| wait_until { z.connected? } }
14
16
  @threadpool_exception = nil
15
17
  @zk.on_exception { |e| @threadpool_exception = e }
16
18
  @zk.rm_rf(@base_path)
19
+
20
+ # logger.debug { "threaded client connection - end before hook" }
17
21
  end
18
22
 
19
23
  after do
20
24
  # raise @threadpool_exception if @threadpool_exception
21
- @zk.reopen if @zk.closed?
25
+ # logger.debug { "threaded client connection - after hook" }
26
+
27
+ if @zk.closed?
28
+ logger.debug { "zk was closed, calling reopen" }
29
+ @zk.reopen
30
+ end
31
+
22
32
  wait_until(5) { @zk.connected? }
23
33
 
24
34
  @zk.rm_rf(@base_path)
25
35
  @zk.close!
26
36
  wait_until(5) { @zk.closed? }
37
+
38
+ # logger.debug { "threaded client connection - end after hook" }
27
39
  end
28
40
  end
29
41
 
@@ -257,7 +257,9 @@ shared_examples_for 'client' do
257
257
 
258
258
  wait_until(2) do
259
259
  begin
260
- @events << @queue.pop(true)
260
+ event = @queue.pop(true)
261
+ logger.debug { "got event: #{event}" }
262
+ @events << event
261
263
  true
262
264
  rescue ThreadError
263
265
  false
@@ -36,15 +36,20 @@ RSpec.configure do |config|
36
36
  config.filter_run_excluding :rbx => :broken
37
37
  end
38
38
 
39
- if ZK.jruby? or not ZK.mri_193?
39
+ if ZK.mri_187?
40
+ config.filter_run_excluding :mri_187 => :broken
41
+ end
42
+
43
+ if ZK.jruby?
40
44
  config.filter_run_excluding :fork_required => true
45
+ config.filter_run_excluding :jruby => :broken
41
46
  end
42
47
 
43
48
  if ZK.spawn_zookeeper?
44
49
  require 'zk-server'
45
50
 
46
51
  config.before(:suite) do
47
- ZK.logger.debug { "Starting zookeeper service" }
52
+ ::Logging.logger['spec'].debug { "Starting zookeeper service" }
48
53
  ZK::Server.run do |c|
49
54
  c.client_port = ZK.test_port
50
55
  c.force_sync = false
@@ -53,10 +58,30 @@ RSpec.configure do |config|
53
58
  end
54
59
 
55
60
  config.after(:suite) do
56
- ZK.logger.debug { "stopping zookeeper service" }
61
+ ::Logging.logger['spec'].debug { "stopping zookeeper service" }
57
62
  ZK::Server.shutdown
58
63
  end
59
64
  end
65
+
66
+ # tester should return true if the object is a leak
67
+ def leak_check(klass, &tester)
68
+ count = 0
69
+ ObjectSpace.each_object(klass) { |o| count += 1 if tester.call(o) }
70
+ unless count.zero?
71
+ raise "There were #{count} leaked #{klass} objects after #{example.full_description.inspect}"
72
+ end
73
+ end
74
+
75
+ # these make tests run slow
76
+ if ENV['ZK_LEAK_CHECK']
77
+ config.after do
78
+ leak_check(ZK::Client::Threaded) { |o| !o.closed? }
79
+ leak_check(ZK::ThreadedCallback, &:alive?)
80
+ leak_check(ZK::Threadpool, &:alive?)
81
+ leak_check(Thread) { |th| Thread.current != th && th.alive? }
82
+ ZK::ForkHook.hooks.values.flatten.should be_empty
83
+ end
84
+ end
60
85
  end
61
86
 
62
87
  class ::Thread
@@ -14,16 +14,34 @@ class ClientForker
14
14
  @pids_root = "#{@base_path}/pid"
15
15
  end
16
16
 
17
+ LBORDER = ('-' * 35) << '< '
18
+ RBORDER = ' >' << ('-' * 35)
19
+
20
+ def mark(thing)
21
+ logger << "\n#{LBORDER}#{thing}#{RBORDER}\n\n"
22
+ end
23
+
24
+ def mark_around(thing)
25
+ mark "#{thing}: ENTER"
26
+ yield
27
+ ensure
28
+ mark "#{thing}: EXIT"
29
+ end
30
+
17
31
  def before
18
- ZK.open(*cnx_args) do |z|
19
- z.rm_rf(@base_path)
20
- z.mkdir_p(@pids_root)
32
+ mark_around('BEFORE') do
33
+ ZK.open(*cnx_args) do |z|
34
+ z.rm_rf(@base_path)
35
+ z.mkdir_p(@pids_root)
36
+ end
21
37
  end
22
38
  end
23
39
 
24
40
  def tear_down
25
- @zk.rm_rf(@base_path)
26
- @zk.close! unless @zk.closed?
41
+ mark_around('TEAR_DOWN') do
42
+ @zk.close! if @zk and !@zk.closed?
43
+ ZK.open(*cnx_args) { |z| z.rm_rf(@base_path) }
44
+ end
27
45
  end
28
46
 
29
47
  def kill_child!
@@ -34,8 +52,20 @@ class ClientForker
34
52
  rescue Errno::ESRCH
35
53
  end
36
54
 
55
+ CLEAR = "\e[0m".freeze
56
+ YELLOW = "\e[33m".freeze # Set the terminal's foreground ANSI color to yellow.
57
+
58
+ def yellow_log_formatter()
59
+ orig_formatter = ::Logger::Formatter.new
60
+
61
+ proc do |s,dt,pn,msg|
62
+ "#{CLEAR}#{YELLOW}#{orig_formatter.call(s,dt,pn,msg)}#{CLEAR}"
63
+ end
64
+ end
65
+
37
66
  def run
38
67
  before
68
+ mark 'BEGIN TEST'
39
69
 
40
70
  logger.debug { "Process.pid of parent: #{Process.pid}" }
41
71
 
@@ -51,7 +81,7 @@ class ClientForker
51
81
 
52
82
  @parent_pid = $$
53
83
 
54
- @zk.create("#{@pids_root}/#{$$}", $$.to_s)
84
+ @zk.create("#{@pids_root}/#{$$}", $$.to_s, :ignore => :node_exists)
55
85
 
56
86
  event_catcher = EventCatcher.new
57
87
 
@@ -66,8 +96,14 @@ class ClientForker
66
96
  logger.debug { "parent watching for children on #{@pids_root}" }
67
97
  @zk.children(@pids_root, :watch => true) # side-effect, register watch
68
98
 
99
+ ZK.install_fork_hook
100
+
101
+ mark 'FORK'
102
+
69
103
  @pid = fork do
70
- @zk.reopen
104
+ Thread.abort_on_exception = true
105
+ ::Logging.reopen
106
+
71
107
  @zk.wait_until_connected
72
108
 
73
109
  child_pid_path = "#{@pids_root}/#{$$}"
@@ -91,7 +127,11 @@ class ClientForker
91
127
  create_sub.unregister
92
128
  else
93
129
  logger.debug { "awaiting the create_latch to release" }
94
- create_latch.await
130
+ create_latch.await(2)
131
+ unless @zk.exists?(child_pid_path)
132
+ logger.debug { require 'pp'; PP.pp(@zk.event_handler, '') }
133
+ raise "child pid path not created after 2 sec"
134
+ end
95
135
  end
96
136
 
97
137
  logger.debug { "now testing for delete event totally created in child" }
@@ -152,6 +192,7 @@ class ClientForker
152
192
 
153
193
  # $stderr.puts "#{@pid} exited with status: #{stat.inspect}"
154
194
  ensure
195
+ mark "END TEST"
155
196
  kill_child!
156
197
  tear_down
157
198
  end
@@ -1,23 +1,5 @@
1
1
  # the much fabled 'latch' that tenderlove and nahi were on about
2
2
 
3
- class Latch
4
- def initialize(count = 1)
5
- @count = count
6
- @mutex = Monitor.new
7
- @cond = @mutex.new_cond
8
- end
9
-
10
- def release
11
- @mutex.synchronize {
12
- @count -= 1 if @count > 0
13
- @cond.broadcast if @count.zero?
14
- }
15
- end
16
-
17
- def await
18
- @mutex.synchronize {
19
- @cond.wait_while { @count > 0 }
20
- }
21
- end
3
+ class Latch < Zookeeper::Latch
22
4
  end
23
5
 
@@ -1,30 +1,46 @@
1
1
  module ZK
2
- # LOG_FILE = File.open(File.join(ZK::ZK_ROOT, 'test.log'), 'a').tap { |f| f.sync = true }
3
- LOG_FILE = File.join(ZK::ZK_ROOT, 'test.log')
4
- # LOG_FILE = $stderr
2
+ TEST_LOG_PATH = File.join(ZK::ZK_ROOT, 'test.log')
5
3
  end
6
4
 
7
- # ZK.logger = ENV['TRAVIS'] ? Logger.new($stderr) : Logger.new(ZK::LOG_FILE)
5
+ layout = Logging.layouts.pattern(
6
+ :pattern => '%.1l, [%d #%p] %30.30c{2}: %m\n',
7
+ :date_pattern => '%Y-%m-%d %H:%M:%S.%6N'
8
+ )
8
9
 
9
- ZK.logger = Logger.new(ZK::LOG_FILE).tap { |l| l.level = Logger::DEBUG }
10
+ appender = ENV['ZK_DEBUG'] ? Logging.appenders.stderr : Logging.appenders.file(ZK::TEST_LOG_PATH)
11
+ appender.layout = layout
12
+
13
+ %w[ZK ClientForker spec Zookeeper].each do |name|
14
+ ::Logging.logger[name].tap do |log|
15
+ log.appenders = [appender]
16
+ log.level = :debug
17
+ end
18
+ end
19
+
20
+ # this logger is kinda noisy
21
+ Logging.logger['ZK::EventHandler'].level = :info
22
+
23
+ Zookeeper.logger = Logging.logger['Zookeeper']
24
+ Zookeeper.logger.level = :info
25
+
26
+ # Zookeeper.logger = ZK.logger.clone_new_log(:progname => 'zoo')
10
27
 
11
28
  # Zookeeper.logger = ZK.logger
12
29
  # Zookeeper.set_debug_level(4)
13
30
 
14
- ZK.logger.debug { "LOG OPEN" }
15
-
16
31
  module SpecGlobalLogger
17
32
  def logger
18
- ZK.logger
33
+ @spec_global_logger ||= ::Logging.logger['spec']
19
34
  end
20
35
 
21
36
  # sets the log level to FATAL for the duration of the block
22
37
  def mute_logger
23
- orig_level, ZK.logger.level = ZK.logger.level, Logger::FATAL
38
+ zk_log = Logging.logger['ZK']
39
+ orig_level, zk_log.level = zk_log.level, :off
24
40
  orig_zk_level, Zookeeper.debug_level = Zookeeper.debug_level, Zookeeper::Constants::ZOO_LOG_LEVEL_ERROR
25
41
  yield
26
42
  ensure
27
- ZK.logger.level = orig_level
43
+ zk_log.level = orig_zk_level
28
44
  end
29
45
  end
30
46
 
@@ -19,7 +19,7 @@ module WaitWatchers
19
19
  #
20
20
  def wait_until(timeout=2)
21
21
  if ZK.travis? and timeout and timeout < 5
22
- ZK.logger.debug { "TRAVIS: adjusting wait_until timeout from #{timeout} to 5 sec" }
22
+ logger.debug { "TRAVIS: adjusting wait_until timeout from #{timeout} to 5 sec" }
23
23
  timeout = 5
24
24
  end
25
25
 
@@ -35,7 +35,7 @@ module WaitWatchers
35
35
  # inverse of wait_until
36
36
  def wait_while(timeout=2)
37
37
  if ZK.travis? and timeout and timeout < 5
38
- ZK.logger.debug { "TRAVIS: adjusting wait_while timeout from #{timeout} to 5 sec" }
38
+ logger.debug { "TRAVIS: adjusting wait_while timeout from #{timeout} to 5 sec" }
39
39
  timeout = 5
40
40
  end
41
41
 
@@ -8,7 +8,7 @@ describe 'forked client integration' do
8
8
  @base_path = '/zktests'
9
9
  @pids_root = "#{@base_path}/pid"
10
10
 
11
- @cnx_args = ["#{ZK.default_host}:#{ZK.test_port}", { :thread => :single, :timeout => 5 }]
11
+ @cnx_args = ["#{ZK.default_host}:#{ZK.test_port}", { :thread => :per_callback, :timeout => 5 }]
12
12
 
13
13
  ZK.open(*@cnx_args) do |z|
14
14
  z.rm_rf(@base_path)
@@ -11,7 +11,7 @@ describe ZK::Client::Threaded do
11
11
  include_context 'connection opts'
12
12
 
13
13
  before do
14
- @zk = ZK::Client::Threaded.new(*connection_args).tap { |z| wait_until { z.connected? } }
14
+ @zk = ZK::Client::Threaded.new(*connection_args)
15
15
  end
16
16
 
17
17
  after do
@@ -23,10 +23,19 @@ describe ZK::Client::Threaded do
23
23
 
24
24
  @zk.should be_kind_of(ZK::Client::Threaded) # yeah yeah, just be sure
25
25
 
26
+ shutdown_thread = nil
27
+
26
28
  @zk.defer do
27
- @zk.close!
29
+ shutdown_thread = @zk.close!
28
30
  end
29
31
 
32
+ wait_while { shutdown_thread.nil? }
33
+
34
+ shutdown_thread.should_not be_nil
35
+ shutdown_thread.should be_kind_of(Thread)
36
+
37
+ shutdown_thread.join(5).should == shutdown_thread
38
+
30
39
  wait_until(5) { @zk.closed? }.should be_true
31
40
  end
32
41
  end
@@ -1,11 +1,11 @@
1
1
  require 'spec_helper'
2
2
 
3
- describe ZK::Election do
3
+ describe ZK::Election, :jruby => :broken do
4
4
  include_context 'connection opts'
5
5
 
6
6
  before do
7
7
  ZK.open(connection_host) do |cnx|
8
- ZK.logger.debug { "REMOVING /_zkelection" }
8
+ logger.debug { "REMOVING /_zkelection" }
9
9
  cnx.rm_rf('/_zkelection')
10
10
  end
11
11
 
@@ -29,10 +29,15 @@ describe ZK::Election do
29
29
  @palin = ZK::Election::Candidate.new(@zk2, @election_name, :data => @data2)
30
30
  end
31
31
 
32
+ after do
33
+ @palin.close
34
+ @obama.close
35
+ end
36
+
32
37
  describe 'vote!' do
33
38
  describe 'loser' do
34
39
  it %[should wait until the leader has acked before firing loser callbacks] do
35
- queue = Queue.new
40
+ latch = Latch.new
36
41
  @do_ack = false
37
42
 
38
43
  @obama_won = nil
@@ -44,7 +49,7 @@ describe ZK::Election do
44
49
  @obama_waiting = true
45
50
 
46
51
  # wait for us to signal
47
- queue.pop
52
+ latch.await
48
53
 
49
54
  # $stderr.puts "obama on_winning_election entered"
50
55
  @obama_won = true
@@ -68,7 +73,7 @@ describe ZK::Election do
68
73
  # palin's callbacks haven't fired
69
74
  @palin_lost.should be_nil
70
75
 
71
- queue << :ok
76
+ latch.release
72
77
 
73
78
  wait_until { @obama_won }
74
79
  @obama_won.should be_true
@@ -83,33 +88,42 @@ describe ZK::Election do
83
88
 
84
89
  describe do
85
90
  before do
91
+ Thread.abort_on_exception = true
86
92
  @obama_won = @obama_lost = @palin_won = @palin_lost = nil
93
+ win_latch, lose_latch = Latch.new, Latch.new
87
94
 
88
95
  @obama.on_winning_election do
89
96
  logger.debug { "obama on_winning_election fired" }
90
97
  @obama_won = true
98
+ win_latch.release
91
99
  end
92
100
 
93
101
  @obama.on_losing_election do
94
102
  logger.debug { "obama on_losing_election fired" }
95
103
  @obama_lost = true
104
+ lose_latch.release
96
105
  end
97
106
 
98
107
  @palin.on_winning_election do
99
108
  logger.debug { "palin on_winning_election fired" }
100
109
  @palin_won = true
110
+ win_latch.release
101
111
  end
102
112
 
103
113
  @palin.on_losing_election do
104
114
  logger.debug { "palin on_losing_election fired" }
105
115
  @palin_lost = true
116
+ lose_latch.release
106
117
  end
107
118
 
108
119
  @obama.vote!
109
120
  @palin.vote!
110
121
 
111
- wait_until { @obama_won }
112
- wait_until { @palin_lost }
122
+ win_latch.await
123
+ @obama_won.should be_true
124
+
125
+ lose_latch.await
126
+ @palin_lost.should be_true
113
127
  end
114
128
 
115
129
  describe 'winner' do