zookeeper-ng 1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +7 -0
  2. data/.ctags_paths +1 -0
  3. data/.dotfiles/ruby-gemset +1 -0
  4. data/.dotfiles/ruby-version +1 -0
  5. data/.dotfiles/rvmrc +2 -0
  6. data/.gitignore +19 -0
  7. data/.gitmodules +3 -0
  8. data/.travis.yml +25 -0
  9. data/CHANGELOG +395 -0
  10. data/Gemfile +30 -0
  11. data/Guardfile +8 -0
  12. data/LICENSE +23 -0
  13. data/Manifest +29 -0
  14. data/README.markdown +85 -0
  15. data/Rakefile +121 -0
  16. data/cause-abort.rb +117 -0
  17. data/ext/.gitignore +6 -0
  18. data/ext/Rakefile +41 -0
  19. data/ext/c_zookeeper.rb +398 -0
  20. data/ext/common.h +17 -0
  21. data/ext/dbg.h +53 -0
  22. data/ext/depend +5 -0
  23. data/ext/event_lib.c +740 -0
  24. data/ext/event_lib.h +175 -0
  25. data/ext/extconf.rb +103 -0
  26. data/ext/generate_gvl_code.rb +321 -0
  27. data/ext/patches/zkc-3.3.5-network.patch +24 -0
  28. data/ext/patches/zkc-3.4.5-fetch-and-add.patch +16 -0
  29. data/ext/patches/zkc-3.4.5-logging.patch +41 -0
  30. data/ext/patches/zkc-3.4.5-out-of-order-ping.patch +163 -0
  31. data/ext/patches/zkc-3.4.5-overflow.patch +11 -0
  32. data/ext/patches/zkc-3.4.5-yosemite-htonl-fix.patch +102 -0
  33. data/ext/zkc-3.4.5.tar.gz +0 -0
  34. data/ext/zkrb.c +1075 -0
  35. data/ext/zkrb_wrapper.c +775 -0
  36. data/ext/zkrb_wrapper.h +350 -0
  37. data/ext/zkrb_wrapper_compat.c +15 -0
  38. data/ext/zkrb_wrapper_compat.h +11 -0
  39. data/ext/zookeeper_base.rb +256 -0
  40. data/java/java_base.rb +503 -0
  41. data/lib/zookeeper.rb +115 -0
  42. data/lib/zookeeper/acls.rb +44 -0
  43. data/lib/zookeeper/callbacks.rb +108 -0
  44. data/lib/zookeeper/client.rb +30 -0
  45. data/lib/zookeeper/client_methods.rb +282 -0
  46. data/lib/zookeeper/common.rb +122 -0
  47. data/lib/zookeeper/common/queue_with_pipe.rb +110 -0
  48. data/lib/zookeeper/compatibility.rb +138 -0
  49. data/lib/zookeeper/constants.rb +97 -0
  50. data/lib/zookeeper/continuation.rb +223 -0
  51. data/lib/zookeeper/core_ext.rb +58 -0
  52. data/lib/zookeeper/em_client.rb +55 -0
  53. data/lib/zookeeper/exceptions.rb +135 -0
  54. data/lib/zookeeper/forked.rb +19 -0
  55. data/lib/zookeeper/latch.rb +34 -0
  56. data/lib/zookeeper/logger.rb +39 -0
  57. data/lib/zookeeper/logger/forwarding_logger.rb +84 -0
  58. data/lib/zookeeper/monitor.rb +19 -0
  59. data/lib/zookeeper/rake_tasks.rb +165 -0
  60. data/lib/zookeeper/request_registry.rb +153 -0
  61. data/lib/zookeeper/stat.rb +21 -0
  62. data/lib/zookeeper/version.rb +4 -0
  63. data/notes.txt +14 -0
  64. data/scripts/upgrade-1.0-sed-alike.rb +46 -0
  65. data/spec/c_zookeeper_spec.rb +51 -0
  66. data/spec/chrooted_connection_spec.rb +83 -0
  67. data/spec/compatibilty_spec.rb +8 -0
  68. data/spec/default_watcher_spec.rb +41 -0
  69. data/spec/em_spec.rb +51 -0
  70. data/spec/ext/zookeeper_base_spec.rb +19 -0
  71. data/spec/forked_connection_spec.rb +124 -0
  72. data/spec/latch_spec.rb +24 -0
  73. data/spec/log4j.properties +17 -0
  74. data/spec/shared/all_success_return_values.rb +10 -0
  75. data/spec/shared/connection_examples.rb +1077 -0
  76. data/spec/spec_helper.rb +61 -0
  77. data/spec/support/00_logging.rb +38 -0
  78. data/spec/support/10_spawn_zookeeper.rb +24 -0
  79. data/spec/support/progress_formatter.rb +15 -0
  80. data/spec/support/zookeeper_spec_helpers.rb +96 -0
  81. data/spec/zookeeper_spec.rb +24 -0
  82. data/zookeeper.gemspec +38 -0
  83. data/zoomonkey/duplicates +3 -0
  84. data/zoomonkey/zoomonkey.rb +194 -0
  85. metadata +157 -0
@@ -0,0 +1,61 @@
1
+ $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
2
+ $LOAD_PATH.unshift(File.expand_path('../../ext', __FILE__))
3
+ $LOAD_PATH.uniq!
4
+
5
+ require 'rubygems'
6
+
7
+ release_ops_path = File.expand_path('../../releaseops/lib', __FILE__)
8
+
9
+ if File.exists?(release_ops_path)
10
+ require File.join(release_ops_path, 'releaseops')
11
+ ReleaseOps::SimpleCov.maybe_start
12
+ end
13
+
14
+ require 'zookeeper'
15
+
16
+ Dir[File.expand_path('../support/**/*.rb', __FILE__)].sort.each { |f| require(f) }
17
+
18
+ if ENV['ZKRB_DEBUG']
19
+ Zookeeper.set_debug_level(4)
20
+ end
21
+
22
+ if ENV['ZKRB_NOLOG']
23
+ SpecGlobalLogger.logger.level = ::Logger::FATAL
24
+ Zookeeper.set_debug_level(0)
25
+ end
26
+
27
+ RSpec.configure do |config|
28
+ config.mock_with :rspec
29
+ [Zookeeper::SpecHelpers, SpecGlobalLogger].each do |mod|
30
+ config.include(mod)
31
+ config.extend(mod)
32
+ end
33
+
34
+ if Zookeeper.spawn_zookeeper?
35
+ require 'zk-server'
36
+
37
+ config.before(:suite) do
38
+ SpecGlobalLogger.logger.debug { "Starting zookeeper service" }
39
+ ZK::Server.run do |c|
40
+ c.base_dir = File.expand_path('../../.zkserver', __FILE__)
41
+ c.client_port = Zookeeper.test_port
42
+ c.force_sync = false
43
+ c.snap_count = 1_000_000
44
+ end
45
+ end
46
+
47
+ config.after(:suite) do
48
+ SpecGlobalLogger.logger.debug { "stopping zookeeper service" }
49
+ ZK::Server.shutdown
50
+ end
51
+ end
52
+ end
53
+
54
+ require 'pp'
55
+
56
+ if RUBY_VERSION == '1.9.3'
57
+ trap('USR1') do
58
+ threads = Thread.list.map { |th| th.backtrace }
59
+ pp threads
60
+ end
61
+ end
@@ -0,0 +1,38 @@
1
+ module Zookeeper
2
+ TEST_LOG_PATH = File.expand_path('../../../test.log', __FILE__)
3
+
4
+ def self.setup_test_logger
5
+ log =
6
+ if (ENV['ZOOKEEPER_DEBUG'] || ENV['ZKRB_DEBUG'])
7
+ ::Logger.new(STDERR)
8
+ else
9
+ ::Logger.new(TEST_LOG_PATH)
10
+ end
11
+
12
+ log.level = ::Logger::DEBUG
13
+
14
+ Zookeeper::Logger.wrapped_logger = log
15
+ end
16
+ end
17
+
18
+ Zookeeper.setup_test_logger
19
+
20
+ module SpecGlobalLogger
21
+ extend self
22
+
23
+ def logger
24
+ @spec_global_logger ||= Zookeeper::Logger::ForwardingLogger.for(Zookeeper::Logger.wrapped_logger, 'spec')
25
+ end
26
+
27
+ # sets the log level to FATAL for the duration of the block
28
+ def mute_logger
29
+ zk_log = Zookeeper::Logger.wrapped_logger
30
+
31
+ orig_level, zk_log.level = zk_log.level, ::Logger::FATAL
32
+ orig_zk_level, Zookeeper.debug_level = Zookeeper.debug_level, Zookeeper::Constants::ZOO_LOG_LEVEL_ERROR
33
+ yield
34
+ ensure
35
+ zk_log.level = orig_zk_level
36
+ end
37
+ end
38
+
@@ -0,0 +1,24 @@
1
+ module Zookeeper
2
+ def self.spawn_zookeeper?
3
+ !!ENV['SPAWN_ZOOKEEPER']
4
+ end
5
+
6
+ def self.travis?
7
+ !!ENV['TRAVIS']
8
+ end
9
+
10
+ def self.default_cnx_host
11
+ ENV['ZK_DEFAULT_HOST'] || 'localhost'
12
+ end
13
+
14
+ @test_port ||= spawn_zookeeper? ? 21811 : 2181
15
+
16
+ class << self
17
+ attr_accessor :test_port
18
+ end
19
+
20
+ def self.default_cnx_str
21
+ "#{default_cnx_host}:#{test_port}"
22
+ end
23
+ end
24
+
@@ -0,0 +1,15 @@
1
+ require 'rspec/core/formatters/progress_formatter'
2
+
3
+ module RSpec
4
+ module Core
5
+ module Formatters
6
+ class ProgressFormatter
7
+ def example_started(example)
8
+ SpecGlobalLogger.logger << pending_color("\n=====<([ #{example.full_description} ])>=====\n")
9
+ super(example)
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
15
+
@@ -0,0 +1,96 @@
1
+ module Zookeeper
2
+ module SpecHelpers
3
+ class TimeoutError < StandardError; end
4
+ include Zookeeper::Constants
5
+ include Zookeeper::Logger
6
+
7
+ def ensure_node(zk, path, data)
8
+ return if zk.closed?
9
+ if zk.stat(:path => path)[:stat].exists?
10
+ zk.set(:path => path, :data => data)
11
+ else
12
+ zk.create(:path => path, :data => data)
13
+ end
14
+ end
15
+
16
+ def with_open_zk(host=nil)
17
+ z = Zookeeper.new(Zookeeper.default_cnx_str)
18
+ yield z
19
+ ensure
20
+ if z
21
+ unless z.closed?
22
+ z.close
23
+
24
+ wait_until do
25
+ begin
26
+ !z.connected?
27
+ rescue RuntimeError
28
+ true
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+
35
+ # this is not as safe as the one in ZK, just to be used to clean up
36
+ # when we're the only one adjusting a particular path
37
+ def rm_rf(z, path)
38
+ z.get_children(:path => path).tap do |h|
39
+ if h[:rc].zero?
40
+ h[:children].each do |child|
41
+ rm_rf(z, File.join(path, child))
42
+ end
43
+ elsif h[:rc] == ZNONODE
44
+ # no-op
45
+ else
46
+ raise "Oh noes! unexpected return value! #{h.inspect}"
47
+ end
48
+ end
49
+
50
+ rv = z.delete(:path => path)
51
+
52
+ unless (rv[:rc].zero? or rv[:rc] == ZNONODE)
53
+ raise "oh noes! failed to delete #{path}"
54
+ end
55
+
56
+ path
57
+ end
58
+
59
+ def mkdir_p(zk, path)
60
+ while true # loop because we can't retry
61
+ h = zk.create(:path => path, :data => '')
62
+ case h[:rc]
63
+ when ZOK, ZNODEEXISTS
64
+ return
65
+ when ZNONODE
66
+ parent = File.dirname(path)
67
+ raise "WTF? we wound up trying to create '/', something is screwed!" if parent == '/'
68
+ mkdir_p(zk, parent)
69
+ end
70
+ end
71
+ end
72
+
73
+ # method to wait until block passed returns true or timeout (default is 10 seconds) is reached
74
+ # raises TiemoutError on timeout
75
+ def wait_until(timeout=10)
76
+ time_to_stop = Time.now + timeout
77
+ while true
78
+ rval = yield
79
+ return rval if rval
80
+ raise TimeoutError, "timeout of #{timeout}s exceeded" if Time.now > time_to_stop
81
+ Thread.pass
82
+ end
83
+ end
84
+
85
+ # inverse of wait_until
86
+ def wait_while(timeout=10)
87
+ time_to_stop = Time.now + timeout
88
+ while true
89
+ rval = yield
90
+ return rval unless rval
91
+ raise TimeoutError, "timeout of #{timeout}s exceeded" if Time.now > time_to_stop
92
+ Thread.pass
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,24 @@
1
+ require 'spec_helper'
2
+ require 'shared/connection_examples'
3
+
4
+
5
+ describe 'Zookeeper' do
6
+ let(:path) { "/_zktest_" }
7
+ let(:data) { "underpants" }
8
+ let(:connection_string) { Zookeeper.default_cnx_str }
9
+
10
+ before do
11
+ @zk = Zookeeper.new(connection_string)
12
+ end
13
+
14
+ after do
15
+ @zk and @zk.close
16
+ end
17
+
18
+ def zk
19
+ @zk
20
+ end
21
+
22
+ it_should_behave_like "connection"
23
+ end
24
+
data/zookeeper.gemspec ADDED
@@ -0,0 +1,38 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require 'zookeeper/version'
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = 'zookeeper-ng'
7
+ s.version = Zookeeper::VERSION
8
+
9
+ s.authors = ["Ben Fritsch", "Phillip Pearson", "Eric Maland", "Evan Weaver", "Brian Wickman", "Neil Conway", "Jonathan D. Simms"]
10
+ s.email = ["ich@abwesend.com"]
11
+ s.summary = %q{Apache ZooKeeper driver for Rubies}
12
+ s.description = <<-EOS
13
+ A low-level multi-Ruby wrapper around the ZooKeeper API bindings. For a
14
+ friendlier interface, see http://github.com/slyphon/zk. Currently supported:
15
+ MRI: {1.8.7, 1.9.2, 1.9.3}, JRuby: ~> 1.6.7, Rubinius: 2.0.testing, REE 1.8.7.
16
+
17
+ This library uses version #{Zookeeper::DRIVER_VERSION} of zookeeper bindings.
18
+
19
+ EOS
20
+
21
+ s.homepage = 'https://github.com/slyphon/zookeeper'
22
+
23
+ s.files = `git ls-files`.split("\n")
24
+ s.require_paths = ["lib"]
25
+
26
+ if ENV['JAVA_GEM'] or defined?(::JRUBY_VERSION)
27
+ s.platform = 'java'
28
+ s.add_runtime_dependency('slyphon-log4j', '= 1.2.15')
29
+ s.add_runtime_dependency('slyphon-zookeeper_jar', '= 3.3.5')
30
+ s.require_paths += %w[java]
31
+ else
32
+ s.require_paths += %w[ext]
33
+ s.extensions = 'ext/extconf.rb'
34
+ end
35
+
36
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
37
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
38
+ end
@@ -0,0 +1,3 @@
1
+ #!/bin/sh
2
+
3
+ awk '{ if (prev == $0) count++; else { if (count > 0) { print " --- Line repeated " count " times --- " } print; prev = $0; count = 0 } }' $*
@@ -0,0 +1,194 @@
1
+ require 'zookeeper'
2
+ require 'zk-server'
3
+ require 'fileutils'
4
+ require 'tmpdir'
5
+
6
+ SLEEP_TIME = 30
7
+ STDOUT.sync = true
8
+
9
+ if ENV['DEBUG']
10
+ def zookeeper_logger(from)
11
+ l = Logger.new(STDOUT)
12
+ l.formatter = proc do |sev, time, c, msg|
13
+ "t=#{time.to_i} from=#{from} level=#{sev.downcase} message=#{msg.inspect}\n"
14
+ end
15
+ l
16
+ end
17
+
18
+ Zookeeper.logger = zookeeper_logger('zookeeper')
19
+ Zookeeper.set_debug_level(Zookeeper::ZOO_LOG_LEVEL_DEBUG)
20
+ end
21
+
22
+ class Worker
23
+ def initialize(body = nil, &block)
24
+ raise ArgumentError, "Cannot include both body and block" if body && block
25
+ @body = body || block
26
+ end
27
+
28
+ def body
29
+ @body || method(:call)
30
+ end
31
+
32
+ def start
33
+ @thread = Thread.new do
34
+ Thread.current.abort_on_exception = true
35
+ body.call
36
+ end
37
+ end
38
+
39
+ def stop
40
+ if @thread
41
+ @thread.kill
42
+ @thread = nil
43
+ end
44
+ end
45
+
46
+ def join
47
+ if @thread
48
+ @thread.join
49
+ end
50
+ end
51
+ end
52
+
53
+
54
+
55
+ base_dir = Dir.mktmpdir('zk-server-cluster')
56
+ num_cluster = 3
57
+ cluster = ZK::Server::Cluster.new(num_cluster, :base_dir => base_dir)
58
+
59
+ class Reader < Worker
60
+ attr_reader :client
61
+
62
+ def initialize(zookeeper_hosts)
63
+ @zookeeper_hosts = zookeeper_hosts
64
+ @log_from = :reader
65
+ end
66
+
67
+ def call
68
+ @client = Zookeeper.new(@zookeeper_hosts, 10, method(:watcher))
69
+ client.wait_until_connected
70
+
71
+ client.create(:path => "/test", :data => '') rescue client.set(:path => "/test", :data => '')
72
+
73
+ while true
74
+ error = nil
75
+ t = Benchmark.realtime do
76
+ begin
77
+ client.get(:path => "/test")
78
+ rescue => e
79
+ error = e
80
+ end
81
+ end
82
+
83
+ msg = "host=#{client.connected_host || 'nil'} session_id=#{client.session_id} state=#{client.state_by_value(client.state)} time=#{"%0.4f" % t}"
84
+ if error
85
+ msg << " error=#{error.class} error_message=#{error.to_s.inspect}"
86
+ msg << " closed=#{client.closed?} running=#{client.running?} shutting_down=#{client.shutting_down?}"
87
+ end
88
+
89
+ log msg
90
+
91
+ sleep 1
92
+ end
93
+ end
94
+
95
+ def log(message)
96
+ puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
97
+ end
98
+
99
+ def watcher(event)
100
+ if event[:state] == Zookeeper::ZOO_EXPIRED_SESSION_STATE
101
+ if client
102
+ log "action=reconnecting state=#{client.state_by_value(event[:state])} session_id=#{client.session_id}"
103
+ client.reopen
104
+ end
105
+ end
106
+
107
+ end
108
+ end
109
+
110
+ class Writer < Worker
111
+ def initialize(zookeeper_hosts)
112
+ @zookeeper_hosts = zookeeper_hosts
113
+ @log_from = :writer
114
+ end
115
+
116
+ def call
117
+ client = Zookeeper.new(@zookeeper_hosts)
118
+ client.wait_until_connected
119
+
120
+ while true
121
+ error = nil
122
+ t = Benchmark.realtime do
123
+ begin
124
+ client.create(:path => "/test", :data => '') rescue client.set(:path => "/test", :data => '')
125
+ rescue => e
126
+ error = e
127
+ end
128
+ end
129
+
130
+ msg = "host=#{client.connected_host || 'nil'} session_id=#{client.session_id} state=#{client.state_by_value(client.state)} time=#{"%0.4f" % t}"
131
+ msg << " error=#{error.class} error_message=#{error.to_s.inspect}" if error
132
+ log msg
133
+
134
+ sleep 1
135
+ end
136
+ end
137
+
138
+ def log(message)
139
+ puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
140
+ end
141
+ end
142
+
143
+ class ZooMonkey < Worker
144
+ attr_reader :cluster
145
+
146
+ def initialize(cluster)
147
+ @cluster = cluster
148
+ @log_from = :server
149
+ end
150
+
151
+ def call
152
+ while true
153
+ sleep SLEEP_TIME
154
+
155
+ cluster.processes.each do |server|
156
+ host = "127.0.0.1:#{server.client_port}"
157
+ log "host=#{host} pid=#{server.pid} action=pausing"
158
+ server.kill "STOP"
159
+ sleep SLEEP_TIME
160
+
161
+ log "host=#{host} pid=#{server.pid} action=resuming"
162
+ server.kill "CONT"
163
+ sleep SLEEP_TIME
164
+ end
165
+ end
166
+ end
167
+
168
+ def log(message)
169
+ puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
170
+ end
171
+ end
172
+
173
+ begin
174
+ cluster.run
175
+
176
+ zookeeper_hosts = cluster.processes.map { |p| "127.0.0.1:#{p.client_port}" }
177
+ zookeeper_spec = (zookeeper_hosts * 2).join(',')
178
+
179
+ reader = Reader.new(zookeeper_spec)
180
+ reader.start
181
+
182
+ # writer = Writer.new(zookeeper_spec)
183
+ # writer.start
184
+
185
+ monkey = ZooMonkey.new(cluster)
186
+ monkey.start
187
+
188
+ reader.join
189
+ writer.join
190
+ monkey.join
191
+ ensure
192
+ cluster.clobber!
193
+ FileUtils.remove_entry(base_dir)
194
+ end