zookeeper-ng 1.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +7 -0
  2. data/.ctags_paths +1 -0
  3. data/.dotfiles/ruby-gemset +1 -0
  4. data/.dotfiles/ruby-version +1 -0
  5. data/.dotfiles/rvmrc +2 -0
  6. data/.gitignore +19 -0
  7. data/.gitmodules +3 -0
  8. data/.travis.yml +25 -0
  9. data/CHANGELOG +395 -0
  10. data/Gemfile +30 -0
  11. data/Guardfile +8 -0
  12. data/LICENSE +23 -0
  13. data/Manifest +29 -0
  14. data/README.markdown +85 -0
  15. data/Rakefile +121 -0
  16. data/cause-abort.rb +117 -0
  17. data/ext/.gitignore +6 -0
  18. data/ext/Rakefile +41 -0
  19. data/ext/c_zookeeper.rb +398 -0
  20. data/ext/common.h +17 -0
  21. data/ext/dbg.h +53 -0
  22. data/ext/depend +5 -0
  23. data/ext/event_lib.c +740 -0
  24. data/ext/event_lib.h +175 -0
  25. data/ext/extconf.rb +103 -0
  26. data/ext/generate_gvl_code.rb +321 -0
  27. data/ext/patches/zkc-3.3.5-network.patch +24 -0
  28. data/ext/patches/zkc-3.4.5-fetch-and-add.patch +16 -0
  29. data/ext/patches/zkc-3.4.5-logging.patch +41 -0
  30. data/ext/patches/zkc-3.4.5-out-of-order-ping.patch +163 -0
  31. data/ext/patches/zkc-3.4.5-overflow.patch +11 -0
  32. data/ext/patches/zkc-3.4.5-yosemite-htonl-fix.patch +102 -0
  33. data/ext/zkc-3.4.5.tar.gz +0 -0
  34. data/ext/zkrb.c +1075 -0
  35. data/ext/zkrb_wrapper.c +775 -0
  36. data/ext/zkrb_wrapper.h +350 -0
  37. data/ext/zkrb_wrapper_compat.c +15 -0
  38. data/ext/zkrb_wrapper_compat.h +11 -0
  39. data/ext/zookeeper_base.rb +256 -0
  40. data/java/java_base.rb +503 -0
  41. data/lib/zookeeper.rb +115 -0
  42. data/lib/zookeeper/acls.rb +44 -0
  43. data/lib/zookeeper/callbacks.rb +108 -0
  44. data/lib/zookeeper/client.rb +30 -0
  45. data/lib/zookeeper/client_methods.rb +282 -0
  46. data/lib/zookeeper/common.rb +122 -0
  47. data/lib/zookeeper/common/queue_with_pipe.rb +110 -0
  48. data/lib/zookeeper/compatibility.rb +138 -0
  49. data/lib/zookeeper/constants.rb +97 -0
  50. data/lib/zookeeper/continuation.rb +223 -0
  51. data/lib/zookeeper/core_ext.rb +58 -0
  52. data/lib/zookeeper/em_client.rb +55 -0
  53. data/lib/zookeeper/exceptions.rb +135 -0
  54. data/lib/zookeeper/forked.rb +19 -0
  55. data/lib/zookeeper/latch.rb +34 -0
  56. data/lib/zookeeper/logger.rb +39 -0
  57. data/lib/zookeeper/logger/forwarding_logger.rb +84 -0
  58. data/lib/zookeeper/monitor.rb +19 -0
  59. data/lib/zookeeper/rake_tasks.rb +165 -0
  60. data/lib/zookeeper/request_registry.rb +153 -0
  61. data/lib/zookeeper/stat.rb +21 -0
  62. data/lib/zookeeper/version.rb +4 -0
  63. data/notes.txt +14 -0
  64. data/scripts/upgrade-1.0-sed-alike.rb +46 -0
  65. data/spec/c_zookeeper_spec.rb +51 -0
  66. data/spec/chrooted_connection_spec.rb +83 -0
  67. data/spec/compatibilty_spec.rb +8 -0
  68. data/spec/default_watcher_spec.rb +41 -0
  69. data/spec/em_spec.rb +51 -0
  70. data/spec/ext/zookeeper_base_spec.rb +19 -0
  71. data/spec/forked_connection_spec.rb +124 -0
  72. data/spec/latch_spec.rb +24 -0
  73. data/spec/log4j.properties +17 -0
  74. data/spec/shared/all_success_return_values.rb +10 -0
  75. data/spec/shared/connection_examples.rb +1077 -0
  76. data/spec/spec_helper.rb +61 -0
  77. data/spec/support/00_logging.rb +38 -0
  78. data/spec/support/10_spawn_zookeeper.rb +24 -0
  79. data/spec/support/progress_formatter.rb +15 -0
  80. data/spec/support/zookeeper_spec_helpers.rb +96 -0
  81. data/spec/zookeeper_spec.rb +24 -0
  82. data/zookeeper.gemspec +38 -0
  83. data/zoomonkey/duplicates +3 -0
  84. data/zoomonkey/zoomonkey.rb +194 -0
  85. metadata +157 -0
@@ -0,0 +1,61 @@
1
+ $LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
2
+ $LOAD_PATH.unshift(File.expand_path('../../ext', __FILE__))
3
+ $LOAD_PATH.uniq!
4
+
5
+ require 'rubygems'
6
+
7
+ release_ops_path = File.expand_path('../../releaseops/lib', __FILE__)
8
+
9
+ if File.exists?(release_ops_path)
10
+ require File.join(release_ops_path, 'releaseops')
11
+ ReleaseOps::SimpleCov.maybe_start
12
+ end
13
+
14
+ require 'zookeeper'
15
+
16
+ Dir[File.expand_path('../support/**/*.rb', __FILE__)].sort.each { |f| require(f) }
17
+
18
+ if ENV['ZKRB_DEBUG']
19
+ Zookeeper.set_debug_level(4)
20
+ end
21
+
22
+ if ENV['ZKRB_NOLOG']
23
+ SpecGlobalLogger.logger.level = ::Logger::FATAL
24
+ Zookeeper.set_debug_level(0)
25
+ end
26
+
27
+ RSpec.configure do |config|
28
+ config.mock_with :rspec
29
+ [Zookeeper::SpecHelpers, SpecGlobalLogger].each do |mod|
30
+ config.include(mod)
31
+ config.extend(mod)
32
+ end
33
+
34
+ if Zookeeper.spawn_zookeeper?
35
+ require 'zk-server'
36
+
37
+ config.before(:suite) do
38
+ SpecGlobalLogger.logger.debug { "Starting zookeeper service" }
39
+ ZK::Server.run do |c|
40
+ c.base_dir = File.expand_path('../../.zkserver', __FILE__)
41
+ c.client_port = Zookeeper.test_port
42
+ c.force_sync = false
43
+ c.snap_count = 1_000_000
44
+ end
45
+ end
46
+
47
+ config.after(:suite) do
48
+ SpecGlobalLogger.logger.debug { "stopping zookeeper service" }
49
+ ZK::Server.shutdown
50
+ end
51
+ end
52
+ end
53
+
54
+ require 'pp'
55
+
56
+ if RUBY_VERSION == '1.9.3'
57
+ trap('USR1') do
58
+ threads = Thread.list.map { |th| th.backtrace }
59
+ pp threads
60
+ end
61
+ end
@@ -0,0 +1,38 @@
1
+ module Zookeeper
2
+ TEST_LOG_PATH = File.expand_path('../../../test.log', __FILE__)
3
+
4
+ def self.setup_test_logger
5
+ log =
6
+ if (ENV['ZOOKEEPER_DEBUG'] || ENV['ZKRB_DEBUG'])
7
+ ::Logger.new(STDERR)
8
+ else
9
+ ::Logger.new(TEST_LOG_PATH)
10
+ end
11
+
12
+ log.level = ::Logger::DEBUG
13
+
14
+ Zookeeper::Logger.wrapped_logger = log
15
+ end
16
+ end
17
+
18
+ Zookeeper.setup_test_logger
19
+
20
+ module SpecGlobalLogger
21
+ extend self
22
+
23
+ def logger
24
+ @spec_global_logger ||= Zookeeper::Logger::ForwardingLogger.for(Zookeeper::Logger.wrapped_logger, 'spec')
25
+ end
26
+
27
+ # sets the log level to FATAL for the duration of the block
28
+ def mute_logger
29
+ zk_log = Zookeeper::Logger.wrapped_logger
30
+
31
+ orig_level, zk_log.level = zk_log.level, ::Logger::FATAL
32
+ orig_zk_level, Zookeeper.debug_level = Zookeeper.debug_level, Zookeeper::Constants::ZOO_LOG_LEVEL_ERROR
33
+ yield
34
+ ensure
35
+ zk_log.level = orig_zk_level
36
+ end
37
+ end
38
+
@@ -0,0 +1,24 @@
1
+ module Zookeeper
2
+ def self.spawn_zookeeper?
3
+ !!ENV['SPAWN_ZOOKEEPER']
4
+ end
5
+
6
+ def self.travis?
7
+ !!ENV['TRAVIS']
8
+ end
9
+
10
+ def self.default_cnx_host
11
+ ENV['ZK_DEFAULT_HOST'] || 'localhost'
12
+ end
13
+
14
+ @test_port ||= spawn_zookeeper? ? 21811 : 2181
15
+
16
+ class << self
17
+ attr_accessor :test_port
18
+ end
19
+
20
+ def self.default_cnx_str
21
+ "#{default_cnx_host}:#{test_port}"
22
+ end
23
+ end
24
+
@@ -0,0 +1,15 @@
1
+ require 'rspec/core/formatters/progress_formatter'
2
+
3
+ module RSpec
4
+ module Core
5
+ module Formatters
6
+ class ProgressFormatter
7
+ def example_started(example)
8
+ SpecGlobalLogger.logger << pending_color("\n=====<([ #{example.full_description} ])>=====\n")
9
+ super(example)
10
+ end
11
+ end
12
+ end
13
+ end
14
+ end
15
+
@@ -0,0 +1,96 @@
1
+ module Zookeeper
2
+ module SpecHelpers
3
+ class TimeoutError < StandardError; end
4
+ include Zookeeper::Constants
5
+ include Zookeeper::Logger
6
+
7
+ def ensure_node(zk, path, data)
8
+ return if zk.closed?
9
+ if zk.stat(:path => path)[:stat].exists?
10
+ zk.set(:path => path, :data => data)
11
+ else
12
+ zk.create(:path => path, :data => data)
13
+ end
14
+ end
15
+
16
+ def with_open_zk(host=nil)
17
+ z = Zookeeper.new(Zookeeper.default_cnx_str)
18
+ yield z
19
+ ensure
20
+ if z
21
+ unless z.closed?
22
+ z.close
23
+
24
+ wait_until do
25
+ begin
26
+ !z.connected?
27
+ rescue RuntimeError
28
+ true
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
34
+
35
+ # this is not as safe as the one in ZK, just to be used to clean up
36
+ # when we're the only one adjusting a particular path
37
+ def rm_rf(z, path)
38
+ z.get_children(:path => path).tap do |h|
39
+ if h[:rc].zero?
40
+ h[:children].each do |child|
41
+ rm_rf(z, File.join(path, child))
42
+ end
43
+ elsif h[:rc] == ZNONODE
44
+ # no-op
45
+ else
46
+ raise "Oh noes! unexpected return value! #{h.inspect}"
47
+ end
48
+ end
49
+
50
+ rv = z.delete(:path => path)
51
+
52
+ unless (rv[:rc].zero? or rv[:rc] == ZNONODE)
53
+ raise "oh noes! failed to delete #{path}"
54
+ end
55
+
56
+ path
57
+ end
58
+
59
+ def mkdir_p(zk, path)
60
+ while true # loop because we can't retry
61
+ h = zk.create(:path => path, :data => '')
62
+ case h[:rc]
63
+ when ZOK, ZNODEEXISTS
64
+ return
65
+ when ZNONODE
66
+ parent = File.dirname(path)
67
+ raise "WTF? we wound up trying to create '/', something is screwed!" if parent == '/'
68
+ mkdir_p(zk, parent)
69
+ end
70
+ end
71
+ end
72
+
73
+ # method to wait until block passed returns true or timeout (default is 10 seconds) is reached
74
+ # raises TiemoutError on timeout
75
+ def wait_until(timeout=10)
76
+ time_to_stop = Time.now + timeout
77
+ while true
78
+ rval = yield
79
+ return rval if rval
80
+ raise TimeoutError, "timeout of #{timeout}s exceeded" if Time.now > time_to_stop
81
+ Thread.pass
82
+ end
83
+ end
84
+
85
+ # inverse of wait_until
86
+ def wait_while(timeout=10)
87
+ time_to_stop = Time.now + timeout
88
+ while true
89
+ rval = yield
90
+ return rval unless rval
91
+ raise TimeoutError, "timeout of #{timeout}s exceeded" if Time.now > time_to_stop
92
+ Thread.pass
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,24 @@
1
+ require 'spec_helper'
2
+ require 'shared/connection_examples'
3
+
4
+
5
+ describe 'Zookeeper' do
6
+ let(:path) { "/_zktest_" }
7
+ let(:data) { "underpants" }
8
+ let(:connection_string) { Zookeeper.default_cnx_str }
9
+
10
+ before do
11
+ @zk = Zookeeper.new(connection_string)
12
+ end
13
+
14
+ after do
15
+ @zk and @zk.close
16
+ end
17
+
18
+ def zk
19
+ @zk
20
+ end
21
+
22
+ it_should_behave_like "connection"
23
+ end
24
+
data/zookeeper.gemspec ADDED
@@ -0,0 +1,38 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require 'zookeeper/version'
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = 'zookeeper-ng'
7
+ s.version = Zookeeper::VERSION
8
+
9
+ s.authors = ["Ben Fritsch", "Phillip Pearson", "Eric Maland", "Evan Weaver", "Brian Wickman", "Neil Conway", "Jonathan D. Simms"]
10
+ s.email = ["ich@abwesend.com"]
11
+ s.summary = %q{Apache ZooKeeper driver for Rubies}
12
+ s.description = <<-EOS
13
+ A low-level multi-Ruby wrapper around the ZooKeeper API bindings. For a
14
+ friendlier interface, see http://github.com/slyphon/zk. Currently supported:
15
+ MRI: {1.8.7, 1.9.2, 1.9.3}, JRuby: ~> 1.6.7, Rubinius: 2.0.testing, REE 1.8.7.
16
+
17
+ This library uses version #{Zookeeper::DRIVER_VERSION} of zookeeper bindings.
18
+
19
+ EOS
20
+
21
+ s.homepage = 'https://github.com/slyphon/zookeeper'
22
+
23
+ s.files = `git ls-files`.split("\n")
24
+ s.require_paths = ["lib"]
25
+
26
+ if ENV['JAVA_GEM'] or defined?(::JRUBY_VERSION)
27
+ s.platform = 'java'
28
+ s.add_runtime_dependency('slyphon-log4j', '= 1.2.15')
29
+ s.add_runtime_dependency('slyphon-zookeeper_jar', '= 3.3.5')
30
+ s.require_paths += %w[java]
31
+ else
32
+ s.require_paths += %w[ext]
33
+ s.extensions = 'ext/extconf.rb'
34
+ end
35
+
36
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
37
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
38
+ end
@@ -0,0 +1,3 @@
1
+ #!/bin/sh
2
+
3
+ awk '{ if (prev == $0) count++; else { if (count > 0) { print " --- Line repeated " count " times --- " } print; prev = $0; count = 0 } }' $*
@@ -0,0 +1,194 @@
1
+ require 'zookeeper'
2
+ require 'zk-server'
3
+ require 'fileutils'
4
+ require 'tmpdir'
5
+
6
+ SLEEP_TIME = 30
7
+ STDOUT.sync = true
8
+
9
+ if ENV['DEBUG']
10
+ def zookeeper_logger(from)
11
+ l = Logger.new(STDOUT)
12
+ l.formatter = proc do |sev, time, c, msg|
13
+ "t=#{time.to_i} from=#{from} level=#{sev.downcase} message=#{msg.inspect}\n"
14
+ end
15
+ l
16
+ end
17
+
18
+ Zookeeper.logger = zookeeper_logger('zookeeper')
19
+ Zookeeper.set_debug_level(Zookeeper::ZOO_LOG_LEVEL_DEBUG)
20
+ end
21
+
22
+ class Worker
23
+ def initialize(body = nil, &block)
24
+ raise ArgumentError, "Cannot include both body and block" if body && block
25
+ @body = body || block
26
+ end
27
+
28
+ def body
29
+ @body || method(:call)
30
+ end
31
+
32
+ def start
33
+ @thread = Thread.new do
34
+ Thread.current.abort_on_exception = true
35
+ body.call
36
+ end
37
+ end
38
+
39
+ def stop
40
+ if @thread
41
+ @thread.kill
42
+ @thread = nil
43
+ end
44
+ end
45
+
46
+ def join
47
+ if @thread
48
+ @thread.join
49
+ end
50
+ end
51
+ end
52
+
53
+
54
+
55
+ base_dir = Dir.mktmpdir('zk-server-cluster')
56
+ num_cluster = 3
57
+ cluster = ZK::Server::Cluster.new(num_cluster, :base_dir => base_dir)
58
+
59
+ class Reader < Worker
60
+ attr_reader :client
61
+
62
+ def initialize(zookeeper_hosts)
63
+ @zookeeper_hosts = zookeeper_hosts
64
+ @log_from = :reader
65
+ end
66
+
67
+ def call
68
+ @client = Zookeeper.new(@zookeeper_hosts, 10, method(:watcher))
69
+ client.wait_until_connected
70
+
71
+ client.create(:path => "/test", :data => '') rescue client.set(:path => "/test", :data => '')
72
+
73
+ while true
74
+ error = nil
75
+ t = Benchmark.realtime do
76
+ begin
77
+ client.get(:path => "/test")
78
+ rescue => e
79
+ error = e
80
+ end
81
+ end
82
+
83
+ msg = "host=#{client.connected_host || 'nil'} session_id=#{client.session_id} state=#{client.state_by_value(client.state)} time=#{"%0.4f" % t}"
84
+ if error
85
+ msg << " error=#{error.class} error_message=#{error.to_s.inspect}"
86
+ msg << " closed=#{client.closed?} running=#{client.running?} shutting_down=#{client.shutting_down?}"
87
+ end
88
+
89
+ log msg
90
+
91
+ sleep 1
92
+ end
93
+ end
94
+
95
+ def log(message)
96
+ puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
97
+ end
98
+
99
+ def watcher(event)
100
+ if event[:state] == Zookeeper::ZOO_EXPIRED_SESSION_STATE
101
+ if client
102
+ log "action=reconnecting state=#{client.state_by_value(event[:state])} session_id=#{client.session_id}"
103
+ client.reopen
104
+ end
105
+ end
106
+
107
+ end
108
+ end
109
+
110
+ class Writer < Worker
111
+ def initialize(zookeeper_hosts)
112
+ @zookeeper_hosts = zookeeper_hosts
113
+ @log_from = :writer
114
+ end
115
+
116
+ def call
117
+ client = Zookeeper.new(@zookeeper_hosts)
118
+ client.wait_until_connected
119
+
120
+ while true
121
+ error = nil
122
+ t = Benchmark.realtime do
123
+ begin
124
+ client.create(:path => "/test", :data => '') rescue client.set(:path => "/test", :data => '')
125
+ rescue => e
126
+ error = e
127
+ end
128
+ end
129
+
130
+ msg = "host=#{client.connected_host || 'nil'} session_id=#{client.session_id} state=#{client.state_by_value(client.state)} time=#{"%0.4f" % t}"
131
+ msg << " error=#{error.class} error_message=#{error.to_s.inspect}" if error
132
+ log msg
133
+
134
+ sleep 1
135
+ end
136
+ end
137
+
138
+ def log(message)
139
+ puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
140
+ end
141
+ end
142
+
143
+ class ZooMonkey < Worker
144
+ attr_reader :cluster
145
+
146
+ def initialize(cluster)
147
+ @cluster = cluster
148
+ @log_from = :server
149
+ end
150
+
151
+ def call
152
+ while true
153
+ sleep SLEEP_TIME
154
+
155
+ cluster.processes.each do |server|
156
+ host = "127.0.0.1:#{server.client_port}"
157
+ log "host=#{host} pid=#{server.pid} action=pausing"
158
+ server.kill "STOP"
159
+ sleep SLEEP_TIME
160
+
161
+ log "host=#{host} pid=#{server.pid} action=resuming"
162
+ server.kill "CONT"
163
+ sleep SLEEP_TIME
164
+ end
165
+ end
166
+ end
167
+
168
+ def log(message)
169
+ puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
170
+ end
171
+ end
172
+
173
+ begin
174
+ cluster.run
175
+
176
+ zookeeper_hosts = cluster.processes.map { |p| "127.0.0.1:#{p.client_port}" }
177
+ zookeeper_spec = (zookeeper_hosts * 2).join(',')
178
+
179
+ reader = Reader.new(zookeeper_spec)
180
+ reader.start
181
+
182
+ # writer = Writer.new(zookeeper_spec)
183
+ # writer.start
184
+
185
+ monkey = ZooMonkey.new(cluster)
186
+ monkey.start
187
+
188
+ reader.join
189
+ writer.join
190
+ monkey.join
191
+ ensure
192
+ cluster.clobber!
193
+ FileUtils.remove_entry(base_dir)
194
+ end