zookeeper 0.4.4 → 0.9.3

data/.gitignore

@@ -0,0 +1,10 @@
+*.gem
+*.o
+*.so
+*~
+*.log
+Makefile
+pkg
+zookeeper.gemspec
+Gemfile.*
+tmp/

data/CHANGELOG

@@ -1,3 +1,97 @@
+v0.9.3 Event thread shutdown fix, Windows compatibility fix
+
+  * Use a 'shutdown thread' to coordinate cleanup if close is called from the
+    event thread (prevents deadlock)
+
+  * Default Logger now uses $stderr instead of opening /dev/null [#16]
+
+  * Gemfile/gemspec/Rakefile refactoring.
+
+v0.9.2 More efficient and simpler wrappers for GIL release
+
+  * After a code review by Andrew Wason (rectalogic), use a much simpler
+    technique for creating the arg structs and passing them to the
+    zkrb_gvl_* functions. No malloc(), no free(), no problem.
+
+v0.9.1 see v0.8.4 notes, same patch
+
+v0.9.0 RELEASE THE KRAK..er, GIL!!
+
+  * In >= 1.9.2 the ruby interpreter allows you to release the GIL when
+    calling into native code, sounds like a good idea.
+    
+    This release makes use of that code by parsing the zookeeper.h header file
+    and extracting the method signatures of all relevant zoo_* functions, then
+    generating boilerplate that allows us to call those functions via the
+    rb_thread_blocking_region function.
+
+    1.8.7 compatibility is maintained by stubbing out that functionality if built
+    under 1.8.7. 
+
+  * 1.8.7 is deprecated! I will continue to support 1.8.7 for the near future
+    but sometime soon, you're gonna have to upgrade.
+
+v0.8.4 fix NameError, require 'forwardable'
+  
+  * Really not sure why this didn't come up in tests
+
+  * issue here https://github.com/slyphon/zk/issues/22
+
+v0.8.3 fix NonLocalJump exception in event delivery thread shutdown code
+
+  * hit a corner case where we're waiting for the zkc handle setup
+    and the user decides to shutdown, but before we've had a chance
+    to enter the delivery loop.
+
+  * Cleaned up some nasty code in ZookeeperConstants
+
+  * removed ZookeeperConstants#print_events and ZookeeperConstants#print_states
+
+  * changed EVENT_TYPE_NAMES and EVENT_STATE_NAMES in ZookeeperConstants
+    to use string values instead of symbols
+
+v0.8.2 fix close after a fork()
+
+  * The dispatch thread will be dead in this situation, so we need to 
+    check to see if it's already dead before waiting on it to exit.
+
+v0.8.1 Java client fix, silence warnings
+
+v0.8.0 Refactor C implementaion, EventMachine client
+
+  * separated CZookeeper and ZookeeperBase implementation
+  
+    This solves issues with reopen not working properly, makes for a much
+    cleaner event delivery implementation. ZookeeperBase controls the lifecycle
+    of the event dispatch thread now, rather than it being tied to CZookeeper.
+  
+  * added support for the 'sync' API call
+  
+  * Refactored zookeeper_c.c and zookeeper_lib.c
+  
+    More error checking in zookeeper_lib.c and restructure some things to make
+    logic easier to follow
+  
+    Fix bug in method_get_next_event that made the shutdown case so complicated
+  
+  * Massively simplified EMClient implementation
+
+    Rather than trying to hook the IO used by zookeeper_lib to notify zookeeper_c
+    about event availabiltiy directly into EventMachine, use the same event delivery
+    thread, but wrap the dispatch call in EM.schedule.
+
+  * Improve implementation of spin-lock-esque code that waits for the connection to be
+    established before returning.
+
+    This cut the test runtime down from 1m 20s to 2s.
+
+  * Java client refactoring, similar correctness changes
+
+  * Change ZookeeperException base class to StandardError instead of Exception
+
+
+v0.4.5 Upgrade to ZooKeeper 3.3.3
+
 v0.4.4 Fix race condition on close, possible data corruption on async get.
 
 v0.4.3 Fix a handful of memory-related bugs, fix SIGSEGV on master change, reduce latency of event handling, fix compilation on OSX.
@@ -22,3 +116,4 @@ v0.2. Bundle C dependencies, like memcached.gem.
 
 v0.1. First release.
 
+# vim:ft=text:ts=2:sw=2:et

data/Gemfile

@@ -0,0 +1,17 @@
+source :rubygems
+
+gemspec
+
+group :test do
+  gem "rspec", "~> 2.8.0"
+  gem 'flexmock', '~> 0.8.11'
+  gem 'eventmachine', '1.0.0.beta.4'
+  gem 'evented-spec', '~> 0.9.0'
+end
+
+group :development do
+  gem 'rake', '~> 0.9.0'
+  gem 'pry'
+end
+
+# vim:ft=ruby

data/Manifest

@@ -1,23 +1,29 @@
 CHANGELOG
 LICENSE
+Manifest
 README
 Rakefile
 examples/cloud_config.rb
 ext/extconf.rb
-ext/zkc-3.3.2.tar.gz
+ext/zkc-3.3.4.tar.gz
+ext/zookeeper_base.rb
 ext/zookeeper_c.c
 ext/zookeeper_lib.c
 ext/zookeeper_lib.h
+java/zookeeper_base.rb
 lib/zookeeper.rb
 lib/zookeeper/acls.rb
 lib/zookeeper/callbacks.rb
+lib/zookeeper/common.rb
 lib/zookeeper/constants.rb
 lib/zookeeper/exceptions.rb
 lib/zookeeper/stat.rb
+spec/log4j.properties
+spec/spec_helper.rb
+spec/zookeeper_spec.rb
 test/test_basic.rb
 test/test_callback1.rb
 test/test_close.rb
 test/test_esoteric.rb
 test/test_watcher1.rb
 test/test_watcher2.rb
-Manifest

data/README.markdown

@@ -0,0 +1,59 @@
+# zookeeper #
+
+An interface to the Zookeeper cluster coordination server.
+
+For a higher-level interface with a more convenient API and features such as locks, have a look at [ZK](https://github.com/slyphon/zk) (also available is [ZK-EventMachine](https://github.com/slyphon/zk-eventmachine) for those who prefer async).
+
+## Big Plans for 1.0 ##
+
+The 1.0 release will feature a reorganization of the heirarchy. There will be a single top-level `Zookeeper` namespace (as opposed to the current layout, with 5-6 different top-level constants), and for the next several releases, there will be a backwards compatible require for users that still need to use the old names.
+
+## License
+
+Copyright 2008 Phillip Pearson, and 2010 Twitter, Inc. 
+Licensed under the MIT License.  See the included LICENSE file.  
+
+Portions copyright 2008-2010 the Apache Software Foundation, licensed under the
+Apache 2 license, and used with permission.
+
+Portions contributed to the open source community by HPDC, L.P.
+
+## Install
+
+sudo gem install zookeeper
+
+## Usage
+
+Connect to a server:
+
+	require 'rubygems'
+	require 'zookeeper'
+	z = Zookeeper.new("localhost:2181")
+	z.get_children(:path => "/")
+
+## Idioms
+
+The following methods are initially supported:
+* get
+* set
+* get\_children
+* stat
+* create
+* delete
+* get\_acl
+* set\_acl
+
+All support async callbacks.  get, get\_children and stat support both watchers and callbacks.
+
+Calls take a dictionary of parameters.  With the exception of set\_acl, the only required parameter is :path.  Each call returns a dictionary with at minimum two keys :req\_id and :rc.
+
+### A Bit about this repository ###
+
+Twitter's open source office was kind enough to transfer this repository to facilitate development and administration of this repository. The `zookeeper` gem's last three releases were recorded in branches `v0.4.2`, `v0.4.3` and `v0.4.4`. Releases of the `slyphon-zookeeper` gem were cut off of the fork, and unfortunately (due to an oversight on my part) were tagged with unrelated versions. Those were tagged with names `release/0.9.2`. 
+
+The plan is to keep the `slyphon-zookeeper` tags, and to tag the `zookeeper` releases `twitter/release/0.4.x`.
+
+Further work will be carried out on this repository. The `0.9.3` release of the zookeeper gem will be released under the 'zookeeper' name, and will bring the two divergent (conceptual) branches of development together.
+
+
+

data/Rakefile

@@ -1,9 +1,139 @@
-require 'echoe'
+# def gemset_name
+#   ENV.fetch('GEM_HOME').split('@').last
+# end
 
-Echoe.new("zookeeper") do |p|
-  p.author = "Phillip Pearson, Eric Maland, Evan Weaver, Brian Wickman"
-  p.project = "twitter"
-  p.summary = "An interface to the Zookeeper distributed configuration server."
-  p.clean_pattern += ["ext/lib", "ext/include", "ext/c", "ext/bin", "ext/conftest.dSYM"]
-  p.rdoc_pattern = /README|TODO|LICENSE|CHANGELOG|BENCH|COMPAT|zookeeper_c.c|zookeeper.rb/
+GEM_FILES = FileList['slyphon-zookeeper-*.gem']
+
+namespace :mb do
+  namespace :gems do
+    task :build do
+      sh "rvm 1.8.7 do gem build zookeeper.gemspec"
+      ENV['JAVA_GEM'] = '1'
+      sh "rvm 1.8.7 do gem build zookeeper.gemspec"
+    end
+
+    task :push do
+      GEM_FILES.each do |gem|
+        sh "gem push #{gem}"
+      end
+    end
+
+    task :clean do
+      rm_rf GEM_FILES
+    end
+
+    task :all => [:build, :push, :clean]
+  end
 end
+
+gemset_name = 'zookeeper'
+
+# this nonsense w/ tmp and the Gemfile is a bundler optimization
+
+%w[1.8.7 1.9.2 jruby rbx 1.9.3].each do |ns_name|
+  rvm_ruby = (ns_name == 'rbx') ? "rbx-2.0.testing" : ns_name
+
+  ruby_with_gemset = "#{rvm_ruby}@#{gemset_name}"
+
+  create_gemset_name  = "mb:#{ns_name}:create_gemset"
+  clobber_task_name   = "mb:#{ns_name}:clobber"
+  clean_task_name     = "mb:#{ns_name}:clean"
+  build_task_name     = "mb:#{ns_name}:build"
+  bundle_task_name    = "mb:#{ns_name}:bundle_install"
+  rspec_task_name     = "mb:#{ns_name}:run_rspec"
+
+  phony_gemfile_link_name = "Gemfile.#{ns_name}"
+  phony_gemfile_lock_name = "#{phony_gemfile_link_name}.lock"
+
+  file phony_gemfile_link_name do
+    # apparently, rake doesn't deal with symlinks intelligently :P
+    ln_s('Gemfile', phony_gemfile_link_name) unless File.symlink?(phony_gemfile_link_name)
+  end
+
+  task :clean do
+    rm_rf [phony_gemfile_lock_name, phony_gemfile_lock_name]
+  end
+
+  task create_gemset_name do
+    sh "rvm #{rvm_ruby} do rvm gemset create #{gemset_name}"
+  end
+
+  task clobber_task_name do
+    unless rvm_ruby == 'jruby'
+      cd 'ext' do
+        sh "rake clobber"
+      end
+    end
+  end
+
+  task clean_task_name do
+    unless rvm_ruby == 'jruby'
+      cd 'ext' do
+        sh "rake clean"
+      end
+    end
+  end
+
+  task build_task_name => [create_gemset_name, clean_task_name] do
+    unless rvm_ruby == 'jruby'
+      cd 'ext' do
+        sh "rvm #{ruby_with_gemset} do rake build"
+      end
+    end
+  end
+
+  task bundle_task_name => [phony_gemfile_link_name, build_task_name] do
+    sh "rvm #{ruby_with_gemset} do bundle install --gemfile #{phony_gemfile_link_name}"
+  end
+
+  task rspec_task_name => bundle_task_name do
+    sh "rvm #{ruby_with_gemset} do env BUNDLE_GEMFILE=#{phony_gemfile_link_name} bundle exec rspec spec --fail-fast"
+  end
+
+  task "mb:#{ns_name}" => rspec_task_name
+
+  task "mb:test_all_rubies" => rspec_task_name
+end
+
+task "mb:test_all" do
+  require 'benchmark'
+  t = Benchmark.realtime do
+    Rake::Task['mb:test_all_rubies'].invoke
+  end
+
+  $stderr.puts "Test run took: #{t} s"
+end
+
+task :default => 'mb:1.9.3'
+
+task :clobber do
+  rm_rf 'tmp'
+end
+
+# cargo culted from http://blog.flavorjon.es/2009/06/easily-valgrind-gdb-your-ruby-c.html
+VALGRIND_BASIC_OPTS = '--num-callers=50 --error-limit=no --partial-loads-ok=yes --undef-value-errors=no'
+
+task 'valgrind' do
+  cd 'ext' do
+    sh "rake clean build"
+  end
+
+  sh "valgrind #{VALGRIND_BASIC_OPTS} bundle exec rspec spec"
+end
+
+namespace :build do
+  task :clean do
+    cd 'ext' do
+      sh 'rake clean'
+    end
+
+    Rake::Task['build'].invoke
+  end
+end
+
+task :build do
+  cd 'ext' do
+    sh "rake"
+  end
+end
+

data/ext/.gitignore

@@ -0,0 +1,6 @@
+bin
+include
+lib
+*.bundle
+
+c

data/ext/Rakefile

@@ -0,0 +1,51 @@
+
+task :clean do
+  if File.exists?('Makefile')
+    sh 'make clean'
+  else
+    $stderr.puts "nothing to clean, no Makefile"
+  end
+end
+
+GENERATE_GVL_CODE_RB = 'generate_gvl_code.rb'
+
+file 'c' do
+  if tarball = Dir['zkc-*.tar.gz'].first
+    sh "tar -zxf #{tarball}"
+  else
+    raise "couldn't find the tarball! wtf?!"
+  end
+end
+
+file GENERATE_GVL_CODE_RB => 'c'
+
+file 'zkrb_wrapper.c' => GENERATE_GVL_CODE_RB do
+  sh "ruby generate_gvl_code.rb code"
+end
+
+file 'zkrb_wrapper.h' => GENERATE_GVL_CODE_RB do
+  sh "ruby generate_gvl_code.rb headers"
+end
+
+ZKRB_WRAPPER = %w[zkrb_wrapper.c zkrb_wrapper.h]
+
+
+task :wrappers => ZKRB_WRAPPER
+
+
+task :clobber => :clean do
+  rm_rf %w[Makefile c lib bin include]
+end
+
+task :build_zkc do
+  sh 'ruby extconf.rb'
+end
+
+file 'Makefile' => :build_zkc
+
+task :build => [ZKRB_WRAPPER, 'Makefile'].flatten do
+  sh 'make'
+end
+
+task :default => :build
+

data/ext/c_zookeeper.rb

@@ -0,0 +1,212 @@
+require 'zookeeper/constants'
+require File.expand_path('../zookeeper_c', __FILE__)
+
+# TODO: see if we can get the destructor to handle thread/event queue teardown
+#       when we're garbage collected
+class CZookeeper
+  include ZookeeperCommon
+  include ZookeeperConstants
+  include ZookeeperExceptions
+
+  DEFAULT_SESSION_TIMEOUT_MSEC = 10000
+
+  class GotNilEventException < StandardError; end
+
+  # assume we're at debug level
+  def self.get_debug_level
+    @debug_level ||= ZOO_LOG_LEVEL_INFO
+  end
+
+  def self.set_debug_level(value)
+    @debug_level = value
+    set_zkrb_debug_level(value)
+  end
+
+  def initialize(host, event_queue, opts={})
+    @host = host
+    @event_queue = event_queue
+    
+    # used by the C layer. CZookeeper sets this to true when the init method
+    # has completed. once this is set to true, it stays true.
+    #
+    # you should grab the @start_stop_mutex before messing with this flag
+    @_running = nil
+
+    # This is set to true after destroy_zkrb_instance has been called and all
+    # CZookeeper state has been cleaned up
+    @_closed = false  # also used by the C layer
+
+    # set by the ruby side to indicate we are in shutdown mode used by method_get_next_event
+    @_shutting_down = false
+
+    # the actual C data is stashed in this ivar. never *ever* touch this
+    @_data = nil
+
+    @_session_timeout_msec = DEFAULT_SESSION_TIMEOUT_MSEC
+
+    @start_stop_mutex = Monitor.new
+    
+    # used to signal that we're running
+    @running_cond = @start_stop_mutex.new_cond
+    
+    @event_thread = nil
+
+    setup_event_thread!
+
+    zkrb_init(@host)
+
+    logger.debug { "init returned!" }
+  end
+
+  def closed?
+    @start_stop_mutex.synchronize { !!@_closed }
+  end
+
+  def running?
+    @start_stop_mutex.synchronize { !!@_running }
+  end
+
+  def shutting_down?
+    @start_stop_mutex.synchronize { !!@_shutting_down }
+  end
+
+  def connected?
+    state == ZOO_CONNECTED_STATE
+  end
+
+  def connecting?
+    state == ZOO_CONNECTING_STATE
+  end
+
+  def associating?
+    state == ZOO_ASSOCIATING_STATE
+  end
+
+  def close
+    return if closed?
+
+    shut_down!
+    stop_event_thread!
+
+    @start_stop_mutex.synchronize do
+      if !@_closed and @_data
+        close_handle
+      end
+    end
+
+    nil
+  end
+
+  def state
+    return ZOO_CLOSED_STATE if closed?
+    zkrb_state
+  end
+
+  # this implementation is gross, but i don't really see another way of doing it
+  # without more grossness
+  #
+  # returns true if we're connected, false if we're not
+  #
+  # if timeout is nil, we never time out, and wait forever for CONNECTED state
+  #
+  def wait_until_connected(timeout=10)
+    return false unless wait_until_running(timeout)
+
+    time_to_stop = timeout ? (Time.now + timeout) : nil
+
+    until connected? or (time_to_stop and Time.now > time_to_stop)
+      Thread.pass
+    end
+
+    connected?
+  end
+
+  private
+    # will wait until the client has entered the running? state
+    # or until timeout seconds have passed.
+    #
+    # returns true if we're running, false if we timed out
+    def wait_until_running(timeout=5)
+      @start_stop_mutex.synchronize do
+        return true if @_running
+        @running_cond.wait(timeout)
+        !!@_running
+      end
+    end
+
+    def setup_event_thread!
+      @event_thread ||= Thread.new(&method(:_event_thread_body))
+    end
+
+    def _event_thread_body
+      logger.debug { "event_thread waiting until running: #{@_running}" }
+
+      @start_stop_mutex.synchronize do
+        @running_cond.wait_until { @_running }
+
+        if @_shutting_down
+          logger.error { "event thread saw @_shutting_down, bailing without entering loop" }
+          return
+        end
+      end
+
+      logger.debug { "event_thread running: #{@_running}" }
+
+      while true
+        begin
+          _iterate_event_delivery
+        rescue GotNilEventException
+          logger.debug { "#{self.class}##{__method__}: event delivery thread is exiting" }
+          break
+        end
+      end
+    end
+
+    def _iterate_event_delivery
+      get_next_event(true).tap do |hash|
+        raise GotNilEventException if hash.nil?
+        @event_queue.push(hash)
+      end
+    end
+
+    # use this method to set the @_shutting_down flag to true
+    def shut_down!
+      logger.debug { "#{self.class}##{__method__}" }
+
+      @start_stop_mutex.synchronize do
+        @_shutting_down = true
+      end
+    end
+
+    # this method is part of the reopen/close code, and is responsible for
+    # shutting down the dispatch thread. 
+    #
+    # @dispatch will be nil when this method exits
+    #
+    def stop_event_thread!
+      logger.debug { "#{self.class}##{__method__}" }
+
+      if @event_thread
+        unless @_closed
+          wake_event_loop! # this is a C method
+        end
+        @event_thread.join 
+        @event_thread = nil
+      end
+    end
+
+    def logger
+      Zookeeper.logger
+    end
+
+    # called by underlying C code to signal we're running
+    def zkc_set_running_and_notify!
+      logger.debug { "#{self.class}##{__method__}" }
+
+      @start_stop_mutex.synchronize do
+        @_running = true
+        @running_cond.broadcast
+      end
+    end
+end
+

data/ext/dbg.h

@@ -0,0 +1,53 @@
+#ifndef __dbg_h__
+#define __dbg_h__
+
+// ALL GLORY TO THE Zed A. Shaw
+// http://c.learncodethehardway.org/book/learn-c-the-hard-waych21.html#x26-10500021
+
+#include <stdio.h>
+#include <errno.h>
+#include <string.h>
+
+#ifdef NDEBUG
+#define debug(M, ...)
+#else
+#define debug(M, ...) fprintf(stderr, "DEBUG %s:%d: " M "\n", __FILE__, __LINE__, ##__VA_ARGS__)
+#endif
+
+#define clean_errno() (errno == 0 ? "None" : strerror(errno))
+
+#define log_err(M, ...) fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) 
+
+#define log_warn(M, ...) fprintf(stderr, "[WARN] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) 
+
+#define log_info(M, ...) fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) 
+
+// acts to assert that A is true
+#define check(A, M, ...) if(!(A)) { log_err(M, ##__VA_ARGS__); errno=0; goto error; } 
+
+// like check, but provide an explicit goto label name
+#define check_goto(A, L, M, ...) if(!(A)) { log_err(M, ##__VA_ARGS__); errno=0; goto L; } 
+
+// like check, but implicit jump to 'unlock' label
+#define check_unlock(A, M, ...) check_goto(A, unlock, M, ##__VA_ARGS__)
+
+#define sentinel(M, ...)  { log_err(M, ##__VA_ARGS__); errno=0; goto error; } 
+
+#define check_mem(A) check((A), "Out of memory.")
+
+// checks the condition A, if not true, logs the message M given using zkrb_debug
+// then does a goto to the label L 
+#define check_debug_goto(A, L, M, ...) if(!(A)) { zkrb_debug(M, ##__VA_ARGS__); errno=0; goto L; } 
+
+// check_debug_goto with implicit 'unlock' label
+#define check_debug_unlock(A, M, ...) check_debug_goto(A, unlock, M, ##__VA_ARGS__) 
+
+// like check_debug_goto, but the label is implicitly 'error'
+#define check_debug(A, M, ...) check_debug_goto(A, error, M, ##__VA_ARGS__) 
+
+#define zkrb_debug(M, ...) if (ZKRBDebugging) fprintf(stderr, "DEBUG %p:%s:%d: " M "\n", pthread_self(), __FILE__, __LINE__, ##__VA_ARGS__)
+#define zkrb_debug_inst(O, M, ...) zkrb_debug("obj_id: %lx, " M, FIX2LONG(rb_obj_id(O)), ##__VA_ARGS__)
+
+// __dbg_h__
+#endif
+

data/ext/depend

@@ -0,0 +1,5 @@
+zookeeper_lib.c:	zookeeper_lib.h
+zkrb_wrapper_compat.c:  zkrb_wrapper_compat.h
+zkrb_wrapper.c:		zkrb_wrapper_compat.c zkrb_wrapper.h
+zookeeper_c.c:	zookeeper_lib.c zookeeper_lib.h zkrb_wrapper.c zkrb_wrapper.h dbg.h
+