slyphon-zookeeper 0.1.0

data/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+*.o
+*.so
+*~
+*.log
+Makefile
+pkg
+zookeeper.gemspec

data/CHANGELOG

@@ -0,0 +1,23 @@
+v0.4.3 Fix a handful of memory-related bugs, fix SIGSEGV on master change,
+       reduce latency of event handling, fix compilation on OSX.
+
+v0.4.2 Add options to Zookeeper#initialize, silence most Zookeeper logs.
+
+v0.4.1 Upgrade to ZooKeeper 3.3.2
+
+v0.4.0. More attr-readers (StarvingMarvin) and 1.9 compatibility (tsuraan)
+
+v0.3.2. Handle close, closed connections and expired sessions a little more gracefully.
+
+v0.3.1. ACL bugfix.
+
+v0.3.0. Wickman's rewrite, breaks dependencies from myelin/emaland port.
+
+v0.2.2. Fix compatibility with stock Leopard fat-binary Ruby.
+
+v0.2.1. No more camelcase classname.
+
+v0.2. Bundle C dependencies, like memcached.gem.
+
+v0.1. First release.
+

data/LICENSE

@@ -0,0 +1,23 @@
+(The MIT License)
+
+Copyright (C) 2008 Phillip Pearson
+Copyright (C) 2010 Twitter, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest

@@ -0,0 +1,33 @@
+CHANGELOG
+Gemfile
+LICENSE
+Manifest
+README
+Rakefile
+Session.vim
+examples/cloud_config.rb
+ext/extconf.rb
+ext/zkc-3.3.2.tar.gz
+ext/zookeeper_base.rb
+ext/zookeeper_c.c
+ext/zookeeper_lib.c
+ext/zookeeper_lib.h
+java/zookeeper_base.rb
+lib/zookeeper.rb
+lib/zookeeper/acls.rb
+lib/zookeeper/callbacks.rb
+lib/zookeeper/common.rb
+lib/zookeeper/constants.rb
+lib/zookeeper/exceptions.rb
+lib/zookeeper/stat.rb
+spec/log4j.properties
+spec/spec_helper.rb
+spec/zookeeper_spec.rb
+tags
+test/test_basic.rb
+test/test_callback1.rb
+test/test_close.rb
+test/test_esoteric.rb
+test/test_watcher1.rb
+test/test_watcher2.rb
+zookeeper.gemspec

data/README

@@ -0,0 +1,42 @@
+zookeeper
+
+An interface to the Zookeeper distributed configuration server.
+
+== License
+
+Copyright 2008 Phillip Pearson, and 2010 Twitter, Inc. Licensed under the
+MIT License.  See the included LICENSE file.  Portions copyright 2008-2010
+the Apache Software Foundation, licensed under the Apache 2 license, and
+used with permission.
+
+== Install
+
+sudo gem install zookeeper
+
+== Usage
+
+Connect to a server:
+
+  require 'rubygems'
+  require 'zookeeper'
+  z = Zookeeper.new("localhost:2181")
+  z.get_children(:path => "/")
+
+== Idioms
+
+  The following methods are initially supported:
+    get
+    set
+    get_children
+    stat
+    create
+    delete
+    get_acl
+    set_acl
+
+  All support async callbacks.  get, get_children and stat support both
+  watchers and callbacks.
+
+  Calls take a dictionary of parameters.  With the exception of set_acl, the
+  only required parameter is :path.  Each call returns a dictionary with at
+  minimum two keys :req_id and :rc.

data/Rakefile

@@ -0,0 +1,19 @@
+require 'echoe'
+
+Echoe.new("zookeeper") do |p|
+  p.author = "Phillip Pearson, Eric Maland, Evan Weaver, Brian Wickman"
+  p.project = "fauna"
+  p.summary = "An interface to the Zookeeper distributed configuration server."
+  p.url = "https://github.com/twitter/zookeeper"
+  p.docs_host = "blog.evanweaver.com:~/www/bax/public/files/doc/"
+  p.clean_pattern += ["ext/lib", "ext/include", "ext/c", "ext/bin", "ext/conftest.dSYM"]
+  p.rdoc_pattern = /README|TODO|LICENSE|CHANGELOG|BENCH|COMPAT|zookeeper_c.c|zookeeper.rb/
+end
+
+namespace :mb do
+  task :build_gems do
+    sh "gem build slyphon-zookeeper.gemspec"
+    ENV['JAVA_GEM'] = '1'
+    sh "gem build slyphon-zookeeper.gemspec"
+  end
+end

data/examples/cloud_config.rb

@@ -0,0 +1,125 @@
+require "rubygems"
+require "zookeeper"
+
+# A basic cloud-based YAML config library.  Ruby Zookeeper client example.
+#
+# If you pass in a file as 'zk:/foo.yml/blah' it will go out to zookeeper. 
+# Otherwise the file is assumed to be local.  The yml file will get parsed
+# and cached locally, and keys after the .yml get interpreted as keys into
+# the YAML.
+#
+# e.g. get(zk:/config/service.yml/key1/key2/key3..) => 
+#   zk.get(:path => /config/service.yml)
+#   yaml <= YAML.parse(data)
+#   yaml[key1][key2][key3]...
+#
+# If keys are unspecified, it returns the parsed YAML as one big object
+#
+# TODO if staleness is set to 0, read in YAML immediately before next
+# get(...)
+
+class CloudConfig
+  class NodeNotFound < StandardError; end
+  class BadPathError < StandardError; end
+
+  DEFAULT_SERVERS = "localhost:2181"
+
+  def initialize(zkservers = DEFAULT_SERVERS, staleness = 15)  # maximum allowed staleness in seconds
+    @staleness = staleness
+    @lock      = Mutex.new
+    @zkservers = DEFAULT_SERVERS
+    
+    # cache
+    @data      = {}
+    @zkcb      = Zookeeper::WatcherCallback.new { dirty_callback(@zkcb.context) }
+    @zk        = nil
+  end
+
+  def get(node)
+    filename, keys = extract_filename(node)
+
+    # read(filename) is potentially a zk call, so do not hold the lock during the read
+    if @lock.synchronize { !@data.has_key?(filename) }
+      d = YAML.load(read(filename))
+      @lock.synchronize { @data[filename] = d }
+    end
+    
+    # synchronized b/c we potentially have a background thread updating data nodes from zk
+    # if keys is empty, return the whole file, otherwise roll up the keys
+    @lock.synchronize {
+      keys.empty? ? @data[filename] : keys.inject(@data[filename]) { |hash, key| hash[key] }
+    }
+  end
+
+  # todo:
+  #   factor get-and-watch into a different subsystem (so you can have
+  #   polling stat() ops on local filesystem.)
+  def read(yaml)
+    # read yaml file and register watcher.  if watcher fires, set up
+    # background thread to do read and update data.
+    if yaml.match(/^zk:/)
+      @zk ||= init_zk
+      yaml = yaml['zk:'.length..-1] # strip off zk: from zk:/config/path.yml
+      resp = get_and_register(yaml)
+
+      if resp[:rc] != Zookeeper::ZOK
+        @zk.unregister_watcher(resp[:req_id])
+        raise NodeNotFound
+      end
+      
+      resp[:data]
+    else
+      raise NodeNotFound unless File.exists?(yaml)
+      File.read(yaml)
+    end
+  end
+
+  def extract_filename(node)
+    path_elements = node.split("/")
+    
+    yamlindex = path_elements.map{ |x| x.match("\.yml$") != nil }.index(true)
+    raise BadPathError unless yamlindex
+    
+    yamlname = path_elements[0..yamlindex].join '/'
+    yamlkeys = path_elements[(yamlindex+1)..-1]
+    
+    return yamlname, yamlkeys
+  end
+    
+ private
+  def init_zk
+    Zookeeper.new(@zkservers)
+  end
+  
+  def get_and_register(znode)
+    @zk.get(:path => znode, :watcher => @zkcb,
+                   :watcher_context => { :path => znode,
+                                         :wait => rand(@staleness) })
+  end
+  
+  def dirty_callback(context)
+    path = context[:path]
+    wait = context[:wait]
+
+    # Fire off a background update that waits a randomized period of time up
+    # to @staleness seconds.
+    Thread.new do
+      sleep wait
+      background_update(path)
+    end
+  end
+
+  def background_update(zkpath)
+    # do a synchronous get/register a new watcher
+    resp = get_and_register(zkpath)
+    if resp[:rc] != Zookeeper::ZOK
+      # puts "Unable to read #{zkpath} from Zookeeper!"  @logger.error
+      zk.unregister_watcher(resp[:req_id])
+    else
+      # puts "Updating data."
+      d = YAML.load(resp[:data])
+      @lock.synchronize { @data["zk:#{zkpath}"] = d }
+    end
+  end
+end
+

data/ext/.gitignore

@@ -0,0 +1,5 @@
+bin
+include
+lib
+*.bundle
+

data/ext/extconf.rb

@@ -0,0 +1,56 @@
+
+require 'mkmf'
+require 'rbconfig'
+
+HERE = File.expand_path(File.dirname(__FILE__))
+BUNDLE = Dir.glob("zkc-*.tar.gz").first
+BUNDLE_PATH = "c"
+
+$CFLAGS = "#{RbConfig::CONFIG['CFLAGS']} #{$CFLAGS}".gsub("$(cflags)", "").gsub("-arch ppc", "")
+$LDFLAGS = "#{RbConfig::CONFIG['LDFLAGS']} #{$LDFLAGS}".gsub("$(ldflags)", "").gsub("-arch ppc", "")
+$CXXFLAGS = " -std=gnu++98 #{$CFLAGS}"
+$CPPFLAGS = $ARCH_FLAG = $DLDFLAGS = ""
+
+if ENV['DEBUG']
+  puts "Setting debug flags."
+  $CFLAGS << " -O0 -ggdb3 -DHAVE_DEBUG"
+  $EXTRA_CONF = " --enable-debug"
+  $CFLAGS.gsub!(/ -O[^0] /, ' ')
+end
+
+$includes = " -I#{HERE}/include"
+$libraries = " -L#{HERE}/lib -L#{RbConfig::CONFIG['libdir']}"
+$CFLAGS = "#{$includes} #{$libraries} #{$CFLAGS}"
+$LDFLAGS = "#{$libraries} #{$LDFLAGS}"
+$LIBPATH = ["#{HERE}/lib"]
+$DEFLIBPATH = []
+
+Dir.chdir(HERE) do
+  if File.exist?("lib")
+    puts "Zkc already built; run 'rake clean' first if you need to rebuild."
+  else
+    puts "Building zkc."
+    puts(cmd = "tar xzf #{BUNDLE} 2>&1")
+    raise "'#{cmd}' failed" unless system(cmd)
+
+    Dir.chdir(BUNDLE_PATH) do        
+      puts(cmd = "env CC=gcc CXX=g++ CFLAGS='-fPIC #{$CFLAGS}' LDFLAGS='-fPIC #{$LDFLAGS}' ./configure --prefix=#{HERE} --without-cppunit --disable-dependency-tracking #{$EXTRA_CONF} 2>&1")
+      raise "'#{cmd}' failed" unless system(cmd)
+      puts(cmd = "make CXXFLAGS='#{$CXXFLAGS}' CFLAGS='-fPIC #{$CFLAGS}' LDFLAGS='-fPIC #{$LDFLAGS}' || true 2>&1")
+      raise "'#{cmd}' failed" unless system(cmd)
+      puts(cmd = "make install || true 2>&1")
+      raise "'#{cmd}' failed" unless system(cmd)
+    end
+
+    system("rm -rf #{BUNDLE_PATH}") unless ENV['DEBUG'] or ENV['DEV']
+  end
+end
+
+# Absolutely prevent the linker from picking up any other zookeeper_mt
+Dir.chdir("#{HERE}/lib") do
+  system("cp -f libzookeeper_mt.a libzookeeper_mt_gem.a") 
+  system("cp -f libzookeeper_mt.la libzookeeper_mt_gem.la") 
+end
+$LIBS << " -lzookeeper_mt_gem"
+
+create_makefile 'zookeeper_c'

data/ext/zookeeper_base.rb

@@ -0,0 +1,117 @@
+# The low-level wrapper-specific methods for the C lib
+# subclassed by the top-level Zookeeper class
+class ZookeeperBase < CZookeeper
+  include ZookeeperCommon
+  include ZookeeperCallbacks
+  include ZookeeperConstants
+  include ZookeeperExceptions
+  include ZookeeperACLs
+  include ZookeeperStat
+
+
+  ZKRB_GLOBAL_CB_REQ   = -1
+
+  # debug levels
+  ZOO_LOG_LEVEL_ERROR  = 1
+  ZOO_LOG_LEVEL_WARN   = 2
+  ZOO_LOG_LEVEL_INFO   = 3
+  ZOO_LOG_LEVEL_DEBUG  = 4
+  
+  def reopen(timeout = 10, watcher=nil)
+    watcher ||= @default_watcher
+
+    @req_mutex.synchronize do
+      # flushes all outstanding watcher reqs.
+      @watcher_req = {}
+      set_default_global_watcher(&watcher)
+    end
+
+    init(@host)
+
+    if timeout > 0
+      time_to_stop = Time.now + timeout
+      until state == Zookeeper::ZOO_CONNECTED_STATE
+        break if Time.now > time_to_stop
+        sleep 0.1
+      end
+    end
+
+    state
+  end
+
+  def initialize(host, timeout = 10, watcher=nil)
+    @watcher_reqs = {}
+    @completion_reqs = {}
+    @req_mutex = Monitor.new
+    @current_req_id = 1
+    @host = host
+
+    watcher ||= get_default_global_watcher
+
+    @_running = nil # used by the C layer
+    reopen(timeout, watcher)
+    return nil unless connected?
+    setup_dispatch_thread!
+  end
+  
+  # if either of these happen, the user will need to renegotiate a connection via reopen
+  def assert_open
+    raise ZookeeperException::SessionExpired if state == ZOO_EXPIRED_SESSION_STATE
+    raise ZookeeperException::ConnectionClosed unless connected?
+  end
+
+  def connected?
+    state == ZOO_CONNECTED_STATE
+  end
+
+  def connecting?
+    state == ZOO_CONNECTING_STATE
+  end
+
+  def associating?
+    state == ZOO_ASSOCIATING_STATE
+  end
+
+  def close
+    @_running = false;
+    wake_event_loop!
+    
+    @dispatcher.join
+
+    super
+  end
+
+  # set the watcher object/proc that will receive all global events (such as session/state events)
+  def set_default_global_watcher(&block)
+    @req_mutex.synchronize do
+      @default_watcher = block # save this here for reopen() to use
+      @watcher_reqs[ZKRB_GLOBAL_CB_REQ] = { :watcher => @default_watcher, :watcher_context => nil }
+    end
+  end
+
+protected
+  def running?
+    false|@_running
+  end
+
+  def setup_dispatch_thread!
+    @dispatcher = Thread.new do
+      while running?
+        begin                     # calling user code, so protect ourselves
+          dispatch_next_callback 
+        rescue Exception => e
+          $stderr.puts "Error in dispatch thread, #{e.class}: #{e.message}\n" << e.backtrace.map{|n| "\t#{n}"}.join("\n")
+        end
+      end
+    end
+  end
+
+  # TODO: Make all global puts configurable
+  def get_default_global_watcher
+    Proc.new { |args|
+      logger.debug { "Ruby ZK Global CB called type=#{event_by_value(args[:type])} state=#{state_by_value(args[:state])}" }
+      true
+    }
+  end
+end
+