zookeeper-ng 1.5

data/spec/spec_helper.rb

@@ -0,0 +1,61 @@
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+$LOAD_PATH.unshift(File.expand_path('../../ext', __FILE__))
+$LOAD_PATH.uniq!
+
+require 'rubygems'
+
+release_ops_path = File.expand_path('../../releaseops/lib', __FILE__)
+
+if File.exists?(release_ops_path)
+  require File.join(release_ops_path, 'releaseops')
+  ReleaseOps::SimpleCov.maybe_start
+end
+
+require 'zookeeper'
+
+Dir[File.expand_path('../support/**/*.rb', __FILE__)].sort.each { |f| require(f) }
+
+if ENV['ZKRB_DEBUG']
+  Zookeeper.set_debug_level(4)
+end
+
+if ENV['ZKRB_NOLOG']
+  SpecGlobalLogger.logger.level = ::Logger::FATAL
+  Zookeeper.set_debug_level(0)
+end
+
+RSpec.configure do |config|
+  config.mock_with :rspec
+  [Zookeeper::SpecHelpers, SpecGlobalLogger].each do |mod|
+    config.include(mod)
+    config.extend(mod)
+  end
+
+  if Zookeeper.spawn_zookeeper?
+    require 'zk-server'
+
+    config.before(:suite) do 
+      SpecGlobalLogger.logger.debug { "Starting zookeeper service" }
+      ZK::Server.run do |c|
+        c.base_dir    = File.expand_path('../../.zkserver', __FILE__)
+        c.client_port = Zookeeper.test_port
+        c.force_sync  = false
+        c.snap_count  = 1_000_000
+      end
+    end
+
+    config.after(:suite) do
+      SpecGlobalLogger.logger.debug  { "stopping zookeeper service" }
+      ZK::Server.shutdown
+    end
+  end
+end
+
+require 'pp'
+
+if RUBY_VERSION == '1.9.3'
+  trap('USR1') do
+    threads = Thread.list.map { |th| th.backtrace }
+    pp threads
+  end
+end

data/spec/support/00_logging.rb

@@ -0,0 +1,38 @@
+module Zookeeper
+  TEST_LOG_PATH = File.expand_path('../../../test.log', __FILE__)
+
+  def self.setup_test_logger
+    log =
+      if (ENV['ZOOKEEPER_DEBUG'] || ENV['ZKRB_DEBUG'])
+        ::Logger.new(STDERR)
+      else
+        ::Logger.new(TEST_LOG_PATH)
+      end
+
+    log.level = ::Logger::DEBUG
+
+    Zookeeper::Logger.wrapped_logger = log
+  end
+end
+
+Zookeeper.setup_test_logger
+
+module SpecGlobalLogger
+  extend self
+
+  def logger
+    @spec_global_logger ||= Zookeeper::Logger::ForwardingLogger.for(Zookeeper::Logger.wrapped_logger, 'spec')
+  end
+
+  # sets the log level to FATAL for the duration of the block
+  def mute_logger
+    zk_log = Zookeeper::Logger.wrapped_logger
+
+    orig_level, zk_log.level = zk_log.level, ::Logger::FATAL
+    orig_zk_level, Zookeeper.debug_level = Zookeeper.debug_level, Zookeeper::Constants::ZOO_LOG_LEVEL_ERROR
+    yield
+  ensure
+    zk_log.level = orig_zk_level
+  end
+end
+

data/spec/support/10_spawn_zookeeper.rb

@@ -0,0 +1,24 @@
+module Zookeeper
+  def self.spawn_zookeeper?
+    !!ENV['SPAWN_ZOOKEEPER']
+  end
+
+  def self.travis?
+    !!ENV['TRAVIS']
+  end
+
+  def self.default_cnx_host
+    ENV['ZK_DEFAULT_HOST'] || 'localhost'
+  end
+
+  @test_port ||= spawn_zookeeper? ? 21811 : 2181
+
+  class << self
+    attr_accessor :test_port
+  end
+
+  def self.default_cnx_str
+    "#{default_cnx_host}:#{test_port}"
+  end
+end
+

data/spec/support/progress_formatter.rb

@@ -0,0 +1,15 @@
+require 'rspec/core/formatters/progress_formatter'
+
+module RSpec
+  module Core
+    module Formatters
+      class ProgressFormatter
+        def example_started(example)
+          SpecGlobalLogger.logger << pending_color("\n=====<([ #{example.full_description} ])>=====\n")
+          super(example)
+        end
+      end
+    end
+  end
+end
+

data/spec/support/zookeeper_spec_helpers.rb

@@ -0,0 +1,96 @@
+module Zookeeper
+  module SpecHelpers
+    class TimeoutError < StandardError; end
+    include Zookeeper::Constants
+    include Zookeeper::Logger
+
+    def ensure_node(zk, path, data)
+      return if zk.closed?
+      if zk.stat(:path => path)[:stat].exists?
+        zk.set(:path => path, :data => data)
+      else
+        zk.create(:path => path, :data => data)
+      end
+    end
+
+    def with_open_zk(host=nil)
+      z = Zookeeper.new(Zookeeper.default_cnx_str)
+      yield z
+    ensure
+      if z
+        unless z.closed?
+          z.close
+
+          wait_until do 
+            begin
+              !z.connected?
+            rescue RuntimeError
+              true
+            end
+          end
+        end
+      end
+    end
+
+    # this is not as safe as the one in ZK, just to be used to clean up
+    # when we're the only one adjusting a particular path
+    def rm_rf(z, path)
+      z.get_children(:path => path).tap do |h|
+        if h[:rc].zero?
+          h[:children].each do |child|
+            rm_rf(z, File.join(path, child))
+          end
+        elsif h[:rc] == ZNONODE
+          # no-op
+        else
+          raise "Oh noes! unexpected return value! #{h.inspect}"
+        end
+      end
+
+      rv = z.delete(:path => path)
+
+      unless (rv[:rc].zero? or rv[:rc] == ZNONODE)
+        raise "oh noes! failed to delete #{path}" 
+      end
+
+      path
+    end
+
+    def mkdir_p(zk, path)
+      while true # loop because we can't retry
+        h = zk.create(:path => path, :data => '')
+        case h[:rc]
+        when ZOK, ZNODEEXISTS
+          return
+        when ZNONODE
+          parent = File.dirname(path)
+          raise "WTF? we wound up trying to create '/', something is screwed!" if parent == '/'
+          mkdir_p(zk, parent)
+        end
+      end
+    end
+
+    # method to wait until block passed returns true or timeout (default is 10 seconds) is reached 
+    # raises TiemoutError on timeout
+    def wait_until(timeout=10)
+      time_to_stop = Time.now + timeout
+      while true
+        rval = yield
+        return rval if rval
+        raise TimeoutError, "timeout of #{timeout}s exceeded" if Time.now > time_to_stop
+        Thread.pass
+      end
+    end
+
+    # inverse of wait_until
+    def wait_while(timeout=10)
+      time_to_stop = Time.now + timeout
+      while true
+        rval = yield
+        return rval unless rval
+        raise TimeoutError, "timeout of #{timeout}s exceeded" if Time.now > time_to_stop
+        Thread.pass
+      end
+    end
+  end
+end

data/spec/zookeeper_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+require 'shared/connection_examples'
+
+
+describe 'Zookeeper' do
+  let(:path) { "/_zktest_" }
+  let(:data) { "underpants" } 
+  let(:connection_string) { Zookeeper.default_cnx_str }
+
+  before do
+    @zk = Zookeeper.new(connection_string)
+  end
+
+  after do
+    @zk and @zk.close
+  end
+
+  def zk
+    @zk
+  end
+
+  it_should_behave_like "connection"
+end
+

data/zookeeper.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'zookeeper/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'zookeeper-ng'
+  s.version     = Zookeeper::VERSION
+
+  s.authors     = ["Ben Fritsch", "Phillip Pearson", "Eric Maland", "Evan Weaver", "Brian Wickman", "Neil Conway", "Jonathan D. Simms"]
+  s.email       = ["ich@abwesend.com"]
+  s.summary     = %q{Apache ZooKeeper driver for Rubies}
+  s.description = <<-EOS
+A low-level multi-Ruby wrapper around the ZooKeeper API bindings. For a
+friendlier interface, see http://github.com/slyphon/zk. Currently supported:
+MRI: {1.8.7, 1.9.2, 1.9.3}, JRuby: ~> 1.6.7, Rubinius: 2.0.testing, REE 1.8.7.
+
+This library uses version #{Zookeeper::DRIVER_VERSION} of zookeeper bindings.
+
+  EOS
+
+  s.homepage    = 'https://github.com/slyphon/zookeeper'
+
+  s.files         = `git ls-files`.split("\n")
+  s.require_paths = ["lib"]
+
+  if ENV['JAVA_GEM'] or defined?(::JRUBY_VERSION)
+    s.platform = 'java'
+    s.add_runtime_dependency('slyphon-log4j',         '= 1.2.15')
+    s.add_runtime_dependency('slyphon-zookeeper_jar', '= 3.3.5')
+    s.require_paths += %w[java]
+  else
+    s.require_paths += %w[ext]
+    s.extensions = 'ext/extconf.rb'
+  end
+
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+end

data/zoomonkey/duplicates

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+awk '{ if (prev == $0) count++; else { if (count > 0) { print " --- Line repeated " count " times --- " } print; prev = $0; count = 0 } }' $*

data/zoomonkey/zoomonkey.rb

@@ -0,0 +1,194 @@
+require 'zookeeper'
+require 'zk-server'
+require 'fileutils'
+require 'tmpdir'
+
+SLEEP_TIME = 30
+STDOUT.sync = true
+
+if ENV['DEBUG']
+  def zookeeper_logger(from)
+    l = Logger.new(STDOUT)
+    l.formatter = proc do |sev, time, c, msg|
+      "t=#{time.to_i} from=#{from} level=#{sev.downcase} message=#{msg.inspect}\n"
+    end
+    l
+  end
+
+  Zookeeper.logger = zookeeper_logger('zookeeper')
+  Zookeeper.set_debug_level(Zookeeper::ZOO_LOG_LEVEL_DEBUG)
+end
+
+class Worker
+  def initialize(body = nil, &block)
+    raise ArgumentError, "Cannot include both body and block" if body && block
+    @body = body || block
+  end
+
+  def body
+    @body || method(:call)
+  end
+
+  def start
+    @thread = Thread.new do
+      Thread.current.abort_on_exception = true
+      body.call
+    end
+  end
+
+  def stop
+    if @thread
+      @thread.kill
+      @thread = nil
+    end
+  end
+
+  def join
+    if @thread
+      @thread.join
+    end
+  end
+end
+
+
+
+base_dir = Dir.mktmpdir('zk-server-cluster')
+num_cluster = 3
+cluster = ZK::Server::Cluster.new(num_cluster, :base_dir => base_dir)
+
+class Reader < Worker
+  attr_reader :client
+
+  def initialize(zookeeper_hosts)
+    @zookeeper_hosts = zookeeper_hosts
+    @log_from = :reader
+  end
+
+  def call
+    @client = Zookeeper.new(@zookeeper_hosts, 10, method(:watcher))
+    client.wait_until_connected
+
+    client.create(:path => "/test", :data => '') rescue client.set(:path => "/test", :data => '')
+
+    while true
+      error = nil
+      t = Benchmark.realtime do
+        begin
+          client.get(:path => "/test")
+        rescue => e
+          error = e
+        end
+      end
+
+      msg = "host=#{client.connected_host || 'nil'} session_id=#{client.session_id} state=#{client.state_by_value(client.state)} time=#{"%0.4f" % t}"
+      if error
+        msg << " error=#{error.class} error_message=#{error.to_s.inspect}"
+        msg << " closed=#{client.closed?} running=#{client.running?} shutting_down=#{client.shutting_down?}"
+      end
+
+      log msg
+
+      sleep 1
+    end
+  end
+
+  def log(message)
+    puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
+  end
+
+  def watcher(event)
+    if event[:state] == Zookeeper::ZOO_EXPIRED_SESSION_STATE
+      if client
+        log "action=reconnecting state=#{client.state_by_value(event[:state])} session_id=#{client.session_id}"
+        client.reopen
+      end
+    end
+
+  end
+end
+
+class Writer < Worker
+  def initialize(zookeeper_hosts)
+    @zookeeper_hosts = zookeeper_hosts
+    @log_from = :writer
+  end
+
+  def call
+    client = Zookeeper.new(@zookeeper_hosts)
+    client.wait_until_connected
+
+    while true
+      error = nil
+      t = Benchmark.realtime do
+        begin
+          client.create(:path => "/test", :data => '') rescue client.set(:path => "/test", :data => '')
+        rescue => e
+          error = e
+        end
+      end
+
+      msg = "host=#{client.connected_host || 'nil'} session_id=#{client.session_id} state=#{client.state_by_value(client.state)} time=#{"%0.4f" % t}"
+      msg << " error=#{error.class} error_message=#{error.to_s.inspect}" if error
+      log msg
+
+      sleep 1
+    end
+  end
+
+  def log(message)
+    puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
+  end
+end
+
+class ZooMonkey < Worker
+  attr_reader :cluster
+
+  def initialize(cluster)
+    @cluster = cluster
+    @log_from = :server
+  end
+
+  def call
+    while true
+      sleep SLEEP_TIME
+
+      cluster.processes.each do |server|
+        host = "127.0.0.1:#{server.client_port}"
+        log "host=#{host} pid=#{server.pid} action=pausing"
+        server.kill "STOP"
+        sleep SLEEP_TIME
+
+        log "host=#{host} pid=#{server.pid} action=resuming"
+        server.kill "CONT"
+        sleep SLEEP_TIME
+      end
+    end
+  end
+
+  def log(message)
+    puts "t=#{Time.now.to_i} from=#{@log_from} #{message}\n"
+  end
+end
+
+begin
+  cluster.run
+
+  zookeeper_hosts = cluster.processes.map { |p| "127.0.0.1:#{p.client_port}" }
+  zookeeper_spec  = (zookeeper_hosts * 2).join(',')
+
+  reader = Reader.new(zookeeper_spec)
+  reader.start
+
+  # writer = Writer.new(zookeeper_spec)
+  # writer.start
+
+  monkey = ZooMonkey.new(cluster)
+  monkey.start
+
+  reader.join
+  writer.join
+  monkey.join
+ensure
+  cluster.clobber!
+  FileUtils.remove_entry(base_dir)
+end