zeevex_cluster 0.2.1

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+tmp

data/Gemfile

@@ -0,0 +1,22 @@
+source "http://rubygems.org"
+
+group :development, :test do
+  gem 'pry'
+  gem 'pry-remote'
+  gem 'pry-doc'
+  gem 'pry-nav'
+  gem 'pry-buffers'
+  gem 'pry-syntax-hacks'
+  gem 'pry-git', :platform => :mri
+  gem 'jist'
+  gem 'ruby18_source_location', :platform => :mri_18
+  gem 'mysql2', :platform => :mri
+end
+
+group :development, :test do
+  gem 'zk',       :path => '/Users/Shared/squid/src/github/cluster/zk'
+  gem 'zk-group', :path => '/Users/Shared/squid/src/github/cluster/zk-group'
+end
+
+# Specify your gem's dependencies in zeevex_cluster.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,44 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+namespace :spec do
+  SPEC_PLATFORMS = ENV.has_key?('SPEC_PLATFORMS') ? 
+        ENV['SPEC_PLATFORMS'].split(/ +/) :
+        %w{1.9.3-p448 2.0.0-p247 1.8.7-p374}
+
+  desc "Run on three Rubies"
+  task :platforms do
+    # current = %x[rbenv version | awk '{print $1}']
+    
+    fail = false
+    SPEC_PLATFORMS.each do |version|
+      puts "Switching to #{version}"
+      Bundler.with_clean_env do
+        system %{bash -c 'eval "$(rbenv init -)" && rbenv use #{version} && rbenv rehash && ruby -v && bundle exec rake spec'}
+      end
+      if $?.exitstatus != 0
+        fail = true
+        break
+      end
+    end
+
+    exit (fail ? 1 : 0)
+  end
+
+  desc 'Install gems for all tested rubies'
+  task :platform_setup do
+     SPEC_PLATFORMS.each do |version|
+      puts "Setting up platform #{version}"
+      Bundler.with_clean_env do
+        system %{bash -c 'eval "$(rbenv init -)" && rbenv use #{version} && rbenv rehash && gem install bundler && bundle install'}
+      end
+    end   
+  end
+end
+
+task :default => 'spec'

data/doc/BUGS-zookeeper.txt

@@ -0,0 +1,60 @@
+When suspending master with Ctrl-Z for a while, the other 2 nodes in a 
+3 node cluster elect a new master and consider the suspended node to 
+have exited.
+
+Resuming the suspended node shows that it is not fully aware of its altered
+status, though note that zk-group shows it's not a member anymore.
+
+[17] MASTER[2] pry(main)>
+$c.@strategy._get_synchronized_object.@elector.leader? 
+=> true
+[18] MASTER[2] pry(main)> 
+[19] MASTER[2] pry(main)> $c.members
+=> ["blacktip.esquimaux.org:/dev/ttys005",
+"blacktip.esquimaux.org:/dev/ttys006"]
+[20] MASTER[2] pry(main)> $c.nodename
+=> "blacktip.esquimaux.org:/dev/ttys003"
+
+Do we need to act on ZK session/connection changes?
+
+ZK::Client::Base#methods: assert_we_are_not_on_the_event_dispatch_thread!  children  delete  event_dispatch_thread?  event_handler  exists?  get  get_acl  inspect  register  session_id  session_passwd  set  set_acl  set_debug_level  stat  wait_until_connected  watcher
+ZK::Client::StateMixin#methods: on_connected  on_connecting  on_expired_session  on_state_change
+Zookeeper::Constants#methods: event_by_value  state_by_value
+ZK::Client::Unixisms#methods: block_until_node_deleted  find  mkdir_p  rm_rf
+ZK::Client::Conveniences#methods: defer  election_candidate  election_observer  exclusive_locker  locker  ping?  queue  shared_locker  with_lock
+ZK::Logging#methods: logger
+ZK::Client::Threaded#methods: associating?  client_state  close  close!  closed?  connect  connected?  connecting?  create  expired_session?  on_exception  on_threadpool?  pause_before_fork_in_parent  raw_event_handler  reopen  resume_after_fork_in_parent  state  wait_until_closed  wait_until_connected_or_dying
+instance variables: @client_state  @cnx  @cond  @connection_timeout  @event_handler  @fork_subs  @host  @last_cnx_state  @mutex  @pid  @reconnect  @reconnect_thread  @retry_duration  @threadpool
+
+----
+log after resume:
+
+[1]  + 28262 continued  bundle exec ./script/election.rb zookeeper
+[8] MASTER[3] pry(main)>
+[9] MASTER[3] pry(main)> D, [2012-12-24T14:08:51.765368 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x109b0a1d0 @path="/_zk/groups/foobs", @type=4, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @completed=false, @context=nil>
+D, [2012-12-24T14:08:51.766028 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x10993b278 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=1, @completed=false, @context=nil>
+D, [2012-12-24T14:08:51.766576 #28262] DEBUG -- : called #<ZK::EventHandlerSubscription::Base:0x109d6d5d0 @mutex=#<ZK::Monitor:0x109d6d530 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="/_zk/groups/foobs", @parent=#<ZK::EventHandler:0x109d7b0e0 @thread_opt=:single, @mutex=#<ZK::Monitor:0x109d7abb8 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @default_watcher_block=#<Proc:0x0000000109f905d8@/Users/Shared/squid/src/github/cluster/zk/lib/zk/event_handler.rb:251>, @orig_pid=28262, @state=:running, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd0078 ...>, @outstanding_watches={:child=>#<Set: {}>, :data=>#<Set: {}>}, @callbacks={"/_zkelection/foobs/leader_ack"=>[], :all_node_events=>[], "/_zk/groups/foobs"=>[#<ZK::EventHandlerSubscription::Base:0x109d6d5d0 ...>], :all_state_events=>[], "state_1"=>[], "state_3"=>[#<ZK::EventHandlerSubscription::Base:0x109d6cf40 @mutex=#<ZK::Monitor:0x109d6cea0 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="state_3", @parent=#<ZK::EventHandler:0x109d7b0e0 ...>, @interests=#<Set: {:deleted, :changed, :child, :created}>, @callable=#<Proc:0x0000000109d96520@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:90>>]}>, @interests=#<Set: {:child}>, @callable=#<Proc:0x0000000109d96868@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:86>> with [#<Zookeeper::Callbacks::WatcherCallback:0x109b0a1d0 @path="/_zk/groups/foobs", @type=4, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd0078 ...>, @completed=true, @context=nil>] on threadpool
+D, [2012-12-24T14:08:51.766643 #28262] DEBUG -- : session state was invalid, calling reopen
+D, [2012-12-24T14:08:51.766687 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x1099259a0 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=-112, @completed=false, @context=nil>
+D, [2012-12-24T14:08:51.766743 #28262] DEBUG -- : reopening, no fork detected
+D, [2012-12-24T14:08:51.771478 #28262] DEBUG -- : EventHandler#process dispatching event: #<Zookeeper::Callbacks::WatcherCallback:0x1098fa188 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @completed=false, @context=nil>
+D, [2012-12-24T14:08:51.771547 #28262] DEBUG -- : wait_until_connected_or_dying @last_cnx_state: 3, time_left? true, @client_state: :running
+D, [2012-12-24T14:08:51.771710 #28262] DEBUG -- : reopen returned: :connected
+D, [2012-12-24T14:08:51.771908 #28262] DEBUG -- : broadcast_membership_change! received event #<Zookeeper::Callbacks::WatcherCallback:0x109b0a1d0 @path="/_zk/groups/foobs", @type=4, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @completed=true, @context=nil>
+D, [2012-12-24T14:08:51.772264 #28262] DEBUG -- : called #<ZK::EventHandlerSubscription::Base:0x109d6cf40 @mutex=#<ZK::Monitor:0x109d6cea0 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="state_3", @parent=#<ZK::EventHandler:0x109d7b0e0 @thread_opt=:single, @mutex=#<ZK::Monitor:0x109d7abb8 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @default_watcher_block=#<Proc:0x0000000109f905d8@/Users/Shared/squid/src/github/cluster/zk/lib/zk/event_handler.rb:251>, @orig_pid=28262, @state=:running, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @outstanding_watches={:child=>#<Set: {}>, :data=>#<Set: {}>}, @callbacks={"/_zkelection/foobs/leader_ack"=>[], :all_node_events=>[], "/_zk/groups/foobs"=>[#<ZK::EventHandlerSubscription::Base:0x109d6d5d0 @mutex=#<ZK::Monitor:0x109d6d530 @mon_owner=nil, @mon_waiting_queue=[], @mon_entering_queue=[], @mon_count=0>, @path="/_zk/groups/foobs", @parent=#<ZK::EventHandler:0x109d7b0e0 ...>, @interests=#<Set: {:child}>, @callable=#<Proc:0x0000000109d96868@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:86>>], :all_state_events=>[], "state_1"=>[], "state_3"=>[#<ZK::EventHandlerSubscription::Base:0x109d6cf40 ...>], "state_-112"=>[]}>, @interests=#<Set: {:deleted, :changed, :child, :created}>, @callable=#<Proc:0x0000000109d96520@/Users/Shared/squid/src/github/cluster/zk-group/lib/zk-group/group.rb:90>> with [#<Zookeeper::Callbacks::WatcherCallback:0x1098fa188 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @completed=true, @context=nil>] on threadpool
+D, [2012-12-24T14:08:51.772987 #28262] DEBUG -- : last_members: ["m0000000045", "m0000000046", "m0000000047"]
+D, [2012-12-24T14:08:51.773031 #28262] DEBUG -- : @known_members: ["m0000000046", "m0000000047"]
+D, [2012-12-24T14:08:51.774129 #28262] DEBUG -- : ZK: membership change from ZK::Group: from ["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"] to ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]
+D, [2012-12-24T14:08:51.774173 #28262] DEBUG -- : <running hook membership_change([["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"], ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]])>
+D, [2012-12-24T14:08:51.774221 #28262] DEBUG -- : ZeevexCluster::Election observed hook: membership_change [["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"], ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]]
+D, [2012-12-24T14:08:51.774276 #28262] DEBUG -- : <running hook strategy_membership_change([["blacktip.esquimaux.org:/dev/ttys003", "blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"], ["blacktip.esquimaux.org:/dev/ttys005", "blacktip.esquimaux.org:/dev/ttys006"]])>
+D, [2012-12-24T14:08:51.774410 #28262] DEBUG -- : broadcast_membership_change! received event #<Zookeeper::Callbacks::WatcherCallback:0x1098fa188 @path="", @type=-1, @proc=#<Proc:0x00000001098d5630@/Users/robertsanders/.rvm/gems/ree-1.8.7-2012.02@zeevex_cluster/gems/zookeeper-1.4.1/lib/zookeeper/callbacks.rb:24>, @state=3, @zk=#<ZK::Client::Threaded:2230049920 zk_session_id=0x13bc3f378bd007b ...>, @completed=true, @context=nil>
+D, [2012-12-24T14:08:51.775273 #28262] DEBUG -- : last_members: ["m0000000046", "m0000000047"]
+D, [2012-12-24T14:08:51.775313 #28262] DEBUG -- : @known_members: ["m0000000046", "m0000000047"]
+D, [2012-12-24T14:08:51.775342 #28262] DEBUG -- : membership data did not actually change, not notifying
+[9] MASTER[3] pry(main)>
+[10] MASTER[2] pry(main)>
+
+
+-----------------------------
+

data/doc/TODO.txt

@@ -0,0 +1,85 @@
+Goals:
+
+- Stable leader election
+- Membership w/node info
+- Simple configuration management
+- Leases and Locks
+- Message queue (low performance target)
+
+----
+
+* Mutex around every method that might have cross-thread access,
+  including:
+    * coordinator objects
+
+* Sort out the strategy thing - is it really useful?  Is CAS really the kind of strategy? Can coordinators
+  be used for more than one kind of strategy?
+
+** TESTS!
+  * CAS should be run against a mock backend
+  * Cluster should be run against a mock strategy
+
+* Other user code integration, including event runloops perhaps?
+
+** Backends
+  * All
+    * In membership or cluster info, check for (sem)version compatibility
+  * Redis
+    * Use queues and PUB/SUB for messaging
+    * Use e.g. atomic hashes instead of CAS writes for group membership to scale better?
+  * MySQL backend
+    * auto-create memory table if it doesn't exist
+    * Use MySQL locks somehow to implement non-poll messaging?
+  * memcached
+    * implement queue-like structure for messaging (using append/prepend, CAS for reads)
+    * Easy way to notify a node it should check for various conditions
+    * switch to dalli as client
+    * disable memcache-client's backoff feature (waits N seconds after
+      conn error before allowing next contact)
+  * Moneto (ruby key/value store abstraction - needs CAS, though)
+  * AR backend
+  * MongoDB backend
+    - replica sets and/or write concerns for replication / HA?
+    - tailable cursors for queues
+  * ZK (ruby Zookeeper wrapper) backend - would be very thin, as ZK provides all that this
+    lib plans to
+  * Local/in-process/filesystem backend for testing
+
+* More core functionality / cluster primitives
+  * Global Cluster info struct, and version checking
+  * Group and single node messaging
+  * Leases - can reuse most of leader election code for this; in fact leader election
+             can be a wrapper around a lease
+  * Locks (auto-released when member exits / times out?)
+
+** General user code API
+  * Health check callback into user code - when failed, notify members, resign, and leave
+  * Clearer policy for cross-thread callbacks
+  * Separate out membership from leader election
+  * Cluster configuration file format
+
+** Leader election
+  * Replace fixed @stale_time with lease duration specified by candidate
+  * Different members will notice master state changes at different times, depending on their polling.
+    Should they coordinate or predict when scheduled changes are happening?
+  * Callback for no master, and suspect master
+
+* Implement the state machines in terms of actual state machine gems instead of spaghetti
+
+** Membership
+    * Callbacks into user code for group membership changes - member joined, suspect, left
+    * Fast failure of member - attempt to query when suspect
+    * Kick live member?
+    * After a long partition (including ^Z in which the client isn't running at all), the lib and
+      user code should (locally) leave the cluster and rejoin from scratch - otherwise other nodes
+      will see it as having left, but it may think it never did
+      - Similarly, if it goes to update its membership record and finds it missing, then it should
+        consider itself kicked
+
+
+
+
+Dreamlist:
+
+* Consistent causal ordering of all callbacks / messages on all members?
+

data/lib/zeevex_cluster/base.rb

@@ -0,0 +1,95 @@
+require 'socket'
+require 'hookem'
+
+module ZeevexCluster
+  class Base
+    include ZeevexCluster::Util::Logging
+    include Hookem
+
+    attr_accessor :nodename, :options
+
+    def initialize(options = {})
+      @options = {:nodename => Socket.gethostname,
+                  :autojoin => true}.merge(options)
+      @logger  = @options[:logger]
+
+      _initialize_hook_module
+
+      if @options[:hooks]
+        add_hooks @options[:hooks]
+      end
+    end
+
+    def join
+      raise NotImplementedError
+    end
+
+    def leave
+      raise NotImplementedError
+    end
+
+    def master?
+      raise NotImplementedError
+    end
+
+    def member?
+      raise NotImplementedError
+    end
+
+    ##
+    ## Make this node the master, returning true if successful.
+    ##
+    def make_master!
+      raise NotImplementedError
+    end
+
+    ##
+    ## Make this node the master if not already the master
+    ## if provided a block, run that IFF we are the master
+    ##
+    def ensure_master(&block)
+      make_master! unless master?
+      if block
+        run_if_master &block
+      end
+    end
+
+    ##
+    ## Run the code block only if this node is the master
+    ##
+    def run_if_master(&block)
+      if master?
+        block.call
+      else
+        false
+      end
+    end
+
+    ##
+    ## Resign from mastership; returns false if this is the only node.
+    ##
+    def resign!
+      raise NotImplementedError
+    end
+
+    ##
+    ## Return name of master node
+    ##
+    def master
+      raise NotImplementedError
+    end
+
+    ##
+    ## Return this node's name
+    ##
+    def nodename
+      options[:nodename]
+    end
+
+    protected
+
+    def after_initialize
+      join if options.fetch(:autojoin, true)
+    end
+  end
+end

data/lib/zeevex_cluster/coordinator/base_key_val_store.rb

@@ -0,0 +1,85 @@
+require 'zeevex_cluster/coordinator'
+
+class ZeevexCluster::Coordinator::BaseKeyValStore
+  include ZeevexCluster::Util::Logging
+
+  def self.setup
+    unless @setup
+      require 'memcache'
+      require 'zeevex_cluster/serializer/json_hash'
+      @setup = true
+    end
+  end
+
+  def initialize(options = {})
+    self.class.setup
+    @options    = options
+    if (!options[:server] && !options[:client]) || !options[:expiration]
+      raise ArgumentError, "Must supply [:server or :client] and :expiration"
+    end
+    if options[:client]
+      @client = options[:client]
+    else
+      @server = options[:server]
+      @port   = options[:port] || 11211
+    end
+    @expiration = options[:expiration] || 60
+
+    @logger     = options[:logger]
+
+    @serializer    = options[:serializer] || ZeevexCluster::Serializer::JsonHash.new
+
+    @retries    = options.fetch(:retries,    20)
+    @retry_wait = options.fetch(:retry_wait,  2)
+    @retry_bo   = options.fetch(:retry_bo,    1.5)
+  end
+
+  [:add, :set, :cas, :get].each do |name|
+    define_method "#{name}_with_retry", lambda { |*args, &block|
+      with_connection_retry name, {}, *args, &block
+    }
+  end
+
+  def with_connection_retry(method, options = {}, *args, &block)
+    retry_left = options.fetch(:retries, @retries)
+    retry_wait = options.fetch(:retry_wait, @retry_wait)
+    begin
+      send "do_#{method}", *args, &block
+    rescue ZeevexCluster::Coordinator::ConnectionError
+      if retry_left > 0
+        logger.debug "retrying after #{retry_wait} seconds"
+        retry_left -= 1
+        sleep retry_wait
+        retry_wait = retry_wait * options.fetch('retry_bo', @retry_bo)
+        retry
+      else
+        logger.error 'Ran out of connection retries, re-raising'
+        raise
+      end
+    end
+  end
+
+  protected
+
+  def serialize_value(obj, raw = false)
+    raw ? obj : @serializer.serialize(obj)
+  end
+
+  def deserialize_value(str, raw = false)
+    raw ? str : @serializer.deserialize(str)
+  end
+
+  def is_raw?(options)
+    (@options && @options[:raw]) || (options && options[:raw])
+  end
+
+  def to_key(key)
+    if @options[:namespace]
+      "#{@options[:namespace]}:#{key}"
+    elsif @options[:to_key_proc]
+      @options[:to_key_proc].call(key)
+    else
+      key.to_s
+    end
+  end
+end

data/lib/zeevex_cluster/coordinator/memcached.rb

@@ -0,0 +1,118 @@
+require 'zeevex_cluster/coordinator/base_key_val_store'
+
+module ZeevexCluster::Coordinator
+  class Memcached < BaseKeyValStore
+    def self.setup
+      unless @setup
+        require 'memcache'
+        BaseKeyValStore.setup
+
+        @setup = true
+      end
+    end
+
+    def initialize(options = {})
+      super
+      @client ||= MemCache.new "#@server:#@port"
+    end
+
+    def add(key, value, options = {})
+      status( @client.add(to_key(key), serialize_value(value, options[:raw]),
+                          options.fetch(:expiration, @expiration), raw?) ) == STORED
+    rescue MemCache::MemCacheError
+      raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
+    end
+
+    def set(key, value, options = {})
+      status( @client.set(to_key(key), serialize_value(value, options[:raw]),
+                          options.fetch(:expiration, @expiration), raw?) ) == STORED
+    rescue MemCache::MemCacheError
+      raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
+    end
+
+    def delete(key, options = {})
+      status( @client.delete(to_key(key)) ) == DELETED
+    end
+
+    #
+    # Block is passed the current value, and returns the updated value.
+    #
+    # Block can raise DontChange to simply exit the block without updating.
+    #
+    # returns nil for no value
+    # returns false for failure (somebody else set)
+    # returns true for success
+    #
+    def cas(key, options = {}, &block)
+      res = @client.cas(to_key(key), options.fetch(:expiration, @expiration), raw?) do |inval|
+        serialize_value(yield(deserialize_value(inval, options[:raw])), options[:raw])
+      end
+      case status(res)
+        when nil then nil
+        when EXISTS then false
+        when STORED then true
+        else raise "Unhandled status code: #{res}"
+      end
+    rescue ZeevexCluster::Coordinator::DontChange
+      false
+    rescue MemCache::MemCacheError
+      raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
+    end
+
+    def get(key, options = {})
+      val = @client.get(to_key(key), raw?)
+      if val && !options[:raw]
+        deserialize_value(val)
+      else
+        val
+      end
+    rescue MemCache::MemCacheError
+      raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
+    end
+
+    def append(key, val, options = {})
+      val = serialize_value(val, options[:raw])
+      key = to_key(key)
+      status( @client.append(key, val) ) == STORED  ||
+          status( @client.add(key, val, options.fetch(:expiration, @expiration), true) ) == STORED ||
+          status( @client.append(key, val) ) == STORED
+    rescue MemCache::MemCacheError
+      raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
+    end
+
+    def prepend(key, val, options = {})
+      val = serialize_value(val, options[:raw])
+      key = to_key(key)
+      status( @client.prepend(key, val) ) == STORED  ||
+          status( @client.add(key, val, options.fetch(:expiration, @expiration), true) ) == STORED ||
+          status( @client.prepend(key, val) ) == STORED
+    rescue MemCache::MemCacheError
+      raise ZeevexCluster::Coordinator::ConnectionError.new 'Connection error', $!
+    end
+
+    def push_to_queue(key, object, options = {})
+
+    end
+
+    protected
+
+    STORED     = 'STORED'
+    EXISTS     = 'EXISTS'
+    NOT_STORED = 'NOT_STORED'
+    NOT_FOUND  = 'NOT_FOUND'
+    DELETED    = 'DELETED'
+
+    def status(response)
+      case response
+        when nil, true, false then response
+        when String then response.chomp
+        else
+          raise ArgumentError, "This should only be called on results from cas, add, set, etc. - got result #{response.inspect}"
+      end
+    end
+
+    def raw?
+      true
+    end
+  end
+end