zk-eventmachine 0.2.0.beta.3 → 0.9.0

data/.dev_extras/rvmrc

@@ -1 +1 @@
-rvm 1.9.2@zk-em --create
+rvm 1.9.3@zk-em

data/.gitignore

@@ -5,5 +5,6 @@ pkg/*
 .rvmrc
 .vimrc
 .rspec
-*.log
+*.log*
 .yardoc
+doc

data/.yardopts

@@ -0,0 +1,8 @@
+--protected
+--no-private
+--tag hidden_example:Hidden
+--hide-tag hidden_example
+-
+LICENSE
+README.markdown
+

data/Gemfile

@@ -1,9 +1,21 @@
-source 'http://localhost:50000'
+source ENV['MBOX_BUNDLER_SOURCE'] if ENV['MBOX_BUNDLER_SOURCE']
 source "http://rubygems.org"
 
+group :test do
+  gem 'rspec',       '~> 2.8.0'
+  gem 'yard',        '~> 0.7.0'
+  gem 'autotest',    '>= 4.4.0'
+  gem 'flexmock',    '~> 0.8.10'
+  gem 'evented-spec','~> 0.9.0'
+end
+
+group :development do
+  gem 'redcarpet',   '~> 2.1.0'
+  gem 'rake'
+  gem 'pry'
+end
 
 # Specify your gem's dependencies in zk-em.gemspec
 gemspec
 
-
 # vim:ft=ruby

data/Rakefile

@@ -1,2 +1,37 @@
 require 'bundler'
 Bundler::GemHelper.install_tasks
+
+task :yard do
+  Bundler.setup
+  require 'yard'
+
+  YARD::Rake::YardocTask.new(:run_yardoc) do |t|
+    t.files = ['lib/**/*.rb']
+  end
+
+  Rake::Task[:run_yardoc].invoke
+end
+
+%w[1.8.7 1.9.2 1.9.3 jruby].each do |rvm_ruby|
+  gemset_name         = 'zk-em'
+  ruby_with_gemset    = "#{rvm_ruby}@#{gemset_name}"
+  create_gemset_name  = "mb:#{rvm_ruby}:create_gemset"
+  bundle_task_name    = "mb:#{rvm_ruby}:bundle_install"
+  rspec_task_name     = "mb:#{rvm_ruby}:run_rspec"
+
+  task create_gemset_name do
+    sh "rvm #{rvm_ruby} do rvm gemset create #{gemset_name}" 
+  end
+
+  task bundle_task_name => create_gemset_name do
+    rm_f 'Gemfile.lock'
+    sh "rvm #{ruby_with_gemset} do bundle install"
+  end
+
+  task rspec_task_name => bundle_task_name do
+    sh "rvm #{ruby_with_gemset} do bundle exec rspec spec --fail-fast"
+  end
+
+  task 'mb:test_all' => rspec_task_name
+end
+

data/lib/z_k/z_k_event_machine/callback.rb

@@ -33,6 +33,12 @@ module ZK
           end
         end
 
+        # save a context object, used for associating delivered events with the request that created them
+        attr_accessor :context
+
+        # saves the request id of this call
+        attr_reader :req_id
+
         def initialize(prok=nil, &block)
           on_result(prok, &block)
         end
@@ -49,7 +55,11 @@ module ZK
         # Checks the return code from the async call. If the return code was not ZOK,
         # then fire the errbacks and do the node-style error call
         # otherwise, does nothing
+        #
+        # in this call we also stash the outgoing req_id so we can sync it up
         def check_async_rc(hash)
+          @req_id = hash[:req_id]
+          logger.debug { "#{__method__}: got #{hash.inspect}" } 
           call(hash) unless success?(hash)
         end
 
@@ -58,11 +68,11 @@ module ZK
         # we take the appropriate actions
         #
         # delegates to #deferred_style_result and #node_style_result
-        def call(hash)
-#           logger.debug { "#{self.class.name}#call hash: #{hash.inspect}" }
+        def call(result)
+          logger.debug { "\n#{self.class.name}##{__method__}\n\treq_id: #{req_id.inspect}\n\tcontext: #{context.inspect}\n\tresult: #{result.inspect}" }
           EM.schedule do
-            deferred_style_result(hash) 
-            node_style_result(hash)
+            deferred_style_result(result) 
+            node_style_result(result)
           end
         end
 
@@ -70,7 +80,7 @@ module ZK
         #
         # @param [Hash] hash the result of the async call 
         # 
-        # @returns [true, false] for success, failure 
+        # @return [true, false] for success, failure 
         def success?(hash)
           hash[:rc] == Zookeeper::ZOK
         end

data/lib/z_k/z_k_event_machine/client.rb

@@ -1,26 +1,67 @@
 module ZK
   module ZKEventMachine
     class Client < ZK::Client::Base
+      include Deferred::Accessors
       include ZK::Logging
       include Unixisms
 
       DEFAULT_TIMEOUT = 10
 
-      # allows the synchrony client to muck with this
-      # @private
-      attr_writer :event_handler, :synchrony_client
+      # If we get a ZK::Exceptions::ConnectionLoss exeption back from any call,
+      # or a EXPIRED_SESSION_STATE event, we will call back any handlers registered
+      # here with the exception instance as the argument.
+      #
+      # once this deferred has been fired, it will be replaced with a new
+      # deferred, so callbacks must be re-registered, and *should* be
+      # re-registered *within* the callback to avoid missing events
+      # 
+      # @method on_connection_lost
+      # @return [Deferred::Default]
+      deferred_event :connection_lost
+
+      # Registers a one-shot callback for the ZOO_CONNECTED_STATE event. 
+      #
+      # @note this is experimental currently. This may or may not fire for the *initial* connection.
+      # it's purpose is to warn an already-existing client with watches that a connection has been
+      # re-established (with session information saved). From the ZooKeeper Programmers' Guide:
+      #
+      #   If you are using watches, you must look for the connected watch event.
+      #   When a ZooKeeper client disconnects from a server, you will not receive
+      #   notification of changes until reconnected. If you are watching for a
+      #   znode to come into existance, you will miss the event if the znode is
+      #   created and deleted while you are disconnected.
+      #
+      # once this deferred has been fired, it will be replaced with a new
+      # deferred, so callbacks must be re-registered, and *should* be
+      # re-registered *within* the callback to avoid missing events
+      #
+      # @method on_connected
+      # @return [Deferred::Default]
+      deferred_event :connected
+
+      # Registers a one-shot callback for the ZOO_CONNECTING_STATE event
+      #
+      # This event is triggered when we have become disconnected from the
+      # cluster and are in the process of reconnecting.
+      deferred_event :connecting
+
+      # called back once the connection has been closed.
+      #
+      # @method on_close
+      # @return [Deferred::Default]
+      deferred_event :close
 
       # Takes same options as ZK::Client::Base
       def initialize(host, opts={})
         @host = host
-        @close_deferred = Deferred::Default.new
-        @connection_lost_deferred = Deferred::Default.new
-        @event_handler = EventHandlerEM.new(self)
+        @event_handler  = EventHandlerEM.new(self)
+        @closing        = false
+        register_default_event_handlers!
       end
 
-      def on_close(&blk)
-        @close_deferred.callback(&blk) if blk
-        @close_deferred
+      # @private
+      def closing?
+        !!@closing
       end
 
       # open a ZK connection, attach it to the reactor. 
@@ -28,37 +69,21 @@ module ZK
       # ready for use
       def connect(&blk)
         # XXX: maybe move this into initialize, need to figure out how to schedule it properly
-        @cnx ||= ZookeeperEM::Client.new(@host, DEFAULT_TIMEOUT, event_handler.get_default_watcher_block)
-#         @cnx.on_attached do
-#           $stderr.puts "@cnx, obj_id: %x" % [@cnx.object_id]
-#         end
+        @cnx ||= (
+          ZookeeperEM::Client.new(@host, DEFAULT_TIMEOUT, event_handler.get_default_watcher_block)
+        )
         @cnx.on_attached(&blk)
       end
 
       # @private
-      # XXX: move this down into ZK::Client::Base
-      def connected?
-        @cnx and @cnx.connected?
-      end
-
-      # If we get a ZK::Exceptions::ConnectionLoss exeption back from any call,
-      # we will call back any handlers registered here with the exception
-      # instance as the argument
-      #
-      # once this deferred has been fired, it will be replaced with a new
-      # deferred, so callbacks must be re-registered
-      # 
-      def on_connection_lost(&blk)
-        @connection_lost_deferred.callback(&blk) if blk
-        @connection_lost_deferred
-      end
-
       def reopen(*a)
         raise NotImplementedError, "reoopen is not implemented for the eventmachine version of the client"
       end
       
       def close!(&blk)
         on_close(&blk)
+        return on_close if @closing
+        @closing = true
 
         if @cnx
           logger.debug { "#{self.class.name}: in close! clearing event_handler" }
@@ -66,11 +91,8 @@ module ZK
 
           logger.debug { "#{self.class.name}: calling @cnx.close" }
           @cnx.close do
-            # fire the on_close handler in next_tick so that @cnx is nil
-            EM.next_tick do
-              logger.debug { "firing on_close handler" }
-              on_close.succeed
-            end
+            logger.debug { "firing on_close handler" }
+            on_close.succeed
             @cnx = nil
           end
         else
@@ -83,12 +105,13 @@ module ZK
 
       # get data at path, optionally enabling a watch on the node
       #
-      # @returns [Callback] returns a Callback which is an EM::Deferred (so you
+      # @return [Callback] returns a Callback which is an EM::Deferred (so you
       #   can assign callbacks/errbacks) see Callback::Base for discussion
       #
       def get(path, opts={}, &block)
         Callback.new_get_cb(block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :opts => opts }
           super(path, opts.merge(:callback => cb))
         end
       end
@@ -96,6 +119,7 @@ module ZK
       def create(path, data='', opts={}, &block)
         Callback.new_create_cb(block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :data => data, :opts => opts }
           super(path, data, opts.merge(:callback => cb))
         end
       end
@@ -103,6 +127,7 @@ module ZK
       def set(path, data, opts={}, &block)
         Callback.new_set_cb(block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :data => data, :opts => opts }
           super(path, data, opts.merge(:callback => cb))
         end
       end
@@ -114,6 +139,7 @@ module ZK
 
         Callback.__send__(meth, block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :meth => meth, :opts => opts }
           super(path, opts.merge(:callback => cb))
         end
       end
@@ -125,6 +151,7 @@ module ZK
       def delete(path, opts={}, &block)
         Callback.new_delete_cb(block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :opts => opts }
           super(path, opts.merge(:callback => cb))
         end
       end
@@ -132,6 +159,7 @@ module ZK
       def children(path, opts={}, &block)
         Callback.new_children_cb(block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :opts => opts }
           super(path, opts.merge(:callback => cb))
         end
       end
@@ -139,6 +167,7 @@ module ZK
       def get_acl(path, opts={}, &block)
         Callback.new_get_acl_cb(block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :opts => opts }
           super(path, opts.merge(:callback => cb))
         end
       end
@@ -146,25 +175,52 @@ module ZK
       def set_acl(path, acls, opts={}, &block)
         Callback.new_set_acl_cb(block) do |cb|
           cb.errback(&method(:connection_lost_hook))
+          cb.context = { :method => __method__, :path => path, :acls => acls, :opts => opts }
           super(path, acls, opts.merge(:callback => cb))
         end
       end
 
-      # returns an EM::Synchrony compatible wrapper around this client
-      def to_synchrony
-        @synchrony_client ||= SynchronyClient.new(self)
+      # @return [Fixnum] The underlying connection's session_id
+      def session_id
+        return nil unless @cnx
+        @cnx.session_id
       end
 
-      # returns self
-      def to_async
-        self
+      # @return [String] The underlying connection's session passwd (an opaque value)
+      def session_passwd
+        return nil unless @cnx
+        @cnx.session_passwd
       end
 
     protected
+      # @private
+      def register_default_event_handlers!
+        @event_handler.register_state_handler(Zookeeper::ZOO_EXPIRED_SESSION_STATE, &method(:handle_expired_session_state_event!))
+        @event_handler.register_state_handler(Zookeeper::ZOO_CONNECTED_STATE,       &method(:handle_connected_state_event!))
+        @event_handler.register_state_handler(Zookeeper::ZOO_CONNECTING_STATE,      &method(:handle_connecting_state_event!))
+      end
+
+      # @private
+      def handle_connected_state_event!(event)
+        EM.schedule { reset_connected_event.succeed(event) }
+      end
+
+      # @private
+      def handle_connecting_state_event!(event)
+        EM.schedule { reset_connecting_event.succeed(event) }
+      end
+
+      # @private
+      def handle_expired_session_state_event!(event)
+        exc = ZK::Exceptions::ConnectionLoss.new("Received EXPIRED_SESSION_STATE event: #{event.inspect}")
+        exc.set_backtrace(caller)
+        connection_lost_hook(exc)
+      end
+
+      # @private
       def connection_lost_hook(exc)
         if exc and exc.kind_of?(ZK::Exceptions::ConnectionLoss)
-          dfr, @connection_lost_deferred = @connection_lost_deferred, Deferred::Default.new
-          dfr.succeed(exc)
+          EM.schedule { reset_connection_lost_event.succeed(exc) }
         end
       end
     end

data/lib/z_k/z_k_event_machine/iterator.rb

@@ -0,0 +1,229 @@
+# Taken from EventMachine release candidate
+module ZK
+  module ZKEventMachine
+    # A simple iterator for concurrent asynchronous work.
+    #
+    # Unlike ruby's built-in iterators, the end of the current iteration cycle is signaled manually,
+    # instead of happening automatically after the yielded block finishes executing. For example:
+    #
+    #  (0..10).each{ |num| }
+    #
+    # becomes:
+    #
+    #  EM::Iterator.new(0..10).each{ |num,iter| iter.next }
+    #
+    # This is especially useful when doing asynchronous work via reactor libraries and
+    # functions. For example, given a sync and async http api:
+    #
+    #  response = sync_http_get(url); ...
+    #  async_http_get(url){ |response| ... }
+    #
+    # a synchronous iterator such as:
+    #
+    #  responses = urls.map{ |url| sync_http_get(url) }
+    #  ...
+    #  puts 'all done!'
+    #
+    # could be written as:
+    #
+    #  EM::Iterator.new(urls).map(proc{ |url,iter|
+    #    async_http_get(url){ |res|
+    #      iter.return(res)
+    #    }
+    #  }, proc{ |responses|
+    #    ...
+    #    puts 'all done!'
+    #  })
+    #
+    # Now, you can take advantage of the asynchronous api to issue requests in parallel. For example,
+    # to fetch 10 urls at a time, simply pass in a concurrency of 10:
+    #
+    #  EM::Iterator.new(urls, 10).each do |url,iter|
+    #    async_http_get(url){ iter.next }
+    #  end
+    #
+    class Iterator
+      # Create a new parallel async iterator with specified concurrency.
+      #
+      #  i = EM::Iterator.new(1..100, 10)
+      #
+      # will create an iterator over the range that processes 10 items at a time. Iteration
+      # is started via #each, #map or #inject
+      #
+      def initialize(list, concurrency = 1)
+        raise ArgumentError, 'argument must be an array' unless list.respond_to?(:to_a)
+        @list = list.to_a.dup
+        @concurrency = concurrency
+
+        @started = false
+        @ended = false
+      end
+
+      # Change the concurrency of this iterator. Workers will automatically be spawned or destroyed
+      # to accomodate the new concurrency level.
+      #
+      def concurrency=(val)
+        old = @concurrency
+        @concurrency = val
+
+        spawn_workers if val > old and @started and !@ended
+      end
+      attr_reader :concurrency
+
+      # Iterate over a set of items using the specified block or proc.
+      #
+      #  EM::Iterator.new(1..100).each do |num, iter|
+      #    puts num
+      #    iter.next
+      #  end
+      #
+      # An optional second proc is invoked after the iteration is complete.
+      #
+      #  EM::Iterator.new(1..100).each(
+      #    proc{ |num,iter| iter.next },
+      #    proc{ puts 'all done' }
+      #  )
+      #
+      def each(foreach=nil, after=nil, &blk)
+        raise ArgumentError, 'proc or block required for iteration' unless foreach ||= blk
+        raise RuntimeError, 'cannot iterate over an iterator more than once' if @started or @ended
+
+        @started = true
+        @pending = 0
+        @workers = 0
+
+        all_done = proc{
+          after.call if after and @ended and @pending == 0
+        }
+
+        @process_next = proc{
+          # p [:process_next, :pending=, @pending, :workers=, @workers, :ended=, @ended, :concurrency=, @concurrency, :list=, @list]
+          unless @ended or @workers > @concurrency
+            if @list.empty?
+              @ended = true
+              @workers -= 1
+              all_done.call
+            else
+              item = @list.shift
+              @pending += 1
+
+              is_done = false
+              on_done = proc{
+                raise RuntimeError, 'already completed this iteration' if is_done
+                is_done = true
+
+                @pending -= 1
+
+                if @ended
+                  all_done.call
+                else
+                  EM.next_tick(@process_next)
+                end
+              }
+              class << on_done
+                alias :next :call
+              end
+
+              foreach.call(item, on_done)
+            end
+          else
+            @workers -= 1
+          end
+        }
+
+        spawn_workers
+
+        self
+      end
+
+      # Collect the results of an asynchronous iteration into an array.
+      #
+      #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).map(proc{ |cmd,iter|
+      #    EM.system(cmd){ |output,status|
+      #      iter.return(output)
+      #    }
+      #  }, proc{ |results|
+      #    p results
+      #  })
+      #
+      def map(foreach, after)
+        index = 0
+
+        inject([], proc{ |results,item,iter|
+          i = index
+          index += 1
+
+          is_done = false
+          on_done = proc{ |res|
+            raise RuntimeError, 'already returned a value for this iteration' if is_done
+            is_done = true
+
+            results[i] = res
+            iter.return(results)
+          }
+          class << on_done
+            alias :return :call
+            def next
+              raise NoMethodError, 'must call #return on a map iterator'
+            end
+          end
+
+          foreach.call(item, on_done)
+        }, proc{ |results|
+          after.call(results)
+        })
+      end
+
+      # Inject the results of an asynchronous iteration onto a given object.
+      #
+      #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+      #    EM.system(cmd){ |output,status|
+      #      hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+      #      iter.return(hash)
+      #    }
+      #  }, proc{ |results|
+      #    p results
+      #  })
+      #
+      def inject(obj, foreach, after)
+        each(proc{ |item,iter|
+          is_done = false
+          on_done = proc{ |res|
+            raise RuntimeError, 'already returned a value for this iteration' if is_done
+            is_done = true
+
+            obj = res
+            iter.next
+          }
+          class << on_done
+            alias :return :call
+            def next
+              raise NoMethodError, 'must call #return on an inject iterator'
+            end
+          end
+
+          foreach.call(obj, item, on_done)
+        }, proc{
+          after.call(obj)
+        })
+      end
+
+      private
+
+      # Spawn workers to consume items from the iterator's enumerator based on the current concurrency level.
+      #
+      def spawn_workers
+        EM.next_tick(start_worker = proc{
+          if @workers < @concurrency and !@ended
+            # p [:spawning_worker, :workers=, @workers, :concurrency=, @concurrency, :ended=, @ended]
+            @workers += 1
+            @process_next.call
+            EM.next_tick(start_worker)
+          end
+        })
+        nil
+      end
+    end
+  end
+end
+

data/lib/z_k/z_k_event_machine/unixisms.rb

@@ -3,7 +3,7 @@ module ZK
     module Unixisms
       def mkdir_p(paths, &block)
         dfr = Deferred::Default.new.tap do |my_dfr|
-          EM::Iterator.new(Array(paths).flatten.compact, 1).map(
+          Iterator.new(Array(paths).flatten.compact, 1).map(
             lambda { |path,iter|          # foreach
               d = _mkdir_p_dfr(path)
               d.callback { |p| iter.return(p) }
@@ -18,7 +18,7 @@ module ZK
 
       def rm_rf(paths, &blk)
         dfr = Deferred::Default.new.tap do |my_dfr|
-          EM::Iterator.new(Array(paths).flatten.compact, 1).each(
+          Iterator.new(Array(paths).flatten.compact, 1).each(
             lambda { |path,iter|          # foreach
               d = _rm_rf_dfr(path)
               d.callback { iter.next }
@@ -31,15 +31,18 @@ module ZK
         _handle_calling_convention(dfr, &blk)
       end
 
+      # @private
       def find(*paths, &block)
         raise NotImplementedError, "Coming soon"
       end
 
+      # @private
       def block_until_node_deleted(abs_node_path)
         raise NotImplementedError, "blocking does not make sense in EventMachine-land"
       end
 
       protected
+        # @private
         def _handle_calling_convention(dfr, &blk)
           return dfr unless blk
           dfr.callback { |*a| blk.call(nil, *a) }
@@ -47,6 +50,7 @@ module ZK
           dfr
         end
 
+        # @private
         def _rm_rf_dfr(path)
           Deferred::Default.new.tap do |my_dfr|
             delete(path) do |exc|
@@ -60,7 +64,7 @@ module ZK
                     my_dfr.succeed
                   when nil
                     abspaths = chldrn.map { |n| [path, n].join('/') }
-                    EM::Iterator.new(abspaths).each(
+                    Iterator.new(abspaths).each(
                       lambda { |absp,iter|  
                         d = _rm_rf_dfr(absp)
                         d.callback  { |*| 
@@ -85,6 +89,7 @@ module ZK
           end
         end
 
+        # @private
         def _mkdir_p_dfr(path)
           Deferred::Default.new.tap do |my_dfr|
             d = create(path, '')

data/lib/z_k/z_k_event_machine/version.rb

@@ -1,5 +1,5 @@
 module ZK
   module ZKEventMachine
-    VERSION = "0.2.0.beta.3"
+    VERSION = "0.9.0"
   end
 end