zk-eventmachine 0.1.1

data/lib/z_k/z_k_event_machine/iterator.rb

@@ -0,0 +1,229 @@
+# Taken from EventMachine release candidate
+module ZK
+  module ZKEventMachine
+    # A simple iterator for concurrent asynchronous work.
+    #
+    # Unlike ruby's built-in iterators, the end of the current iteration cycle is signaled manually,
+    # instead of happening automatically after the yielded block finishes executing. For example:
+    #
+    #  (0..10).each{ |num| }
+    #
+    # becomes:
+    #
+    #  EM::Iterator.new(0..10).each{ |num,iter| iter.next }
+    #
+    # This is especially useful when doing asynchronous work via reactor libraries and
+    # functions. For example, given a sync and async http api:
+    #
+    #  response = sync_http_get(url); ...
+    #  async_http_get(url){ |response| ... }
+    #
+    # a synchronous iterator such as:
+    #
+    #  responses = urls.map{ |url| sync_http_get(url) }
+    #  ...
+    #  puts 'all done!'
+    #
+    # could be written as:
+    #
+    #  EM::Iterator.new(urls).map(proc{ |url,iter|
+    #    async_http_get(url){ |res|
+    #      iter.return(res)
+    #    }
+    #  }, proc{ |responses|
+    #    ...
+    #    puts 'all done!'
+    #  })
+    #
+    # Now, you can take advantage of the asynchronous api to issue requests in parallel. For example,
+    # to fetch 10 urls at a time, simply pass in a concurrency of 10:
+    #
+    #  EM::Iterator.new(urls, 10).each do |url,iter|
+    #    async_http_get(url){ iter.next }
+    #  end
+    #
+    class Iterator
+      # Create a new parallel async iterator with specified concurrency.
+      #
+      #  i = EM::Iterator.new(1..100, 10)
+      #
+      # will create an iterator over the range that processes 10 items at a time. Iteration
+      # is started via #each, #map or #inject
+      #
+      def initialize(list, concurrency = 1)
+        raise ArgumentError, 'argument must be an array' unless list.respond_to?(:to_a)
+        @list = list.to_a.dup
+        @concurrency = concurrency
+
+        @started = false
+        @ended = false
+      end
+
+      # Change the concurrency of this iterator. Workers will automatically be spawned or destroyed
+      # to accomodate the new concurrency level.
+      #
+      def concurrency=(val)
+        old = @concurrency
+        @concurrency = val
+
+        spawn_workers if val > old and @started and !@ended
+      end
+      attr_reader :concurrency
+
+      # Iterate over a set of items using the specified block or proc.
+      #
+      #  EM::Iterator.new(1..100).each do |num, iter|
+      #    puts num
+      #    iter.next
+      #  end
+      #
+      # An optional second proc is invoked after the iteration is complete.
+      #
+      #  EM::Iterator.new(1..100).each(
+      #    proc{ |num,iter| iter.next },
+      #    proc{ puts 'all done' }
+      #  )
+      #
+      def each(foreach=nil, after=nil, &blk)
+        raise ArgumentError, 'proc or block required for iteration' unless foreach ||= blk
+        raise RuntimeError, 'cannot iterate over an iterator more than once' if @started or @ended
+
+        @started = true
+        @pending = 0
+        @workers = 0
+
+        all_done = proc{
+          after.call if after and @ended and @pending == 0
+        }
+
+        @process_next = proc{
+          # p [:process_next, :pending=, @pending, :workers=, @workers, :ended=, @ended, :concurrency=, @concurrency, :list=, @list]
+          unless @ended or @workers > @concurrency
+            if @list.empty?
+              @ended = true
+              @workers -= 1
+              all_done.call
+            else
+              item = @list.shift
+              @pending += 1
+
+              is_done = false
+              on_done = proc{
+                raise RuntimeError, 'already completed this iteration' if is_done
+                is_done = true
+
+                @pending -= 1
+
+                if @ended
+                  all_done.call
+                else
+                  EM.next_tick(@process_next)
+                end
+              }
+              class << on_done
+                alias :next :call
+              end
+
+              foreach.call(item, on_done)
+            end
+          else
+            @workers -= 1
+          end
+        }
+
+        spawn_workers
+
+        self
+      end
+
+      # Collect the results of an asynchronous iteration into an array.
+      #
+      #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).map(proc{ |cmd,iter|
+      #    EM.system(cmd){ |output,status|
+      #      iter.return(output)
+      #    }
+      #  }, proc{ |results|
+      #    p results
+      #  })
+      #
+      def map(foreach, after)
+        index = 0
+
+        inject([], proc{ |results,item,iter|
+          i = index
+          index += 1
+
+          is_done = false
+          on_done = proc{ |res|
+            raise RuntimeError, 'already returned a value for this iteration' if is_done
+            is_done = true
+
+            results[i] = res
+            iter.return(results)
+          }
+          class << on_done
+            alias :return :call
+            def next
+              raise NoMethodError, 'must call #return on a map iterator'
+            end
+          end
+
+          foreach.call(item, on_done)
+        }, proc{ |results|
+          after.call(results)
+        })
+      end
+
+      # Inject the results of an asynchronous iteration onto a given object.
+      #
+      #  EM::Iterator.new(%w[ pwd uptime uname date ], 2).inject({}, proc{ |hash,cmd,iter|
+      #    EM.system(cmd){ |output,status|
+      #      hash[cmd] = status.exitstatus == 0 ? output.strip : nil
+      #      iter.return(hash)
+      #    }
+      #  }, proc{ |results|
+      #    p results
+      #  })
+      #
+      def inject(obj, foreach, after)
+        each(proc{ |item,iter|
+          is_done = false
+          on_done = proc{ |res|
+            raise RuntimeError, 'already returned a value for this iteration' if is_done
+            is_done = true
+
+            obj = res
+            iter.next
+          }
+          class << on_done
+            alias :return :call
+            def next
+              raise NoMethodError, 'must call #return on an inject iterator'
+            end
+          end
+
+          foreach.call(obj, item, on_done)
+        }, proc{
+          after.call(obj)
+        })
+      end
+
+      private
+
+      # Spawn workers to consume items from the iterator's enumerator based on the current concurrency level.
+      #
+      def spawn_workers
+        EM.next_tick(start_worker = proc{
+          if @workers < @concurrency and !@ended
+            # p [:spawning_worker, :workers=, @workers, :concurrency=, @concurrency, :ended=, @ended]
+            @workers += 1
+            @process_next.call
+            EM.next_tick(start_worker)
+          end
+        })
+        nil
+      end
+    end
+  end
+end
+

data/lib/z_k/z_k_event_machine/unixisms.rb

@@ -0,0 +1,124 @@
+module ZK
+  module ZKEventMachine
+    module Unixisms
+      def mkdir_p(paths, &block)
+        dfr = Deferred::Default.new.tap do |my_dfr|
+          Iterator.new(Array(paths).flatten.compact, 1).map(
+            lambda { |path,iter|          # foreach
+              d = _mkdir_p_dfr(path)
+              d.callback { |p| iter.return(p) }
+              d.errback  { |e| my_dfr.fail(e) }
+            },
+            lambda { |results| my_dfr.succeed(results) }     # after completion
+          )
+        end
+
+        _handle_calling_convention(dfr, &block)
+      end
+
+      def rm_rf(paths, &blk)
+        dfr = Deferred::Default.new.tap do |my_dfr|
+          Iterator.new(Array(paths).flatten.compact, 1).each(
+            lambda { |path,iter|          # foreach
+              d = _rm_rf_dfr(path)
+              d.callback { iter.next }
+              d.errback  { |e| my_dfr.fail(e) }
+            },
+            lambda { my_dfr.succeed }     # after completion
+          )
+        end
+
+        _handle_calling_convention(dfr, &blk)
+      end
+
+      def find(*paths, &block)
+        raise NotImplementedError, "Coming soon"
+      end
+
+      def block_until_node_deleted(abs_node_path)
+        raise NotImplementedError, "blocking does not make sense in EventMachine-land"
+      end
+
+      protected
+        def _handle_calling_convention(dfr, &blk)
+          return dfr unless blk
+          dfr.callback { |*a| blk.call(nil, *a) }
+          dfr.errback { |exc| blk.call(exc) }
+          dfr
+        end
+
+        def _rm_rf_dfr(path)
+          Deferred::Default.new.tap do |my_dfr|
+            delete(path) do |exc|
+              case exc
+              when nil, Exceptions::NoNode
+                my_dfr.succeed
+              when Exceptions::NotEmpty
+                children(path) do |exc,chldrn,_|
+                  case exc
+                  when Exceptions::NoNode
+                    my_dfr.succeed
+                  when nil
+                    abspaths = chldrn.map { |n| [path, n].join('/') }
+                    Iterator.new(abspaths).each(
+                      lambda { |absp,iter|  
+                        d = _rm_rf_dfr(absp)
+                        d.callback  { |*| 
+                          logger.debug { "removed #{absp}" }
+                          iter.next
+                        }
+                        d.errback   { |e| 
+                          logger.debug { "got failure #{e.inspect}" }
+                          my_dfr.fail(e)   # this will stop the iteration
+                        }
+                      },
+                      lambda { 
+                        my_dfr.chain_to(_rm_rf_dfr(path))
+                      }
+                    )
+                  else
+                    my_dfr.fail(exc)
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def _mkdir_p_dfr(path)
+          Deferred::Default.new.tap do |my_dfr|
+            d = create(path, '')
+
+            d.callback do |new_path|
+              my_dfr.succeed(new_path)
+            end
+
+            d.errback do |exc|
+              case exc
+              when Exceptions::NodeExists
+                # this is the bottom of the stack, where we start bubbling back up
+                # or the first call, path already exists, return
+                my_dfr.succeed(path)
+              when Exceptions::NoNode
+                # our node didn't exist now, so we try an recreate it after our
+                # parent has been created
+
+                parent_d = mkdir_p(File.dirname(path))            # set up our parent to be created
+
+                parent_d.callback do |parent_path|                # once our parent exists
+                  create(path, '') do |exc,p|                     # create our path again
+                    exc ? my_dfr.fail(exc) : my_dfr.succeed(p)    # pass our success or failure up the chain
+                  end
+                end
+
+                parent_d.errback do |e|                           # if creating our parent fails
+                  my_dfr.fail(e)                                  # pass that along too
+                end
+              end
+            end
+          end
+        end
+    end
+  end
+end
+

data/lib/z_k/z_k_event_machine/version.rb

@@ -0,0 +1,5 @@
+module ZK
+  module ZKEventMachine
+    VERSION = "0.1.1"
+  end
+end

data/lib/z_k/z_k_event_machine.rb

@@ -0,0 +1,24 @@
+require 'eventmachine'
+
+require 'zookeeper'
+require 'zookeeper/em_client'
+
+require 'zk'
+
+
+module ZK
+  module ZKEventMachine
+  end
+end
+
+
+$LOAD_PATH.unshift(File.expand_path('../..', __FILE__)).uniq!
+
+require 'z_k/z_k_event_machine/iterator'
+require 'z_k/z_k_event_machine/deferred'
+require 'z_k/z_k_event_machine/callback'
+require 'z_k/z_k_event_machine/event_handler_e_m'
+require 'z_k/z_k_event_machine/unixisms'
+require 'z_k/z_k_event_machine/client'
+
+

data/lib/zk-eventmachine.rb

@@ -0,0 +1,2 @@
+require File.expand_path('../z_k/z_k_event_machine', __FILE__)
+

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'rubygems'
+require 'bundler/setup'
+
+$LOAD_PATH.unshift(File.expand_path('../../lib', __FILE__))
+
+require 'zk-eventmachine'
+require 'evented-spec'
+
+# Requires supporting ruby files with custom matchers and macros, etc,
+# in spec/support/ and its subdirectories.
+Dir[File.expand_path("../support/**/*.rb", __FILE__)].each {|f| require f}
+
+$stderr.sync = true
+
+RSpec.configure do |config|
+  config.mock_with :flexmock
+end
+
+

data/spec/support/extensions.rb

@@ -0,0 +1,44 @@
+# method to wait until block passed returns true or timeout (default is 2 seconds) is reached 
+def wait_until(timeout=2)
+  time_to_stop = Time.now + timeout
+
+  until yield 
+    break if Time.now > time_to_stop
+    Thread.pass
+  end
+end
+
+def wait_while(timeout=2)
+  time_to_stop = Time.now + timeout
+
+  while yield 
+    break if Time.now > time_to_stop
+    Thread.pass
+  end
+end
+
+class ::Thread
+  # join with thread until given block is true, the thread joins successfully, 
+  # or timeout seconds have passed
+  #
+  def join_until(timeout=2)
+    time_to_stop = Time.now + timeout
+
+    until yield
+      break if Time.now > time_to_stop
+      break if join(0.1)
+    end
+  end
+  
+  def join_while(timeout=2)
+    time_to_stop = Time.now + timeout
+
+    while yield
+      break if Time.now > time_to_stop
+      break if join(0.1)
+    end
+  end
+end
+
+
+

data/spec/support/logging.rb

@@ -0,0 +1,9 @@
+require 'logger'
+
+ZK.logger = Logger.new(File.expand_path('../../../test.log', __FILE__)).tap {|l| l.level = Logger::DEBUG}
+Zookeeper.logger = ZK.logger
+
+def logger
+  ZK.logger
+end
+

data/spec/support/logging_progress_bar_formatter.rb

@@ -0,0 +1,26 @@
+require 'rspec/core/formatters/progress_formatter'
+
+# module Motionbox
+#   # essentially a monkey-patch to the ProgressBarFormatter, outputs 
+#   # '== #{example_proxy.description} ==' in the logs before each test.  makes it
+#   # easier to match up tests with the SQL they produce
+#   class LoggingProgressBarFormatter < RSpec::Core::Formatters::ProgressFormatter
+#     def example_started(example)
+#       ZK.logger.info(yellow("\n=====<([ #{example.full_description} ])>=====\n"))
+#       super
+#     end
+#   end
+# end
+
+module RSpec
+  module Core
+    module Formatters
+      class ProgressFormatter
+        def example_started(example)
+          ZK.logger.info(yellow("\n=====<([ #{example.full_description} ])>=====\n"))
+          super(example)
+        end
+      end
+    end
+  end
+end

data/spec/z_k/z_k_event_machine/callback_spec.rb

@@ -0,0 +1,135 @@
+require 'spec_helper'
+
+describe 'ZK::ZKEventMachine::Callback' do
+  before do
+    @stat_mock = flexmock(:stat)
+    @context_mock = flexmock(:context)
+
+    flexmock(::EM) do |em|
+      em.should_receive(:next_tick).with(Proc).and_return { |b| b.call }
+    end
+
+  end
+
+  describe 'DataCallback' do
+    before do
+      @cb = ZK::ZKEventMachine::Callback::DataCallback.new
+    end
+
+    describe 'call' do
+      describe 'with callbacks and errbacks set' do
+        before do
+          @callback_args = @errback_args = nil
+
+          @cb.callback do |*a|
+            @callback_args = a
+          end
+
+          @cb.errback do |*a|
+            @errback_args = a
+          end
+        end
+
+        describe 'success' do
+          before do
+            @cb.call(:rc => 0, :data => 'data', :stat => @stat_mock, :context => @context_mock)
+          end
+
+          it %[should have called the callback] do
+            @callback_args.should_not be_nil
+          end
+
+          it %[should have the correct number of args] do
+            @callback_args.length.should == 2
+          end
+
+          it %[should have the correct args] do
+            @callback_args[0].should == 'data'
+            @callback_args[1].should == @stat_mock
+          end
+        end
+
+        describe 'failure' do
+          before do
+            @cb.call(:rc => ::ZK::Exceptions::NONODE)
+          end
+
+          it %[should have called the errback] do
+            @errback_args.should_not be_nil
+          end
+
+          it %[should be called with the appropriate exception instance] do
+            @errback_args.first.should be_instance_of(::ZK::Exceptions::NoNode)
+          end
+        end
+      end
+
+      describe 'with an on_result block set' do
+        before do
+          @args = nil
+
+          @cb.on_result do |*a|
+            @args = a
+          end
+        end
+
+        describe 'success' do
+          before do
+            @cb.call(:rc => 0, :data => 'data', :stat => @stat_mock, :context => @context_mock)
+          end
+
+          it %[should have called the block] do
+            @args.should_not be_nil
+          end
+
+          it %[should have used the correct arguments] do
+            @args[0].should == nil
+            @args[1].should == 'data'
+            @args[2].should == @stat_mock
+          end
+        end
+
+        describe 'failure' do
+          before do
+            @cb.call(:rc => ::ZK::Exceptions::NONODE)
+          end
+
+          it %[should have called the block] do
+            @args.should_not be_nil
+          end
+
+          it %[should have used the correct arguments] do
+            @args.first.should be_instance_of(::ZK::Exceptions::NoNode)
+          end
+        end
+      end
+
+      describe 'on_result can be handed a block' do
+        before do
+          @args = nil
+
+          blk = lambda { |*a| @args = a }
+
+          @cb.on_result(blk)
+        end
+
+        describe 'success' do
+          before do
+            @cb.call(:rc => 0, :data => 'data', :stat => @stat_mock, :context => @context_mock)
+          end
+
+          it %[should have called the block] do
+            @args.should_not be_nil
+          end
+
+          it %[should have used the correct arguments] do
+            @args[0].should == nil
+            @args[1].should == 'data'
+            @args[2].should == @stat_mock
+          end
+        end
+      end
+    end
+  end
+end
+