hoodoo 1.0.2

data/spec/client/endpoint/endpoint_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+# Much of this class gets nontrivial coverage from middleware tests, which
+# existed before the refactor of code from there, into endpoints.
+#
+# This file just picks up the loose ends.
+
+describe Hoodoo::Client::Endpoint do
+  it 'complains about unknown discovery results' do
+    mock_discoverer = OpenStruct.new
+    expect( mock_discoverer ).to receive( :discover ).once.and_return( OpenStruct.new )
+
+    expect {
+      described_class.endpoint_for( :Anything, 1, { :discoverer => mock_discoverer } )
+    }.to raise_error( RuntimeError, "Hoodoo::Client::Endpoint::endpoint_for: Unrecognised discoverer result class of 'OpenStruct'" )
+  end
+
+  context 'with subclass mandatory methods' do
+
+    # Create a subclas that must at least implement configure_with, else
+    # instantiation would fail as the constructor calls this.
+    #
+    class RSpecBadEndpointSubclass < described_class
+      def configure_with( a, b, c )
+      end
+    end
+
+    before :each do
+      @endpoint = RSpecBadEndpointSubclass.new(
+        :Anything,
+        1,
+        {}
+      )
+    end
+
+    it 'complains about missing #configure_with' do
+      # Trying to instantiate Endpoint directly leads it to call its own
+      # implementation of #configure_with, which should raise an exception.
+
+      expect {
+        described_class.new(
+          :Anything,
+          1,
+          {}
+        )
+      }.to raise_error( RuntimeError, 'Subclasses must implement Hoodoo::Client::Endpoint#configure_with' )
+    end
+
+    it 'complains about missing #list' do
+      expect { @endpoint.list }.to raise_error( RuntimeError, 'Subclasses must implement Hoodoo::Client::Endpoint#list' )
+    end
+
+    it 'complains about missing #show' do
+      expect { @endpoint.show( 'foo' ) }.to raise_error( RuntimeError, 'Subclasses must implement Hoodoo::Client::Endpoint#show' )
+    end
+
+    it 'complains about missing #create' do
+      expect { @endpoint.create( {} ) }.to raise_error( RuntimeError, 'Subclasses must implement Hoodoo::Client::Endpoint#create' )
+    end
+
+    it 'complains about missing #update' do
+      expect { @endpoint.update( 'foo', {} ) }.to raise_error( RuntimeError, 'Subclasses must implement Hoodoo::Client::Endpoint#update' )
+    end
+
+    it 'complains about missing #delete' do
+      expect { @endpoint.delete( 'foo' ) }.to raise_error( RuntimeError, 'Subclasses must implement Hoodoo::Client::Endpoint#delete' )
+    end
+
+  end
+end

data/spec/client/endpoint/endpoints/amqp_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+# Much of this class gets nontrivial coverage from middleware tests, which
+# existed before the refactor of code from there, into endpoints.
+#
+# This file just picks up the loose ends.
+
+describe Hoodoo::Client::Endpoint::AMQP do
+
+  it 'complains if instantiated with the wrong discovery result type' do
+    expect {
+      described_class.new( :Anything, 1, { :discovery_result => OpenStruct.new } )
+    }.to raise_error( RuntimeError, "Hoodoo::Client::Endpoint::AMQP must be configured with a Hoodoo::Services::Discovery::ForAMQP instance - got 'OpenStruct'" )
+  end
+
+end

data/spec/client/endpoint/endpoints/auto_session_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe Hoodoo::Client::Endpoint::AutoSession do
+
+  # No tests here yet. Full coverage is achieved by other tests running
+  # through the HTTP and AMQP endpoints (amongst other stuff), since
+  # those inherit from this code.
+
+end

data/spec/client/endpoint/endpoints/http_based_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe Hoodoo::Client::Endpoint::HTTPBased do
+
+  # No tests here yet. Full coverage is achieved by other tests running
+  # through the HTTP and AMQP endpoints (amongst other stuff), since
+  # those inherit from this code.
+
+end

data/spec/client/endpoint/endpoints/http_spec.rb

@@ -0,0 +1,103 @@
+require 'spec_helper'
+
+# Much of this class gets nontrivial coverage from middleware tests, which
+# existed before the refactor of code from there, into Endpoints.
+#
+# This file just picks up the loose ends.
+
+describe Hoodoo::Client::Endpoint::HTTP do
+
+  it 'complains if instantiated with the wrong discovery result type' do
+    expect {
+      described_class.new( :Anything, 1, { :discovery_result => OpenStruct.new } )
+    }.to raise_error( RuntimeError, "Hoodoo::Client::Endpoint::HTTP must be configured with a Hoodoo::Services::Discovery::ForHTTP instance - got 'OpenStruct'" )
+  end
+
+  context 'SSL Cert chain verification' do
+
+    before(:all) do
+      # Start a service listening on https 127.0.0.1 with a self-signed cert.
+      #
+      # Note: we are skipping hoodoo middleware with +skip_hoodoo_middleware+
+      # because this test is about verifying SSL on the client, and we need
+      # to be certain that any errors thrown are from client, and not from
+      # middleware (e.g. server).
+      #
+      # This test effectively ensures we aren't vulnerable to a MiTM attack, and malicious parties
+      # won't be good upstanding Hoodoo::Middleware using citizens.
+      #
+      @https_port = spec_helper_start_svc_app_in_thread_for(SslSelfSignedApp, true, skip_hoodoo_middleware: true)
+    end
+
+    it 'should successfuly connect with valid certificate chain' do
+      endpoint = connect_to_real_https_endpoint('127.0.0.1', 'spec/files/ca/ca-cert.pem')
+      response = endpoint.list()
+      expect(response["message"]).to eq("This data is a secret")
+    end
+
+    it "should fail when certificate doesn't match the hostname" do
+      endpoint = connect_to_real_https_endpoint('localhost', 'spec/files/ca/ca-cert.pem')
+      response = endpoint.list()
+      expect(response.platform_errors.has_errors?).to eq(true)
+      expect(response.platform_errors.errors.first["code"]).to eq("platform.fault")
+    end
+
+    it "should fail when the certificate isn't in the ca_file" do
+      endpoint = connect_to_real_https_endpoint('127.0.0.1', nil)
+      response = endpoint.list()
+      expect(response.platform_errors.has_errors?).to eq(true)
+      expect(response.platform_errors.errors.first["code"]).to eq("platform.fault")
+    end
+
+    class SslSelfSignedApp
+      def call(env)
+        return [200, {'Content-Type' => 'application/json'}, ['{"message": "This data is a secret"}'] ]
+      end
+    end
+
+    def connect_to_real_https_endpoint(hostname, ca_file)
+      mw = SslSelfSignedApp.new
+      interaction = Hoodoo::Services::Middleware::Interaction.new(
+        {},
+        mw,
+        Hoodoo::Services::Middleware.test_session()
+      )
+      interaction.target_interface = OpenStruct.new
+
+      mock_wrapped_discovery_result = Hoodoo::Services::Discovery::ForHTTP.new(
+        resource:     'SecureData',
+        version:      2,
+        endpoint_uri: URI.parse( "https://#{hostname}:#{ @https_port }/v2/secure_data" ),
+        ca_file:      ca_file
+      )
+
+      mock_wrapped_endpoint = Hoodoo::Client::Endpoint::HTTP.new(
+        'SecureData',
+        2,
+        :session => Hoodoo::Services::Middleware.test_session(),
+        :discovery_result => mock_wrapped_discovery_result
+      )
+
+      # Synthesise a remote resource discovery result for the HTTP(S) endpoint
+      # built above and use that to make a remote call endpoint.
+
+      discovery_result = Hoodoo::Services::Discovery::ForRemote.new(
+        :resource         => 'SecureData',
+        :version          => 2,
+        :wrapped_endpoint => mock_wrapped_endpoint
+      )
+
+      endpoint = Hoodoo::Services::Middleware::InterResourceRemote.new(
+        'SecureData',
+        2,
+        {
+          :interaction      => interaction,
+          :discovery_result => discovery_result
+        }
+      )
+      return endpoint
+    end
+
+  end
+
+end

data/spec/client/endpoint/endpoints/not_found_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+describe Hoodoo::Client::Endpoint::NotFound do
+  before :each do
+    @endpoint = described_class.new( :NotFoundResource, 1, {} )
+  end
+
+  context 'produces "not found" result for' do
+    def run_expectations( result )
+      expect( result.platform_errors.has_errors? ).to eq( true )
+      expect( result.platform_errors.errors.size ).to eq( 1 )
+      expect( result.platform_errors.errors[ 0 ][ 'code' ] ).to eq( 'platform.not_found' )
+    end
+
+    it '#list' do
+      run_expectations( @endpoint.list() )
+    end
+
+    it '#show' do
+      run_expectations( @endpoint.show( 'foo' ) )
+    end
+
+    it '#create' do
+      run_expectations( @endpoint.create( {} ) )
+    end
+
+    it '#update' do
+      run_expectations( @endpoint.update( 'foo', {} ) )
+    end
+
+    it '#delete' do
+      run_expectations( @endpoint.delete( 'foo' ) )
+    end
+  end
+end

data/spec/client/headers_spec.rb

@@ -0,0 +1,172 @@
+require 'spec_helper'
+
+# Full test coverage already happens "by osmosis" through "client_spec.rb" and
+# the middleware suite.
+
+describe Hoodoo::Client::Headers do
+  context 'UUID_PROPERTY_PROC' do
+    it 'converts valid values' do
+      uuid = Hoodoo::UUID.generate()
+      expect( described_class::UUID_PROPERTY_PROC.call( uuid ) ).to eq( uuid )
+    end
+
+    it 'rejects invalid values' do
+      uuid = "not a UUID"
+      expect( described_class::UUID_PROPERTY_PROC.call( uuid ) ).to be_nil
+    end
+  end
+
+  context 'UUID_HEADER_PROC' do
+    it 'converts values' do
+      uuid = Hoodoo::UUID.generate()
+      expect( described_class::UUID_HEADER_PROC.call( uuid ) ).to eq( uuid )
+    end
+  end
+
+  context 'DATETIME_IN_PAST_ONLY_PROPERTY_PROC' do
+    it 'converts valid values' do
+      date_time = DateTime.now - 10.seconds
+      date_time_str = Hoodoo::Utilities.nanosecond_iso8601( date_time )
+      expect( described_class::DATETIME_IN_PAST_ONLY_PROPERTY_PROC.call( date_time_str ) ).to eq( date_time )
+    end
+
+    context 'rejects invalid values' do
+      it 'that are not date/times' do
+        date_time_str = 'not a date/time'
+        expect( described_class::DATETIME_IN_PAST_ONLY_PROPERTY_PROC.call( date_time_str ) ).to be_nil
+      end
+
+      it 'that are Ruby-valid but not in supported ISO 8601 subset format' do
+        date_time_str = Time.now.to_s
+        expect( described_class::DATETIME_IN_PAST_ONLY_PROPERTY_PROC.call( date_time_str ) ).to be_nil
+      end
+
+      it 'that are valid but in the future' do
+        date_time = DateTime.now + 1.hour
+        date_time_str = Hoodoo::Utilities.nanosecond_iso8601( date_time )
+        expect( described_class::DATETIME_IN_PAST_ONLY_PROPERTY_PROC.call( date_time_str ) ).to be_nil
+      end
+    end
+  end
+
+  it 'DATETIME_WRITER_PROC calls rationalisation method' do
+    now = Time.now
+    expect( Hoodoo::Utilities ).to receive( :rationalise_datetime ).once.with( now ).and_call_original
+    expect( described_class::DATETIME_WRITER_PROC.call( now ) ).to eq( now.to_datetime )
+  end
+
+  context 'DATETIME_HEADER_PROC converts values' do
+    it 'that are DateTime instances' do
+      date_time = DateTime.now - 10.seconds
+      date_time_str = Hoodoo::Utilities.nanosecond_iso8601( date_time )
+      expect( described_class::DATETIME_HEADER_PROC.call( date_time ) ).to eq( date_time_str )
+    end
+
+    # Technically undocumented but highly likely to happen by accident and
+    # the Proc is written using the Hoodoo::Utilities support methods so
+    # this should always work.
+    #
+    it 'that are Time instances' do
+      time = Time.now - 10.seconds
+      time_str = Hoodoo::Utilities.nanosecond_iso8601( time )
+      expect( described_class::DATETIME_HEADER_PROC.call( time ) ).to eq( time_str )
+    end
+  end
+
+  context 'BOOLEAN_PROPERTY_PROC converts values' do
+    it 'that are yes/no' do
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'yes' ) ).to eq( true  )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'Yes' ) ).to eq( true  )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'YES' ) ).to eq( true  )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'no'  ) ).to eq( false )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'No'  ) ).to eq( false )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'NO'  ) ).to eq( false )
+    end
+
+    it 'that are not yes/no' do
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( nil     ) ).to eq( false )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( ''      ) ).to eq( false )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'Yess'  ) ).to eq( false )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'oui'   ) ).to eq( false )
+      expect( described_class::BOOLEAN_PROPERTY_PROC.call( 'true'  ) ).to eq( false )
+    end
+  end
+
+  context 'BOOLEAN_HEADER_PROC converts values' do
+    it 'that are true/false' do
+      expect( described_class::BOOLEAN_HEADER_PROC.call( true   ) ).to eq( 'yes' )
+      expect( described_class::BOOLEAN_HEADER_PROC.call( false  ) ).to eq( 'no' )
+    end
+
+    it 'that are not true/false' do
+      expect( described_class::BOOLEAN_HEADER_PROC.call( nil    ) ).to eq( 'no' )
+      expect( described_class::BOOLEAN_HEADER_PROC.call( 'yes'  ) ).to eq( 'no' )
+      expect( described_class::BOOLEAN_HEADER_PROC.call( 'true' ) ).to eq( 'no' )
+    end
+  end
+
+  context 'HEADER_TO_PROPERTY' do
+    it 'contains the minimum required properties for all descriptions' do
+      expect( described_class::HEADER_TO_PROPERTY.keys.size ).to_not eq( 0 )
+
+      described_class::HEADER_TO_PROPERTY.each do | rack_header, description |
+        expect( rack_header[ 0..6 ] ).to eq( 'HTTP_X_' )
+
+        expect( description[ :property        ] ).to_not be_nil
+        expect( description[ :property_writer ] ).to     eq( "#{ description[ :property ] }=" )
+        expect( description[ :property_writer ] ).to_not eq( '=' )
+        expect( description[ :property_proc   ] ).to     be_a( Proc )
+        expect( description[ :header          ] ).to_not be_nil
+        expect( description[ :header_proc     ] ).to     be_a( Proc )
+      end
+    end
+  end
+
+  # Slightly weak but test coverage for the varied bespoke property writers
+  # - which have individual requirements on data types and so-on - gets
+  # brought in by all the other tests which explicitly check that headers
+  # in the HEADER_TO_PROPERTY set are working properly. In the end, it all
+  # gets proper coverage.
+  #
+  context '#define_accessors_for_header_equivalents' do
+    class RSpecDefineAccessorsForHeaderEquivalentsTest
+    end
+
+    before :all do
+      described_class.define_accessors_for_header_equivalents( RSpecDefineAccessorsForHeaderEquivalentsTest )
+      @instance = RSpecDefineAccessorsForHeaderEquivalentsTest.new
+    end
+
+    it 'defines read accessors' do
+      described_class::HEADER_TO_PROPERTY.each do | rack_header, description |
+        method = description[ :property ]
+        expect( @instance ).to respond_to( method )
+      end
+    end
+
+    it 'defines write accessors' do
+      described_class::HEADER_TO_PROPERTY.each do | rack_header, description |
+        method = description[ :property_writer ]
+        expect( @instance ).to respond_to( method )
+      end
+    end
+  end
+
+  context '#x_header_to_options' do
+    it 'converts headers' do
+      hash = {
+        'x-interaction-id'      => '23',
+        'X-Foo-Bar'             => '42',
+        'x_underscored_item'    => 'hello world',
+        'X_CAPITAL_UNDERSCORES' => 'yes'
+      }
+
+      options = Hoodoo::Client::Headers.x_header_to_options( hash )
+
+      expect( options[ 'interaction_id'      ] ).to eq( '23'          )
+      expect( options[ 'foo_bar'             ] ).to eq( '42'          )
+      expect( options[ 'underscored_item'    ] ).to eq( 'hello world' )
+      expect( options[ 'capital_underscores' ] ).to eq( 'yes'         )
+    end
+  end
+end

data/spec/communicators/fast_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe Hoodoo::Communicators::Fast do
+  it 'complains if called directly' do
+    expect {
+      Hoodoo::Communicators::Fast.new.communicate( {} )
+    }.to raise_error( RuntimeError )
+  end
+end

data/spec/communicators/pool_spec.rb

@@ -0,0 +1,339 @@
+require 'spec_helper'
+
+describe Hoodoo::Communicators::Pool do
+
+  before :each do
+    @pool = Hoodoo::Communicators::Pool.new
+  end
+
+  context 'general operation' do
+    after :each do
+      @pool.wait()
+    end
+
+    class TestFastCommunicatorA < Hoodoo::Communicators::Fast
+      def communicate( obj )
+        expectable_hook_fast_a( obj )
+      end
+    end
+
+    class TestFastCommunicatorB < Hoodoo::Communicators::Fast
+      def communicate( obj )
+        expectable_hook_fast_b( obj )
+      end
+    end
+
+    class TestFastCommunicatorC < Hoodoo::Communicators::Fast
+      def communicate( obj )
+        raise 'I am broken (Fast C)'
+      end
+    end
+
+    class TestSlowCommunicatorA < Hoodoo::Communicators::Slow
+      def communicate( obj )
+        expectable_hook_slow_a( obj )
+        sleep 0.2 # Deliberate delay to make sure #wait() works;
+                  # intermittent failures would imply it doesn't.
+      end
+    end
+
+    class TestSlowCommunicatorB < Hoodoo::Communicators::Slow
+      def communicate( obj )
+        expectable_hook_slow_b( obj )
+      end
+    end
+
+    class TestSlowCommunicatorC < Hoodoo::Communicators::Slow
+      def communicate( obj )
+        raise 'I am broken (Slow C)'
+      end
+    end
+
+    it 'lets me add and remove handlers' do
+      c1 = @pool.add( TestFastCommunicatorA.new )
+      c2 = @pool.add( TestFastCommunicatorB.new )
+      c3 = @pool.add( TestSlowCommunicatorA.new )
+      c4 = @pool.add( TestSlowCommunicatorB.new )
+
+      # Should be able to remove them all
+
+      @pool.remove( c1 )
+      @pool.remove( c2 )
+      @pool.remove( c3 )
+      @pool.remove( c4 )
+
+      expect( @pool.instance_variable_get( '@pool' ) ).to be_empty
+
+      # Should be harmess to remove the same thing twice, or remove an instance
+      # not in the pool.
+
+      @pool.remove( c1 )
+      @pool.remove( TestSlowCommunicatorB.new )
+
+      expect( @pool.instance_variable_get( '@pool' ) ).to be_empty
+    end
+
+    it 'calls fast handler A' do
+      c1 = @pool.add( TestFastCommunicatorA.new )
+      ob = { :foo => :bar }
+      expect( c1 ).to receive( :expectable_hook_fast_a ).once.with( ob )
+      @pool.communicate( ob )
+    end
+
+    it 'calls fast handler A and B' do
+      c1 = @pool.add( TestFastCommunicatorA.new )
+      c2 = @pool.add( TestFastCommunicatorB.new )
+      ob = { :foo => :bar }
+      expect( c1 ).to receive( :expectable_hook_fast_a ).once.with( ob )
+      expect( c2 ).to receive( :expectable_hook_fast_b ).once.with( ob )
+      @pool.communicate( ob )
+    end
+
+    it 'calls slow handler A' do
+      c1 = @pool.add( TestSlowCommunicatorA.new )
+      ob = { :bar => :baz }
+      expect( c1 ).to receive( :expectable_hook_slow_a ).once.with( ob )
+      @pool.communicate( ob )
+    end
+
+    it 'calls slow handler A and B' do
+      c1 = @pool.add( TestSlowCommunicatorA.new )
+      c2 = @pool.add( TestSlowCommunicatorB.new )
+      ob = { :bar => :baz }
+      expect( c1 ).to receive( :expectable_hook_slow_a ).once.with( ob )
+      expect( c2 ).to receive( :expectable_hook_slow_b ).once.with( ob )
+      @pool.communicate( ob )
+    end
+
+    it 'calls all handlers' do
+      c1 = @pool.add( TestFastCommunicatorA.new )
+      c2 = @pool.add( TestFastCommunicatorB.new )
+      c3 = @pool.add( TestSlowCommunicatorA.new )
+      c4 = @pool.add( TestSlowCommunicatorB.new )
+      ob = { :baz => :foo }
+      expect( c1 ).to receive( :expectable_hook_fast_a ).once.with( ob )
+      expect( c2 ).to receive( :expectable_hook_fast_b ).once.with( ob )
+      expect( c3 ).to receive( :expectable_hook_slow_a ).once.with( ob )
+      expect( c4 ).to receive( :expectable_hook_slow_b ).once.with( ob )
+      @pool.communicate( ob )
+    end
+
+    it 'complains about bad additions' do
+      expect {
+        @pool.add( Object )
+      }.to raise_exception( RuntimeError )
+    end
+
+    it 'complains about bad removals' do
+      expect {
+        @pool.remove( Object )
+      }.to raise_exception( RuntimeError )
+    end
+
+    it 'ignores exceptions in reporters' do
+      c1 = @pool.add( TestFastCommunicatorC.new ) # Add exception raisers first
+      c2 = @pool.add( TestSlowCommunicatorC.new )
+      c3 = @pool.add( TestFastCommunicatorA.new ) # Then these after, which should still be called
+      c4 = @pool.add( TestSlowCommunicatorA.new )
+      ob = { :foo => :bar }
+      expect( c3 ).to receive( :expectable_hook_fast_a ).once.with( ob )
+      expect( c4 ).to receive( :expectable_hook_slow_a ).once.with( ob )
+      @pool.communicate( ob )
+    end
+
+    it 'ignores exceptions in exception handler' do
+      c1 = @pool.add( TestFastCommunicatorC.new ) # Add exception raisers first
+      c2 = @pool.add( TestSlowCommunicatorC.new )
+      c3 = @pool.add( TestFastCommunicatorA.new ) # Then these after, which should still be called
+      c4 = @pool.add( TestSlowCommunicatorA.new )
+      ob = { :foo => :bar }
+
+      # Expect a debug report on 'stderr'; force the first one to fail, let
+      # the next one through quietly. Everything should be called as expected
+      # even though 'stderr' failed.
+
+      expect( $stderr ).to receive( :puts ).once do
+        raise 'stderr failure'
+      end
+      expect( $stderr ).to receive( :puts ).once.and_call_original
+      expect( c3 ).to receive( :expectable_hook_fast_a ).once.with( ob )
+      expect( c4 ).to receive( :expectable_hook_slow_a ).once.with( ob )
+      @pool.communicate( ob )
+    end
+  end
+
+  context 'message dropping' do
+    before :each do
+
+      # This sync queue is pushed from the running test and popped from the
+      # communicators inside their message handler, so that we can force the
+      # communicators to halt "mid-message" by not pushing until we're ready.
+
+      $sync_queue = Queue.new
+
+    end
+
+    class TestSlowCommunicatorDrops < Hoodoo::Communicators::Slow
+      def initialize
+        @count = 0
+      end
+
+      attr_reader :count
+
+      def communicate( obj )
+        $sync_queue.pop if @count == 0
+        @count += 1
+      end
+
+      def dropped( number )
+        @count += number
+      end
+    end
+
+    it 'reports dropped messages' do
+      c = @pool.add( TestSlowCommunicatorDrops.new )
+      o = { :foo => :bar }
+
+      # Send a message. The only way we have to sync up in this test here
+      # is to busy poll our sync queue until we see the communicator thread
+      # waiting on it. That means it pulled the message of its work queue and
+      # is now blocked in the message handler until we push to the sync queue.
+
+      expect(c).to receive(:communicate).once.with( o ).and_call_original
+      @pool.communicate( o )
+
+      loop do
+        sleep 0.01
+        break if $sync_queue.num_waiting > 0
+      end
+
+      # Now send a full queue of messages. That'll send "limit", none of which
+      # will be processed yet, then queue "additional".
+
+      limit      = Hoodoo::Communicators::Pool::MAX_SLOW_QUEUE_SIZE
+      additional = 10
+
+      1.upto( limit + additional ) do | i |
+        @pool.communicate( o )
+        sleep 1 if i == 1
+      end
+
+      # The communicator's still waiting on our sync queue. We expect it to get
+      # the "limit" queued calls; let it start processing by pushing something
+      # to the sync queue.
+
+      expect(c).to receive(:communicate).exactly( limit ).times.with( o ).and_call_original
+      $sync_queue << :go!
+
+      # Wait for the communicator to finish processing. Implicit test -
+      # variant of "wait" that takes a specific communicator.
+
+      @pool.wait( communicator: c )
+
+      # Now send one final message. This will prompt the 'dropped' call first,
+      # saying "<x> were dropped between the last communication and this one",
+      # then send in the recent message.
+
+      expect(c).to receive(:dropped).once.with( additional ).and_call_original
+      expect(c).to receive(:communicate).once.with( o ).and_call_original
+
+      @pool.communicate( o )
+      @pool.wait( communicator: c )
+
+      # The 'count' variable in the communicator records the number of
+      # messages it received, or that were dropped, adding them all up.
+      # There was the first 'sync up' message, 'limit' messages in the
+      # queue, 'additional' counted via the "dropped" call and one more
+      # message that provoked the "dropped" call.
+
+      expect( c.count ).to eq( 1 + limit + additional + 1 )
+    end
+  end
+
+  context 'waiting, termination and timeouts' do
+
+    before :each do
+
+      # This sync queue gets pushed from the communicators before they sleep,
+      # and popped by the running test so it knows the communicator threads
+      # received a message and are about to sleep for a while within their
+      # message handler.
+      #
+      $sync_queue = Queue.new
+
+    end
+
+    class TestSlowCommunicatorSleeps < Hoodoo::Communicators::Slow
+      def communicate( obj )
+        $sync_queue << :sync
+        sleep( obj )
+      end
+    end
+
+    it 'times out waiting' do
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+
+      expect( @pool.group.list.size ).to eq( 3 )
+
+      # Sleep each for 1 second, wait with a timeout of 0.02 seconds, expect
+      # the test to take less than 0.5 seconds overall. Non-timeout waiting
+      # is already tested elsewhere.
+
+      now = Time.now
+
+      @pool.communicate( 1 )
+      1.upto( 3 ) { $sync_queue.pop() } # Make sure all threads got the message
+
+      @pool.wait( per_instance_timeout: 0.02 )
+
+      expect( Time.now - now ).to be < 0.5
+    end
+
+    it 'times out termination' do
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+
+      expect( @pool.group.list.size ).to eq( 3 )
+
+      # Sleep each for 5 seconds, terminate with a 0.02 second timeout.
+      # Expect to exit the termination early, with the pool still intact
+      # (since all threads should've timed out).
+      #
+      # The test comms threads will hang around in the runtime for a while
+      # then expire naturally or, if the test suite exits first, get killed
+      # off along with the running Ruby process.
+
+      now = Time.now
+
+      @pool.communicate( 5 )
+      1.upto( 3 ) { $sync_queue.pop() } # Make sure all threads got the message
+      @pool.terminate( per_instance_timeout: 0.02 ) # Now we know they're all asleep, try a timed-out termination.
+
+      expect( Time.now - now ).to be < 1
+      expect( @pool.group.list.size ).to eq( 3 ) # All threads timed out
+    end
+
+    it 'terminates properly without timeout' do
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+      c = @pool.add( TestSlowCommunicatorSleeps.new )
+
+      expect( @pool.group.list.size ).to eq( 3 )
+
+      # Sleep each communicator for just 0.02 seconds and terminate without
+      # timeouts. Expect it to all succeed, with nothing left in the pool.
+
+      now = Time.now
+
+      @pool.communicate( 0.02 )
+      1.upto( 3 ) { $sync_queue.pop() } # Make sure all threads got the message
+      @pool.terminate()
+
+      expect( Time.now - now ).to be < 1
+      expect( @pool.group.list.size ).to eq( 0 ) # All threads exited
+    end
+  end
+end