hoodoo 1.0.2

data/lib/hoodoo/client/endpoint/endpoints/not_found.rb

@@ -0,0 +1,59 @@
+
+module Hoodoo
+  class Client     # Just used as a namespace here
+    class Endpoint # Just used as a namespace here
+
+      # An endpoint that, when called, returns 'Not Found' for
+      # the resource at hand. Used to emulate lazily resolved
+      # endpoints when in fact, the lack of endpoint presence
+      # is already known.
+      #
+      # Ignores any discovery result data if provided.
+      #
+      class NotFound < Hoodoo::Client::Endpoint::HTTPBased
+
+        protected
+
+          # See Hoodoo::Client::Endpoint#configure_with.
+          #
+          # Does nothing in this subclass.
+          #
+          def configure_with( resource, version, options )
+          end
+
+        public
+
+          # See Hoodoo::Client::Endpoint#list.
+          #
+          def list( query_hash = nil )
+            return generate_404_response_for( :list )
+          end
+
+          # See Hoodoo::Client::Endpoint#show.
+          #
+          def show( ident, query_hash = nil )
+            return generate_404_response_for( :show )
+          end
+
+          # See Hoodoo::Client::Endpoint#create.
+          #
+          def create( body_hash, query_hash = nil )
+            return generate_404_response_for( :create )
+          end
+
+          # See Hoodoo::Client::Endpoint#update.
+          #
+          def update( ident, body_hash, query_hash = nil )
+            return generate_404_response_for( :update )
+          end
+
+          # See Hoodoo::Client::Endpoint#delete.
+          #
+          def delete( ident, query_hash = nil )
+            return generate_404_response_for( :delete )
+          end
+
+      end
+    end
+  end
+end

data/lib/hoodoo/client/headers.rb

@@ -0,0 +1,269 @@
+########################################################################
+# File::    headers.rb
+# (C)::     Loyalty New Zealand 2015
+#
+# Purpose:: If you can't think of where some code should live, use a
+#           Class or Module as a namespace for what amounts to library
+#           routines. The class defined here has support data and code
+#           for Hoodoo::Client, Hoodoo::Services::Middleware and others.
+# ----------------------------------------------------------------------
+#           22-Sep-2015 (ADH): Created.
+########################################################################
+
+module Hoodoo
+  class Client
+
+    # Hoodoo::Client and related software such as
+    # Hoodoo::Services::Middleware need common access to information about
+    # special processing headers defined by Hoodoo and the Hoodoo API. This
+    # class is just a container - pretty much a namespaced library - holding
+    # that kind of information and support methods.
+    #
+    class Headers
+
+      # Used by HEADER_TO_PROPERTY; this Proc when called with some non-nil
+      # value from an HTTP header representing a UUID, evaluates to either the
+      # UUID as a String or +nil+ if the value appeared to not be a UUID.
+      #
+      UUID_PROPERTY_PROC = -> ( value ) {
+        value = Hoodoo::UUID.valid?( value ) && value
+        value || nil # => 'value' if 'value' is truthy, 'nil' if 'value' falsy
+      }
+
+      # Used by HEADER_TO_PROPERTY; this Proc when called with some UUID
+      # evaluates to the input value coerced to a String and no other changes.
+      #
+      UUID_HEADER_PROC = -> ( value ) { value }
+
+      # Used by HEADER_TO_PROPERTY; this Proc when called with some non-nil
+      # value from an HTTP header representing a Date/Time in a supported
+      # format, evaluates to either a parsed DateTime instance or +nil+ if the
+      # value appeared to not be in a supported format.
+      #
+      DATETIME_IN_PAST_ONLY_PROPERTY_PROC = -> ( value ) {
+        value = Hoodoo::Utilities.valid_iso8601_subset_datetime?( value )
+        value = nil if value && value > DateTime.now
+        value || nil # => 'value' if 'value' is truthy, 'nil' if 'value' falsy
+      }
+
+      # Used by HEADER_TO_PROPERTY; this Proc is called with a Time, Date,
+      # DateTime or DateTime-parseable String and returns a DateTime. It is
+      # used for a custom write accessor for the property associated with
+      # a header entry and works independently of the validation mechanism
+      # for inbound String-only from-header data.
+      #
+      DATETIME_WRITER_PROC = -> ( value ) { Hoodoo::Utilities.rationalise_datetime( value ) }
+
+      # Used by HEADER_TO_PROPERTY; this Proc when called with a DateTime
+      # instance evaluates to a String representing the DateTime as an
+      # ISO 8601 subset value given to nanosecond precision.
+      #
+      DATETIME_HEADER_PROC = -> ( value ) { Hoodoo::Utilities.nanosecond_iso8601( value ) }
+
+      # Used by HEADER_TO_PROPERTY; this Proc when called with some non-nil
+      # value from an HTTP header representing a Boolean as "yes" or "no",
+      # evaluates to either +true+ for "yes" or +false+ for any other value.
+      # Case insensitive.
+      #
+      BOOLEAN_PROPERTY_PROC = -> ( value ) {
+        value.to_s.downcase == 'yes' || value == true ? true : false
+      }
+
+      # Used by HEADER_TO_PROPERTY; this Proc when called with +true+ or
+      # +false+ evaluates to String "yes" for +true+ or "no" for any other
+      # value.
+      #
+      BOOLEAN_HEADER_PROC = -> ( value ) { value == true ? 'yes' : 'no' }
+
+      # Various "X-Foo"-style HTTP headers specified in the Hoodoo API
+      # Specification have special meanings and values for those need to be
+      # set up in request data and Hoodoo::Client endpoints. Processing
+      # around these is data driven by this mapping Hash.
+      #
+      # Keys are the HTTP header names in Rack (upper case, "HTTP_"-prefix)
+      # format. Values are options bundles as follows:
+      #
+      # +property+::      The property name to be associated with the header,
+      #                   as a Symbol.
+      #
+      # +property_proc+:: A Proc that's called to both validate and clean up
+      #                   the raw value from the HTTP header. It evaluates to
+      #                   +nil+ if the value is invalid, or non-+nil+ for any
+      #                   other case. Note that there is no way for an HTTP
+      #                   header to explicitly convey a corresponding value
+      #                   internally of +nil+ as a result, by design; instead
+      #                   the relevant header would simply be omitted by the
+      #                   caller (and/or change your header design!).
+      #
+      # +writer_proc+::   If a property has a possible amigbuity of input
+      #                   data types when set externally, independently of
+      #                   any validation etc. from the +property_proc+
+      #                   option, then this optional entry contains a Proc
+      #                   that is used for a custom write accessor and
+      #                   canonicalises assumed-valid but possibly not
+      #                   canonical input for writing. An example would be
+      #                   the conversion of String or Time instances to a
+      #                   DateTime so that a property always reads back with
+      #                   a DateTime instance.
+      #
+      # +header+::        For speed in lookups where it's needed, this is the
+      #                   "real" (not Rack format) HTTP header name.
+      #
+      # +header_proc+::   A Proc that's called to convert a cleaned-up value
+      #                   set in the +property+ by its +property_proc+. It
+      #                   is called with this value and returns an equivalent
+      #                   appropriate value for use with the HTTP header
+      #                   given in +header+. This _MUST_ always be a String.
+      #
+      # +secured+::       Optional, default +nil+. If +true+, marks that
+      #                   this header and its associated value can only be
+      #                   processed if there is a Session with a Caller that
+      #                   has an +authorised_http_headers+ entry for this
+      #                   header.
+      #
+      # +auto_transfer+:: Optional, default +nil+. Only relevant to
+      #                   inter-resource call scenarios. If +true+, when one
+      #                   resource calls another, the value of this property
+      #                   is automatically transferred to the downstream
+      #                   resource. Otherwise, it is not, and the downstream
+      #                   resource will operate under whatever defaults are
+      #                   present. An inter-resource call endpoint which
+      #                   inherits an auto-transfer property can always have
+      #                   this property explicitly overwritten before any
+      #                   calls are made through it.
+      #
+      # An additional key of +:property_writer+ will be set up automatically
+      # which contains the value of the +:property+ key with an "=" sign added,
+      # resulting in the name of a write accessor method for that property.
+      #
+      HEADER_TO_PROPERTY =
+      {
+        # Take care not to define any property name which clashes with an
+        # option in any other part of this entire system where these "other
+        # options" get merged in. A project search for
+        # 'HEADER_TO_PROPERTY' in comments should find those.
+
+        'HTTP_X_RESOURCE_UUID' => {
+          :property      => :resource_uuid,
+          :property_proc => UUID_PROPERTY_PROC,
+          :header        => 'X-Resource-UUID',
+          :header_proc   => UUID_HEADER_PROC,
+
+          :secured       => true,
+        },
+
+        'HTTP_X_DATED_AT' => {
+          :property      => :dated_at,
+          :property_proc => DATETIME_IN_PAST_ONLY_PROPERTY_PROC,
+          :writer_proc   => DATETIME_WRITER_PROC,
+          :header        => 'X-Dated-At',
+          :header_proc   => DATETIME_HEADER_PROC,
+
+          :auto_transfer => true,
+        },
+
+        'HTTP_X_DATED_FROM' => {
+          :property      => :dated_from,
+          :property_proc => DATETIME_IN_PAST_ONLY_PROPERTY_PROC,
+          :writer_proc   => DATETIME_WRITER_PROC,
+          :header        => 'X-Dated-From',
+          :header_proc   => DATETIME_HEADER_PROC,
+
+          :auto_transfer => true,
+        },
+
+        'HTTP_X_DEJA_VU' => {
+          :property      => :deja_vu,
+          :property_proc => BOOLEAN_PROPERTY_PROC,
+          :header        => 'X-Deja-Vu',
+          :header_proc   => BOOLEAN_HEADER_PROC,
+        },
+      }
+
+      # For speed, fill in a "property_writer" value, where "foo" becomes
+      # "foo=" - otherwise this has to be done in lots of speed-sensitive
+      # code sections.
+      #
+      HEADER_TO_PROPERTY.each_value do | value |
+        value[ :property_writer ] = "#{ value[ :property ] }="
+      end
+
+      # Define a series of read and custom write accessors according to the
+      # HTTP_HEADER_OPTIONS_MAP. For example, a property of "dated_at" results
+      # in a <tt>dated_at</tt> reader, a <tt>dated_at=</tt> writer which calls
+      # Hoodoo::Utilities.rationalise_datetime to clean up the input value
+      # and sets the result into the <tt>@dated_at</tt> instance variable which
+      # the read accessor will be expecting to use.
+      #
+      # +klass+:: The Class to which the instance methods will be added.
+      #
+      def self.define_accessors_for_header_equivalents( klass )
+        klass.class_eval do
+          HEADER_TO_PROPERTY.each do | rack_header, description |
+            attr_reader( description[ :property ] )
+
+            custom_writer = description[ :writer_proc ]
+
+            if custom_writer.nil?
+              attr_writer( description[ :property ] )
+            else
+              define_method( "#{ description[ :property ] }=" ) do | parameter |
+                instance_variable_set(
+                  "@#{ description[ :property ] }",
+                  description[ :writer_proc ].call( parameter )
+                )
+                result = instance_variable_get("@#{ description[ :property ] }")
+              end
+            end
+          end
+        end
+      end
+
+      # From a Hash-like source where keys are HTTP header names and values
+      # are the corresponding HTTP header values, extract interesting values
+      # and return a Hash of options as described below.
+      #
+      # Any <tt>X-Foo</tt> header is extracted, including core Hoodoo extension
+      # headers such as <tt>X-Interaction-ID</tt>, which is present in any
+      # response. The "X-" is stripped, the rest converted to lower case and
+      # hyphens converted to underscores. The interaction ID, therefore, would
+      # be set as an +interaction_id+ option. <tt>X-Foo</tt> would be set as a
+      # +foo+ option - and so-on.
+      #
+      # The header matcher accepts headers from the Hash-like source in upper
+      # or lower case with hyphens or underscores inside; extracted headers can
+      # therefore start with any of <tt>X_</tt>, <tt>x_</tt>, <tt>X-</tt> or
+      # <tt>x-</tt>. The Hash-like source must support the +each+ operator
+      # yielding a key and value to the block on each iteration.
+      #
+      # Header values are not translated at all, so (unless something very
+      # unsual is going on) the option values will be Strings.
+      #
+      # If the same header is encountered more than once, only the first one
+      # encountered (in enumeration order, whatever that might be) is stored.
+      #
+      # Parameters:
+      #
+      # +hashlike_source+:: Hash-like source containing HTTP headers/values.
+      #
+      def self.x_header_to_options( hashlike_source )
+        hashlike_source ||= {}
+        options           = {}
+
+        hashlike_source.each do | key, value |
+          next unless ( key[ 0 ] == 'x' || key[ 0 ] == 'X' ) &&
+                      ( key[ 1 ] == '-' || key[ 1 ] == '_' )
+
+          entry = key.to_s.downcase.gsub( '-', '_' )[ 2..-1 ]
+
+          unless entry == '' || options.has_key?( entry )
+            options[ entry ] = value
+          end
+        end
+
+        return options
+      end
+
+    end
+  end
+end

data/lib/hoodoo/communicators.rb

@@ -0,0 +1,23 @@
+########################################################################
+# File::    communicators.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: Include the code providing a pool of fast or slow workers
+#           that communicate with the outside world.
+# ----------------------------------------------------------------------
+#           26-Jan-2015 (ADH): Split from top-level inclusion file.
+########################################################################
+
+module Hoodoo
+
+  # The Communicators module is used as a namespace for
+  # Hoodoo::Communicators::Pool and its related utility classes,
+  # Hoodoo::Communicators::Fast and Hoodoo::Communicators::Fast.
+  #
+  module Communicators
+  end
+end
+
+require 'hoodoo/communicators/pool'
+require 'hoodoo/communicators/fast'
+require 'hoodoo/communicators/slow'

data/lib/hoodoo/communicators/fast.rb

@@ -0,0 +1,44 @@
+########################################################################
+# File::    fast.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: A fast communication-orientated object intended to be called
+#           synchronously via Hoodoo::Communicators::Pool.
+# ----------------------------------------------------------------------
+#           15-Dec-2014 (ADH): Created.
+########################################################################
+
+module Hoodoo
+  module Communicators
+
+    # See Hoodoo::Communicators::Pool for details.
+    #
+    # A "fast communicator". Subclass this to create a class where instances
+    # are invoked via Hoodoo::Communicators::Fast#communicate with some
+    # parameter and, in response, they talk to some other piece of software to
+    # communicate information related to that parameter. The communication is
+    # expected to be fast and will be called from Ruby's main execution thread.
+    #
+    # If the communicator takes a long time to complete its operation, other
+    # processing will be delayed. If you expect this to happen, subclass
+    # Hoodoo::Communicators::Slow instead.
+    #
+    # Example: A communicator might be part of a logging scheme which writes
+    # to +STDOUT+. The parameter it expects would be a log message string.
+    #
+    class Fast
+
+      # Communicate quickly with the piece of external software for which your
+      # subclass is designed. Subclasses _must_ implement this method. There is
+      # no need to call +super+ in your implementation.
+      #
+      # +object+:: Parameter sent by the communication pool, in response to
+      #            someone calling Hoodoo::Communicators::Pool#communicate
+      #            with that value.
+      #
+      def communicate( object )
+        raise( 'Subclasses must implement #communicate' )
+      end
+    end
+  end
+end

data/lib/hoodoo/communicators/pool.rb

@@ -0,0 +1,601 @@
+########################################################################
+# File::    pool.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: A pool of communication-orientated objects which are either
+#           fast and operate synchronously, or are slow and are called
+#           asynchronously via a Ruby Thread.
+# ----------------------------------------------------------------------
+#           15-Dec-2014 (ADH): Created.
+########################################################################
+
+module Hoodoo
+  module Communicators
+
+    # Maintains a pool of object instances which are expected to be
+    # communicating with "the outside world" in some way. A message
+    # sent to the pool is replicated to all the communicators in
+    # that pool. Some communicators are fast, which means they are
+    # called synchronously and expected to return very quickly. Some
+    # communicators are slow, which means they are called
+    # asynchronously through a work queue.
+    #
+    # See #add for more information.
+    #
+    class Pool
+
+      # Hoodoo::Communicators::Slow subclass communicators are called in
+      # their own Threads via a processing Queue. There is the potential for
+      # a flood of communications to cause the queue to back up considerably,
+      # so a maximum number of messages is defined. If the queue size is
+      # _equal to or greater_ than this amount when a message arrives, it
+      # will be dropped and a 'dropped message' count incremented.
+      #
+      MAX_SLOW_QUEUE_SIZE = 50
+
+      # When asking slow communicator threads to exit, a timeout must be used
+      # in case the thread doesn't seem to be responsive. This is the timeout
+      # value in seconds - it can take a floating point or integer value.
+      #
+      THREAD_EXIT_TIMEOUT = 5
+
+      # Analogous to THREAD_WAIT_TIMEOUT but used when waiting for a
+      # processing Thread to drain its Queue, without asking it to exit.
+      #
+      THREAD_WAIT_TIMEOUT = 5
+
+      # Retrieve the ThreadGroup instance managing the collection of slow
+      # communicator threads. This is mostly used for testing purposes and
+      # has little general purpose utility.
+      #
+      attr_accessor :group
+
+      # Create a new pool of communicators - instances of subclasses of
+      # Hoodoo::Communicators::Fast or Hoodoo::Communicators::Slow,
+      # are added with #add and called with #communicate.
+      #
+      def initialize
+        @pool  = {}
+        @group = ::ThreadGroup.new
+      end
+
+      # Add a communicator instance to the pool. Future calls to #communicate
+      # will call the same-named method in that instance.
+      #
+      # Subclasses of Hoodoo::Communicators::Slow are called within a
+      # processing Thread. Subclasses of Hoodoo::Communicators::Fast are
+      # called inline. The instances are called in the order of addition, but
+      # since each slow communicator runs in its own Thread, the execution
+      # order is indeterminate for such instances.
+      #
+      # If a slow communicator's inbound message queue length matches or
+      # exceeds MAX_SLOW_QUEUE_SIZE, messages for that specific communicator
+      # will start being dropped until the communicator clears the backlog and
+      # at last one space opens on the queue. Slow communicators can detect
+      # when this has happened by implementing
+      # Hoodoo::Communicators::Slow#dropped in the subclass.
+      #
+      # If you pass the same instance more than once, the subsequent calls are
+      # ignored. You can add many instances of the same class if that's useful
+      # for any reason.
+      #
+      # Returns the passed-in communicator instance parameter, for convenience.
+      #
+      # +communicator+:: Instance is to be added to the pool. Must be
+      #                  either a Hoodoo::Communicators::Fast or
+      #                  Hoodoo::Communicators::Slow subclass instance.
+      #
+      def add( communicator )
+        unless ( communicator.class < Hoodoo::Communicators::Fast ||
+                 communicator.class < Hoodoo::Communicators::Slow )
+          raise "Hoodoo::Communicators::Pool\#add must be called with an instance of a subclass of Hoodoo::Communicators::Fast or Hoodoo::Communicators::Slow only"
+        end
+
+        return if @pool.has_key?( communicator )
+
+        if communicator.is_a?( Hoodoo::Communicators::Fast )
+          add_fast_communicator( communicator )
+        else
+          add_slow_communicator( communicator )
+        end
+
+        return communicator
+      end
+
+      # Remove a communicator previously added by #add. See that for details.
+      #
+      # It is harmless to try and remove communicator instances more than once
+      # or to try to remove something that was never added in the first place;
+      # the call simply has no side effects.
+      #
+      # If removing a slow communicator, its thread will be terminated with
+      # default timeout value of THREAD_EXIT_TIMEOUT seconds. For this
+      # reason, removing a slow communicator may take a long time.
+      #
+      # Returns the passed-in communicator instance parameter, for convenience.
+      #
+      # +communicator+:: Instance is to be removed from the pool. Must be
+      #                  either a Hoodoo::Communicators::Fast or
+      #                  Hoodoo::Communicators::Slow subclass instance.
+      #
+      def remove( communicator )
+        unless ( communicator.class < Hoodoo::Communicators::Fast ||
+                 communicator.class < Hoodoo::Communicators::Slow )
+          raise "Hoodoo::Communicators::Pool\#remove must be called with an instance of a subclass of Hoodoo::Communicators::Fast or Hoodoo::Communicators::Slow only"
+        end
+
+        return unless @pool.has_key?( communicator )
+
+        if communicator.is_a?( Hoodoo::Communicators::Fast )
+          remove_fast_communicator( communicator )
+        else
+          remove_slow_communicator( communicator )
+        end
+
+        return communicator
+      end
+
+      # Call the #communicate method on each communicator instance added via
+      # #add. Each instance is called in the same order as corresponding
+      # calls are made to the pool. _Across_ instances, fast communicators are
+      # called in the order they were added to the pool, but since each slow
+      # communicator runs in its own Thread, execution order is indeterminate.
+      #
+      # +object+:: Parameter passed to the communicator subclass instance
+      #            #communicate methods.
+      #
+      def communicate( object )
+        @pool.each do | communicator, item |
+
+          if item.has_key?( :fast )
+            begin
+              communicator.communicate( object )
+            rescue => exception
+              handle_exception( exception, communicator )
+            end
+
+          else
+            data       = item[ :slow       ]
+            thread     = data[ :thread     ]
+            work_queue = data[ :work_queue ]
+
+            # This is inaccurate if one or more "dropped messages" reports are
+            # on the queue, but since some communicators might report them in
+            # the same way as other messages, it's not necessarily incorrect
+            # either.
+            #
+            if work_queue.size < MAX_SLOW_QUEUE_SIZE
+              dropped = thread[ :dropped_messages ]
+
+              if dropped > 0
+                thread[ :dropped_messages ] = 0
+
+                # Opposite of comment above on MAX_SLOW_QUEUE_SIZE check...
+                # Yes, this takes up a queue entry and the payload addition
+                # afterwards might take it one above max size, but that's OK
+                # since this is just a "dropped messages" report and though
+                # some communicators might deal with them slowly, others may
+                # just ignore them.
+                #
+                work_queue << QueueEntry.new( dropped: dropped )
+              end
+
+              work_queue << QueueEntry.new( payload: object )
+
+            else
+              thread[ :dropped_messages ] += 1
+
+            end
+          end
+
+        end
+      end
+
+      # This method is only useful if there are any
+      # Hoodoo::Communicators::Slow subclass instances in the communication
+      # pool. Each instance is called via a worker Thread; this method waits
+      # for each communicator to drain its queue before returning. This is
+      # useful if you have a requirement to wait for all communications to
+      # finish on all threads, presumably for wider synchronisation reasons.
+      #
+      # Since fast communicators are called synchronously there is never any
+      # need to wait for them, so this call ignores such pool entries.
+      #
+      # The following *named* parameters are supported:
+      #
+      # +per_instance_timeout+:: Timeout for _each_ slow communicator Thread
+      #                          in seconds. Optional. Default is the value
+      #                          in THREAD_WAIT_TIMEOUT.
+      #
+      # +communicator+::         If you want to wait for specific instance only
+      #                          (see #add), pass it here. If the instance is a
+      #                          fast communicator, or any object not added to
+      #                          the pool, then there is no error raised. The
+      #                          method simply returns immediately.
+      #
+      def wait( per_instance_timeout: THREAD_WAIT_TIMEOUT,
+                communicator:         nil )
+
+        if communicator.nil?
+          @pool.each do | communicator, item |
+            next unless item.has_key?( :slow )
+            data = item[ :slow ]
+
+            wait_for(
+              work_queue: data[ :work_queue ],
+              sync_queue: data[ :sync_queue ],
+              timeout:    per_instance_timeout
+            )
+          end
+
+        else
+          return unless @pool.has_key?( communicator )
+          item = @pool[ communicator ]
+
+          return unless item.has_key?( :slow )
+          data = item[ :slow ]
+
+          wait_for(
+            work_queue: data[ :work_queue ],
+            sync_queue: data[ :sync_queue ],
+            timeout:    per_instance_timeout
+          )
+
+        end
+      end
+
+      # The communication pool is "emptied" by this call, going back to a
+      # clean state as if just initialised. New workers can be added via #add
+      # and then called via #communicate if you so wish.
+      #
+      # Hoodoo::Communciators::Fast subclass instances are removed
+      # immediately without complications.
+      #
+      # Hoodoo::Communicators::Slow subclass instances in the communication
+      # pool are called via a worker Thread; this method shuts down all such
+      # worker Threads, clearing their work queues and asking each one to exit
+      # (politely). There is no mechanism (other than overall Ruby process
+      # exit) available to shut down the Threads by force, so some Threads may
+      # not respond and time out.
+      #
+      # When this method exits, all workers will have either exited or timed
+      # out and possibly still be running, but are considered too slow or dead.
+      # No further communications are made to them.
+      #
+      # The following *named* parameters are supported:
+      #
+      # +per_instance_timeout+:: Timeout for _each_ slow communicator Thread
+      #                          in seconds. Optional. Default is the value
+      #                          in THREAD_EXIT_TIMEOUT. For example,
+      #                          with three slow communicators in the pool
+      #                          and all three reached a 5 second timeout,
+      #                          the termination method would not return for
+      #                          15 seconds (3 * 5 seconds full timeout).
+      #
+      def terminate( per_instance_timeout: THREAD_EXIT_TIMEOUT )
+        loop do
+          klass, item = @pool.shift() # Hash#shift -> remove a key/value pair.
+          break if klass.nil?
+
+          next unless item.has_key?( :slow )
+          data = item[ :slow ]
+
+          request_termination_for(
+            thread:     data[ :thread     ],
+            work_queue: data[ :work_queue ],
+            timeout:    per_instance_timeout
+          )
+        end
+      end
+
+    private
+
+      # Add a fast communicator to the pool. Requires no thread or queue.
+      #
+      # Trusted internal interface - pass the correct subclass and don't pass
+      # it more than once unless #terminate has cleared the pool beforehand.
+      #
+      # +communicator+:: The Hoodoo::Communicators::Fast subclass instance
+      #                  to add to the pool.
+      #
+      def add_fast_communicator( communicator )
+        @pool[ communicator ] = { :fast => true }
+      end
+
+      # Remove a fast communicator from the pool. See #add_fast_communicator.
+      #
+      # +communicator+:: The Hoodoo::Communicators::Fast subclass instance
+      #                  to remove from the pool.
+      #
+      def remove_fast_communicator( communicator )
+        @pool.delete( communicator )
+      end
+
+      # Add a slow communicator to the pool. Requires a thread and queue.
+      #
+      # Trusted internal interface - pass the correct subclass and don't pass
+      # it more than once unless #terminate has cleared the pool beforehand.
+      #
+      # +communicator+:: The Hoodoo::Communicators::Slow subclass instance
+      #                  to add to the pool.
+      #
+      def add_slow_communicator( communicator )
+
+        work_queue = ::Queue.new
+        sync_queue = QueueWithTimeout.new
+
+        # Start (and keep a reference to) a thread that just loops around
+        # processing queue messages until asked to exit.
+
+        thread = ::Thread.new do
+
+          # Outer infinite loop restarts queue processing if exceptions occur.
+          #
+          loop do
+
+            # Exception handler block.
+            #
+            begin
+
+              # Inner infinite loop processes queue objects until asked to exit
+              # via a +nil+ queue entry.
+              #
+              loop do
+                entry = work_queue.shift() # ".shift" => FIFO, ".pop" would be LIFO
+
+                if entry.terminate?
+                  ::Thread.exit
+                elsif entry.sync?
+                  sync_queue << :sync
+                elsif entry.dropped?
+                  communicator.dropped( entry.dropped )
+                else
+                  communicator.communicate( entry.payload )
+                end
+              end
+
+            rescue => exception
+              handle_exception( exception, communicator )
+
+            end
+          end
+        end
+
+        thread[ :dropped_messages ] = 0
+
+        @group.add( thread )
+        @pool[ communicator ] = {
+          :slow => {
+            :thread     => thread,
+            :work_queue => work_queue,
+            :sync_queue => sync_queue,
+          }
+        }
+      end
+
+      # Remove a slow communicator from the pool. See #add_slow_communicator.
+      #
+      # May take a while to return, as it must request thread shutdown via
+      # #request_termination_for (and uses the default timeout for that).
+      #
+      # +communicator+:: The Hoodoo::Communicators::Slow subclass instance
+      #                  to remove from the pool.
+      #
+      def remove_slow_communicator( communicator )
+        item = @pool[ communicator ]
+        data = item[ :slow ]
+
+        request_termination_for(
+          thread:     data[ :thread     ],
+          work_queue: data[ :work_queue ]
+        )
+
+        @pool.delete( communicator )
+      end
+
+      # Ask a slow communicator Thread to exit. Existing work on any Queues
+      # is cleared first, so only the current in-process message for a given
+      # communicator has to finish prior to exit.
+      #
+      # *Named* parameters are:
+      #
+      # +:thread+::     Mandatory. Worker Thread for the communicator.
+      # +:work_queue+:: Mandatory. Queue used to send work to the Thread.
+      # +timeout+::     Optional timeout in seconds - default is
+      #                 THREAD_EXIT_TIMEOUT.
+      #
+      # The method returns if the timeout threshold is exceeded, without
+      # raising any exceptions.
+      #
+      def request_termination_for( thread:, work_queue:, timeout: THREAD_EXIT_TIMEOUT )
+        work_queue.clear()
+        work_queue << QueueEntry.new( terminate: true )
+
+        thread.join( timeout )
+      end
+
+      # Wait for a slow communicator Thread to empty its work Queue. *Named*
+      # parameters are:
+      #
+      # +:work_queue+:: Mandatory. Queue used to send work to the Thread.
+      # +:sync_queue+:: Mandatory. Queue used by that Thread to send back a
+      #                 sync notification to the pool.
+      # +timeout+::     Optional timeout in seconds - default is
+      #                 THREAD_WAIT_TIMEOUT.
+      #
+      # The method returns if the timeout threshold is exceeded, without
+      # raising any exceptions.
+      #
+      def wait_for( work_queue:, sync_queue:, timeout: THREAD_WAIT_TIMEOUT )
+
+        # Push a 'sync' entry onto the work Queue. Once the worker Thread gets
+        # through other Queue items and reaches this entry, it'll respond
+        # by pushing an item onto its sync Queue.
+
+        work_queue << QueueEntry.new( sync: true )
+
+        # Wait on the sync Queue for the worker Thread to send the requested
+        # message indicating that we're in sync.
+
+        begin
+          sync_queue.shift( timeout )
+
+        rescue ThreadError
+          # Do nothing
+
+        end
+      end
+
+      # Intended for cases where a communicator raised an exception - print
+      # details to $stderr. This is all we can do; the logging engine runs
+      # through the communications pool so attempting to log an exception
+      # might cause an exception that we then attempt to log - and so-on.
+      #
+      # +exception+::    Exception (or Exception subclass) instance to print.
+      # +communicator+:: Communicator instance that raised the exception.
+      #
+      def handle_exception( exception, communicator )
+        begin
+          report = "Slow communicator class #{ communicator.class.name } raised exception '#{ exception }': #{ exception.backtrace }"
+          $stderr.puts( report )
+
+        rescue
+          # If the above fails then everything else is probably about to
+          # collapse, but optimistically try to ignore the error and keep
+          # the wider processing code alive.
+
+        end
+      end
+
+      # Internal implementation detail of Hoodoo::Communicators::Pool.
+      #
+      # Since pool clients can say "wait until (one or all) workers have
+      # processed their Queue contents", we need to have some way of seeing
+      # when all work is done. The clean way to do it is to push 'sync now'
+      # messages onto the communicator Threads work Queues, so that as they
+      # work through the Queue they'll eventually reach that message. They then
+      # push a message onto a sync Queue for that worker. Meanwhile the waiting
+      # pool does (e.g.) a +pop+ on the sync Queue, which means it blocks until
+      # the workers say they've finished. No busy waiting, Ruby gets to make
+      # its best guess at scheduling, etc.; all good.
+      #
+      # The catch? You can't use +Timeout::timeout...do...+ around a Queue
+      # +pop+. It just doesn't work. It's a strange omission and requires code
+      # gymnastics to work around.
+      #
+      # Enter QueueWithTimeout, from:
+      #
+      #   http://spin.atomicobject.com/2014/07/07/ruby-queue-pop-timeout/
+      #
+      class QueueWithTimeout
+
+        # Create a new instance.
+        #
+        def initialize
+          @mutex    = ::Mutex.new
+          @queue    = []
+          @recieved = ::ConditionVariable.new
+        end
+
+        # Push a new entry to the end of the queue.
+        #
+        # +entry+:: Entry to put onto the end of the queue.
+        #
+        def <<( entry )
+          @mutex.synchronize do
+            @queue << entry
+            @recieved.signal
+          end
+        end
+
+        # Take an entry from the front of the queue (FIFO) with optional
+        # timeout if the queue is empty.
+        #
+        # +timeout+:: Timeout (in seconds, Integer or Float) to wait for an
+        #             item to appear on the queue, if the queue is empty. If
+        #             +nil+, there is no timeout (waits indefinitely).
+        #             Optional; default is +nil+.
+        #
+        # If given a non-+nil+ timeout value and the timeout expires, raises
+        # a ThreadError exception (just as non-blocking Ruby Queue#pop would).
+        #
+        def shift( timeout = nil )
+          @mutex.synchronize do
+            if @queue.empty?
+              @recieved.wait( @mutex, timeout ) if timeout != 0
+              raise( ThreadError, 'queue empty' ) if @queue.empty?
+            end
+
+            @queue.shift
+          end
+        end
+      end
+
+      # Internal implementation detail of Hoodoo::Communicators::Pool which
+      # is placed on a Ruby Queue and used as part of thread processing for
+      # slow communicators.
+      #
+      class QueueEntry
+
+        # If +true+, the processing Thread should exit. See also #terminate?.
+        #
+        attr_accessor :terminate
+
+        # If +true+, the processing Thread should push one item with any
+        # payload onto its sync Queue. See also #sync?
+        #
+        attr_accessor :sync
+
+        # If not +nil+ or zero, the number of dropped messages that should be
+        # send to the slow communicator subclass's #dropped method. See also
+        # #dropped?
+        #
+        attr_accessor :dropped
+
+        # If the entry represents neither a termination request nor a dropped
+        # message count (see #terminate? and #dropped?), the payload to send to
+        # the slow communicator subclass's #communicate method.
+        #
+        attr_accessor :payload
+
+        # Create a new instance, ready to be added to the Queue.
+        #
+        # *ONLY* *USE* *ONE* of the named parameters:
+        #
+        # +payload+::   A parameter to send to #communicate in the communicator.
+        # +dropped+::   The integer to send to #dropped in the communicator.
+        # +terminate+:: Set to +true+ to exit the processing thread when the
+        #               entry is read from the Queue.
+        # +sync+::      Set to +true+ to push a message onto the sync Queue.
+        #
+        def initialize( payload: nil, dropped: nil, terminate: false, sync: false )
+          @payload   = payload
+          @dropped   = dropped
+          @terminate = terminate
+          @sync      = sync
+        end
+
+        # Returns +true+ if encountering this queue entry should terminate the
+        # processing thread, else +false+ (see #dropped? then #payload).
+        #
+        def terminate?
+          @terminate == true
+        end
+
+        # Returns +true+ if this queue entry represents a request to push a
+        # message onto the processing Thread's sync Queue.
+        #
+        def sync?
+          @sync == true
+        end
+
+        # Returns +true+ if this queue entry represents a dropped message count
+        # (see #dropped), else +false (see #terminate? then #payload).
+        #
+        def dropped?
+          @dropped != nil && @dropped > 0
+        end
+      end
+    end
+  end
+end