mercury_amqp 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ce3a4a09a924946d240cfbabeea944dffba4ec7
-  data.tar.gz: 772194f0c84be6d336fc16b2821f0e728e8b1f4b
+  metadata.gz: 459b852ab72eefefec630bac89fde701f7fc2018
+  data.tar.gz: a0ae43b454314876e4dc1be0e6ccf9480d8a9ed6
 SHA512:
-  metadata.gz: ab8452d14bdb64949877c748bd8eafbe37c5bd5ff652772950025f63b70f3db9d0754397606a9d223292d33142c2561d88f410cf4b9c9dd7acc416e0a7b1290b
-  data.tar.gz: b63c33292ee6299d6b5832d2c1157fc6a229b281266e6fc7489dec4fed6f8126b4d023fce49abe2f40132816022c3411230d1ec078547b7fe910a8b8837a1db0
+  metadata.gz: 296ef00ec39d0419c4f55b185528a4d0ed905d236014ad0f8a432a310a01033cf4105c6c2c19a82d02cec850c4de3e8af2a96eba8f0e6d8f4761002a1ef971df
+  data.tar.gz: 7fcbe0aa189f7d5924312210ddaf7eb1fb6111ff0f4f74e38fddc1a7410f26a8820fd871cac8daaf0c93b363bee153cb61b1b85ca4ee760bc8c6240048758ca3

data/.gitignore

@@ -1,5 +1,5 @@
+*.png
 .idea
-.bundle
 Gemfile.lock
-*~
 *.gem
+*~

data/README.md

@@ -1,26 +1,140 @@
 mercury
 =======
 
-Mercury is a messaging layer intended to hide complexity for typical
-messaging scenarios. It is backed by the AMQP gem and consequently
-runs in an EventMachine reactor and has an asynchronous API. Mercury
-consists of _sources_, _work queues_, and _listeners_. A message is
-published to a source, to which one or more work queues and/or
-listeners are attached. These map roughly to AMQP constructs:
+Mercury is a messaging layer that hides complexity for typical messaging scenarios.
 
-- **source**: topic exchange
-- **work queue**: durable named queue
-- **listener**: temporary anonymous queue
-- **tag**: routing key
+- backed by [AMQP][amqp]
+- runs in an [EventMachine][em] reactor
+- asynchronous API
+- serializes messages as JSON
+- optional monadic interface makes testing easier
+- simulator allows tests to run without a RabbitMQ server
 
-At the moment, mercury is backed by AMQP and serializes messages as
-JSON. In the future, additional transports and message formats may be
-supported.
+### Constructs
 
-Mercury writes string messages directly without encoding; this allows
-a client to pre-encode a message using an arbitrary encoding. The
-receiving client receives the encoded bytes as the message content
-(assuming the encoded message fails to parse as JSON).
+![Constructs diagram](constructs.png)
+
+In the above example, two publishers independently publish messages to the same source.
+One of them chooses to attach the tag `foo.success` to the messages it publishes.
+When the source receives a message, it immediately broadcasts the message to all attached
+queues.
+
+Two independent listeners receive messages from the source. A listener has a private
+queue that is automatically removed when the listener disconnects from the server (i.e.,
+any queued messages the listener would have received are lost).
+
+A worker pool "A" has two workers handling messages. Messages in the queue are dealt
+to the workers as the workers acknowledge handling prior messages. If a worker disconnects
+without acknowledging completion, the message(s) it was working on are automatically replaced at the
+head of the queue. Worker queues persist even if all workers disconnect. Each worker pool
+has its own queue. Consequently, workers within a pool compete for messages, whereas workers
+across pools do not compete for messages.
+
+Worker pool "B" has just one worker and has specified a tag filter.
+
+Listeners and worker pools can specify _tag filters_ which limit the messages they receive
+to only those with tags matching a particular pattern. A _tag_ is zero or more words delimited
+by periods. A _word_ is a string of alphanumeric characters. A _filter_ is a string of zero or more
+words or wildcards. A _wildcard_ is either `*` or `#`.
+
+- `*` matches exactly one word.
+- `#` matches zero or more words.
+
+_Note:_ A filter with zero words (`''`) matches no messages.
+
+_Note:_ Wildcards match only entire words. The filter `f*.success` does not match the tag `foo.success`.
+
+A typical scenario is to have a single worker pool processing messages.
+
+![Typical scenario diagram](typical.png)
+
+The publishers are often instances of the same service: either some
+other mercury worker pool or a web service.
+
+
+### Operations
+
+**open**(_parallelism_)
+
+Creates a mercury instance. _paralellism_ is a natural number that indicates
+the maximum total number of outstanding (unacknowledged) messages allowed
+for this instance's workers. See the code for additional parameters.
+
+**publish**(_source_name_, _msg_, _tag_)
+
+Publishes a message to a source, with an optional tag.
+
+**start_worker**(_worker_pool_name_, _source_name_, _tag_filter_, _msg_handler_)
+
+Starts a worker in the specified pool subscribing to the specified
+source with the specified filter (optional). Incoming messages are
+passed to the message handler procedure.
+
+**start_listener**(_source_name_, _tag_filter_, _msg_handler_)
+
+Starts a listener.
+
+**ack**(_msg_)
+
+Indicates message handling succeeded. The message is removed from the queue.
+
+**nack**(_msg_)
+
+Indicates message handling failed, with the assumption that it might succeed at
+a later time. The message is returned to the queue.
+
+**reject**(_msg_)
+
+Indicates message handling failed, with the assumption that it can never succeed.
+The message is removed from the queue.
+
+_Note:_ All operations create the referenced constructs if they do not already exist.
+
+### Serialization
+
+If a message is a hash, mercury serializes it to JSON upon sending and deserializes
+it upon receiving.
+
+If a message is a string, mercury writes it directly without any
+serialization; this allows a client to pre-encode a message using an
+arbitrary encoding. The receiving client receives the exact same
+string as the message content (assuming the serialized message fails to
+parse as JSON).
+
+### Logatron integration
+
+Mercury depends on the
+[`logatron`][logatron] gem and propagates
+logatron's request ID (`Logatron.msg_id`) through the AMQP header
+`X-Ascent-Log-Id`. This enables a log aggregation service to find the
+logs associated with a particular incoming request, even though the
+log entries may be scattered across various services.
+
+### Thread safety
+
+Mercury is not threadsafe. All calls to a particular instance must be made from the
+thread in which the instance was created.
+
+### Error handling
+
+If there is a communication error (e.g., the connection to the server breaks)
+mercury raises an error. This behavior can be overridden.
+
+### Long running operations
+
+The AMQP gem uses EventMachine timers to send heartbeats to the
+server. Message handling blocks the event loop, so each message must
+be handled within a few seconds, otherwise the server will miss a
+heartbeat and disconnect the client. Long-running message handlers
+should move their work to a thread pool to avoid blocking the event
+loop.
+[`EventMachine.defer`][em_defer]
+and
+[`fiber_defer`][fiber_defer]
+are two facilities for accomplishing this.
+
+
+### Example Usage
 
 
 ```ruby
@@ -30,6 +144,8 @@ def run
   EventMachine.run do
     Mercury.open do |m|
       m.start_worker('cooks', 'orders', method(:handle_message)) do
+        # When this continuation block is called, the worker is guaranteed
+        # to have been started, so we can start publishing orders.
         m.publish('orders', {'table' => 5, 'items' => ['salad', 'steak', 'cake']})
       end
     end
@@ -71,7 +187,7 @@ end
 
 require 'mercury/monadic'
 
-seq do
+seql do
   let(:m)  { Mercury::Monadic.open    }
   and_then { m.start_listener(source) }
   let(:r1) { m.source_exists?(source) }
@@ -82,4 +198,21 @@ seq do
   and_then { m.close                  }
   and_lift { done                     }
 end
-```
+```
+
+Design Decisions
+----------------
+
+#### Continuation blocks are required
+
+It would be possible to make continuation blocks optional. The problem is that this allows the user
+to make the mistake of treating the API as synchronous and only discover their error when tests fail,
+probably intermittently. An empty block can be passed, but at least it indicates that the continuation
+is being intentionally ignored.
+
+
+[amqp]: https://github.com/ruby-amqp/amqp
+[em]: https://github.com/eventmachine/eventmachine
+[logatron]: https://github.com/indigobio/logatron
+[em_defer]: http://www.rubydoc.info/github/eventmachine/eventmachine/EventMachine.defer
+[fiber_defer]: https://github.com/indigobio/abstractivator/blob/master/lib/abstractivator/fiber_defer.rb

data/constructs.dot

@@ -0,0 +1,24 @@
+digraph G {
+        rankdir=LR
+        p1 [label="publisher", shape=rect]
+        p2 [label="publisher", shape=rect]
+        s [label="source", shape=circle]
+        l1q [label="{|||||}", shape=record, fixedsize=true, height=0.3, width=1.5]
+        l2q [label="{|||||}", shape=record, fixedsize=true, height=0.3, width=1.5]
+        w1q [label="{|||||}", shape=record, fixedsize=true, height=0.3, width=1.5]
+        w2q [label="{|||||}", shape=record, fixedsize=true, height=0.3, width=1.5]
+        l1 [label="listener", shape=rect]
+        l2 [label="listener", shape=rect]
+        w1a1 [label="worker A", shape=rect]
+        w1a2 [label="worker A", shape=rect]
+        w2b [label="worker B", shape=rect]
+
+        p1 -> s [tailport=e]
+        p2 -> s [tailport=e, label="tag: foo.success", fontsize=8, fontname="mono"]
+        s -> {l1q, l2q, w1q} [headport=w]
+        s -> w2q [headport=w, label="tag_filter: *.success", fontsize=8, fontname="mono"]
+        l1q -> l1 [tailport=e, headport=w]
+        l2q -> l2 [tailport=e, headport=w]
+        w1q -> {w1a1, w1a2} [tailport=e, headport=w]
+        w2q -> w2b [tailport=e, headport=w]
+}

data/lib/mercury/cps.rb

@@ -37,7 +37,7 @@ class Mercury
       Cps.new do |*args, &k|
         self.run(*args) do |*args2|
           next_cps = pm.call(*args2)
-          next_cps.is_a?(Cps) or raise "'and_then' block did not return a Cps. Did you want 'and_lift'? at #{pm.source_location}"
+          next_cps.is_a?(Cps) or raise "'and_then' block did not return a Cps object. Did you want 'and_lift'? at #{pm.source_location}"
           next_cps.run(&k)
         end
       end
@@ -52,7 +52,14 @@ class Mercury
 
     # Returns a Cps for a non-CPS proc.
     def self.lift(&p)
-      new { |*args, &k| k.call(p.call(*args)) }
+      new do |*args, &k|
+        value = p.call(*args)
+        if value.is_a?(Cps)
+          # This is technically valid, but 99% of the time it indicates a programming error.
+          raise "'lift' block returned a Cps object. Did you want 'and_then'? at #{p.source_location}"
+        end
+        k.call(value)
+      end
     end
 
     # The identity function as a Cps.

data/lib/mercury/fake.rb

@@ -15,9 +15,21 @@ require 'mercury/fake/subscriber'
 # broken sockets, etc.
 class Mercury
   class Fake
-    def initialize(domain=:default, parallelism: 1)
+    def self.install(rspec_context, domain=:default)
+      rspec_context.instance_exec do
+        allow(Mercury).to receive(:open) do |**kws, &k|
+          EM.next_tick { k.call(Mercury::Fake.new(domain, **kws)) } # EM.next_tick is required to emulate the real Mercury.open
+        end
+      end
+    end
+
+    def initialize(domain=:default, **kws)
       @domain = Fake.domains[domain]
-      @parallelism = parallelism
+      @parallelism = kws.fetch(:parallelism, 1)
+      ignored_keys = kws.keys - [:parallelism]
+      if ignored_keys.any?
+        $stderr.puts "Warning: Mercury::Fake::new is ignoring keyword arguments: #{ignored_keys.join(', ')}"
+      end
     end
 
     def self.domains
@@ -30,21 +42,22 @@ class Mercury
     end
 
     def publish(source_name, msg, tag: '', headers: {}, &k)
-      assert_not_closed
+      guard_public(k)
       queues.values.select{|q| q.binds?(source_name, tag)}.each{|q| q.enqueue(roundtrip(msg), tag, headers)}
       ret(k)
     end
 
-    def start_listener(source_name, handler, tag_filter: '#', &k)
+    def start_listener(source_name, handler, tag_filter: nil, &k)
       start_worker_or_listener(source_name, handler, tag_filter, &k)
     end
 
-    def start_worker(worker_group, source_name, handler, tag_filter: '#', &k)
+    def start_worker(worker_group, source_name, handler, tag_filter: nil, &k)
       start_worker_or_listener(source_name, handler, tag_filter, worker_group, &k)
     end
 
     def start_worker_or_listener(source_name, handler, tag_filter, worker_group=nil, &k)
-      assert_not_closed
+      guard_public(k)
+      tag_filter ||= '#'
       q = ensure_queue(source_name, tag_filter, !!worker_group, worker_group)
       ret(k) # it's important we show the "start" operation finishing before delivery starts (in add_subscriber)
       q.add_subscriber(Subscriber.new(handler, @parallelism))
@@ -52,23 +65,25 @@ class Mercury
     private :start_worker_or_listener
 
     def delete_source(source_name, &k)
-      assert_not_closed
+      guard_public(k)
       queues.delete_if{|_k, v| v.source == source_name}
       ret(k)
     end
 
     def delete_work_queue(worker_group, &k)
-      assert_not_closed
+      guard_public(k)
       queues.delete_if{|_k, v| v.worker == worker_group}
       ret(k)
     end
 
     def source_exists?(source, &k)
+      guard_public(k)
       built_in_sources = %w(direct topic fanout headers match rabbitmq.log rabbitmq.trace).map{|x| "amq.#{x}"}
       ret(k, (queues.values.map(&:source) + built_in_sources).include?(source))
     end
 
     def queue_exists?(worker, &k)
+      guard_public(k)
       ret(k, queues.values.map(&:worker).include?(worker))
     end
 
@@ -98,8 +113,8 @@ class Mercury
       [source, tag_filter, worker].join('^')
     end
 
-    def assert_not_closed
-      raise 'connection is closed' if @closed
+    def guard_public(k, initializing: false)
+      Mercury.guard_public(@closed, k, initializing: initializing)
     end
   end
 end

data/lib/mercury/mercury.rb

@@ -13,8 +13,13 @@ class Mercury
   end
 
   def close(&k)
-    @amqp.close do
-      k.call
+    if @amqp
+      @amqp.close do
+        @amqp = nil
+        k.call
+      end
+    else
+      EM.next_tick(&k)
     end
   end
 
@@ -26,16 +31,18 @@ class Mercury
                  parallelism: 1,
                  on_error: nil,
                  wait_for_publisher_confirms: true,
-                 logger: logger,
+                 logger:,
                  &k)
+    guard_public(k, initializing: true)
     @logger = logger
     @on_error = on_error
     AMQP.connect(host: host, port: port, vhost: vhost, username: username, password: password,
                  on_tcp_connection_failure: server_down_error_handler) do |amqp|
       @amqp = amqp
-      @channel = AMQP::Channel.new(amqp, prefetch: parallelism) do
+      install_lost_connection_error_handler
+      AMQP::Channel.new(amqp, prefetch: parallelism) do |channel|
+        @channel = channel
         install_channel_error_handler
-        install_lost_connection_error_handler
         if wait_for_publisher_confirms
           enable_publisher_confirms do
             k.call(self)
@@ -49,6 +56,7 @@ class Mercury
   private_class_method :new
 
   def publish(source_name, msg, tag: '', headers: {}, &k)
+    guard_public(k)
     # The amqp gem caches exchange objects, so it's fine to
     # redeclare the exchange every time we publish.
     with_source(source_name) do |exchange|
@@ -67,7 +75,8 @@ class Mercury
     { routing_key: tag, persistent: true, headers: Logatron.http_headers.merge(headers) }
   end
 
-  def start_listener(source_name, handler, tag_filter: '#', &k)
+  def start_listener(source_name, handler, tag_filter: nil, &k)
+    guard_public(k)
     with_source(source_name) do |exchange|
       with_listener_queue(exchange, tag_filter) do |queue|
         queue.subscribe(ack: false) do |metadata, payload|
@@ -78,7 +87,8 @@ class Mercury
     end
   end
 
-  def start_worker(worker_group, source_name, handler, tag_filter: '#', &k)
+  def start_worker(worker_group, source_name, handler, tag_filter: nil, &k)
+    guard_public(k)
     with_source(source_name) do |exchange|
       with_work_queue(worker_group, exchange, tag_filter) do |queue|
         queue.subscribe(ack: true) do |metadata, payload|
@@ -90,6 +100,7 @@ class Mercury
   end
 
   def delete_source(source_name, &k)
+    guard_public(k)
     with_source(source_name) do |exchange|
       exchange.delete do
         k.call
@@ -98,6 +109,7 @@ class Mercury
   end
 
   def delete_work_queue(worker_group, &k)
+    guard_public(k)
     @channel.queue(worker_group, work_queue_opts) do |queue|
       queue.delete do
         k.call
@@ -106,6 +118,7 @@ class Mercury
   end
 
   def source_exists?(source_name, &k)
+    guard_public(k)
     existence_check(k) do |ch, &ret|
       with_source_no_cache(ch, source_name, passive: true) do
         ret.call(true)
@@ -114,6 +127,7 @@ class Mercury
   end
 
   def queue_exists?(queue_name, &k)
+    guard_public(k)
     existence_check(k) do |ch, &ret|
       ch.queue(queue_name, passive: true) do
         ret.call(true)
@@ -213,9 +227,7 @@ class Mercury
   end
 
   def handle_channel_error(_ch, info)
-    @amqp.close do
-      make_error_handler("An error occurred: #{info.reply_code} - #{info.reply_text}").call
-    end
+    make_error_handler("An error occurred: #{info.reply_code} - #{info.reply_text}").call
   end
 
   def make_error_handler(msg)
@@ -228,10 +240,12 @@ class Mercury
       current_exception = $!
       unless current_exception
         @logger.error(msg)
-        if @on_error.respond_to?(:call)
-          @on_error.call(msg)
-        else
-          raise msg
+        close do
+          if @on_error.respond_to?(:call)
+            @on_error.call(msg)
+          else
+            raise msg
+          end
         end
       end
     end
@@ -274,10 +288,24 @@ class Mercury
   end
 
   def bind_queue(exchange, queue_name, tag_filter, opts, &k)
+    tag_filter ||= '#'
     queue = @channel.queue(queue_name, opts)
     queue.bind(exchange, routing_key: tag_filter) do
       k.call(queue)
     end
   end
 
+  def guard_public(k, initializing: false)
+    Mercury.guard_public(@amqp.nil?, k, initializing: initializing)
+  end
+
+  def self.guard_public(is_closed, k, initializing: false)
+    if is_closed && !initializing
+      raise 'This mercury instance is defunct. Either it was purposely closed or an error occurred.'
+    end
+    unless k
+      raise 'A continuation block is required but none was provided.'
+    end
+  end
+
 end

data/lib/mercury/version.rb

@@ -1,3 +1,3 @@
 class Mercury
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/spec/lib/mercury/cps_spec.rb

@@ -12,6 +12,9 @@ describe Cps do
     it 'CPS-transforms a non-CPS proc' do
       expect(Cps.lift{rand}.run).to be_a Numeric
     end
+    it 'raises an error if the block returns a Cps object' do
+      expect{lift{lift{'cps'}}.run}.to raise_error /returned a Cps/
+    end
   end
 
   describe '::run' do
@@ -30,12 +33,18 @@ describe Cps do
     it 'composes two Cps instances' do
       expect(lift{a}.and_then{|x| to_string(x)}.run).to eql a.to_s
     end
+    it 'raises an error if the block does not return a Cps object' do
+      expect{lift{a}.and_then{'not-cps'}.run}.to raise_error /did not return a Cps/
+    end
   end
 
   describe '#and_lift' do
     it 'composes a Cps instance with a normal proc' do
       expect(lift{a}.and_lift{|x| x.to_s}.run).to eql a.to_s
     end
+    it 'raises an error if the block returns a Cps object' do
+      expect{lift{a}.and_lift{lift{'cps'}}.run}.to raise_error /returned a Cps/
+    end
   end
 
   describe '::concurrently' do

data/spec/lib/mercury/mercury_spec.rb

@@ -29,7 +29,7 @@ describe Mercury do
       em do
         Mercury.open do |m|
           m.close do
-            expect { m.publish(queue, {'a' => 1}) }.to raise_error /connection is closed/
+            expect { m.publish(queue, {'a' => 1}) }.to raise_error /closed/
             done
           end
         end
@@ -134,6 +134,34 @@ describe Mercury do
     end
   end
 
+  it 'raises when an error occurs' do
+    expect do
+      em do
+        Mercury.open do |m|
+          ch = m.instance_variable_get(:@channel)
+          ch.acknowledge(42) # force a channel error
+        end
+      end
+    end.to raise_error 'An error occurred: 406 - PRECONDITION_FAILED - unknown delivery tag 42'
+  end
+
+  it 'raises a helpful exception if used after a custom error handler suppresses an error' do
+    expect do
+      em do
+        handler = proc do
+          EM.next_tick do
+            @mercury.publish(source, 'hello')
+          end
+        end
+        Mercury.open(on_error: handler) do |m|
+          @mercury = m
+          ch = m.instance_variable_get(:@channel)
+          ch.acknowledge(42) # force a channel error
+        end
+      end
+    end.to raise_error /defunct/
+  end
+
   def start_rabbitmq_server
     Dir.chdir(File.expand_path('~/git/docker/local_orchestration')) do
       system('docker-compose start rabbitmq')

data/spec/lib/mercury/monadic_spec.rb

@@ -145,16 +145,24 @@ describe Mercury::Monadic do
       failures = []
       bars = []
       everything = []
+      everything2 = []
+      everything3 = []
+      all_msgs_received = proc do
+        successes.size == 2 && failures.size == 2 && bars.size == 2 &&
+          everything.size == 4 && everything2.size == 4 && everything3.size == 4
+      end
       seql do
         and_then { m.start_listener(source, tag_filter: '*.success', &successes.method(:push)) }
         and_then { m.start_listener(source, tag_filter: '*.failure', &failures.method(:push)) }
         and_then { m.start_listener(source, tag_filter: 'bar.*', &bars.method(:push)) }
         and_then { m.start_listener(source, tag_filter: '#', &everything.method(:push)) }
+        and_then { m.start_listener(source, tag_filter: nil, &everything2.method(:push)) }
+        and_then { m.start_worker(worker, source, tag_filter: nil) { |msg| everything3.push(msg); msg.ack } }
         and_then { m.publish(source, msg1, tag: 'foo.success') }
         and_then { m.publish(source, msg2, tag: 'foo.failure') }
         and_then { m.publish(source, msg3, tag: 'bar.success') }
         and_then { m.publish(source, msg4, tag: 'bar.failure') }
-        and_then { wait_until { successes.size == 2 && failures.size == 2 && bars.size == 2 && everything.size == 4 } }
+        and_then { wait_until(&all_msgs_received) }
         and_lift do
           expect(successes[0].content).to eql(msg1)
           expect(successes[1].content).to eql(msg3)
@@ -166,6 +174,8 @@ describe Mercury::Monadic do
           expect(everything[1].content).to eql(msg2)
           expect(everything[2].content).to eql(msg3)
           expect(everything[3].content).to eql(msg4)
+          expect(everything2.map(&:content)).to eql(everything.map(&:content))
+          expect(everything3.map(&:content)).to eql(everything.map(&:content))
         end
       end
     end
@@ -273,17 +283,6 @@ describe Mercury::Monadic do
     end
   end
 
-  it 'raises when an error occurs' do
-    expect do
-      em do
-        Mercury.open do |m|
-          ch = m.instance_variable_get(:@channel)
-          ch.acknowledge(42) # force a channel error
-        end
-      end
-    end.to raise_error 'An error occurred: 406 - PRECONDITION_FAILED - unknown delivery tag 42'
-  end
-
   describe '#delete_source' do
     itt 'deletes the source if it exists' do
       test_with_mercury do |m|

data/spec/spec_helper.rb

@@ -36,11 +36,7 @@ module MercuryFakeSpec
         it(name, &block)
       end
       context 'with Mercury::Fake' do
-        before :each do
-          allow(Mercury).to receive(:open) do |parallelism:1, &k|
-            EM.next_tick { k.call(Mercury::Fake.new(parallelism: parallelism)) }
-          end
-        end
+        before(:each) { Mercury::Fake.install(self) }
         it(name, &block)
       end
     end

data/typical.dot

@@ -0,0 +1,17 @@
+digraph G {
+        rankdir=LR
+        p1 [label="publisher", shape=rect]
+        p2 [label="publisher", shape=rect]
+        p3 [label="publisher", shape=rect]
+        s [label="source", shape=circle]
+        wq [label="{|||||}", shape=record, fixedsize=true, height=0.3, width=1.5]
+        w1 [label="worker A", shape=rect]
+        w2 [label="worker A", shape=rect]
+        w3 [label="worker A", shape=rect]
+
+        p1 -> s [tailport=e]
+        p2 -> s [tailport=e]
+        p3 -> s [tailport=e]
+        s -> wq [headport=w]
+        wq -> {w1, w2, w3} [tailport=e, headport=w]
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mercury_amqp
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Peter Winton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-22 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -190,6 +190,8 @@ files:
 - Gemfile
 - README.md
 - Rakefile
+- constructs.dot
+- constructs.png
 - lib/mercury.rb
 - lib/mercury/cps.rb
 - lib/mercury/cps/methods.rb
@@ -218,6 +220,8 @@ files:
 - spec/lib/mercury/utils_spec.rb
 - spec/lib/mercury/wire_serializer_spec.rb
 - spec/spec_helper.rb
+- typical.dot
+- typical.png
 homepage: https://github.com/wintonpc/mercury_amqp
 licenses:
 - MIT