mercury_amqp 0.1.9 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b38b3dede66b9e8be986f8b8f3fc480e62d326b6
-  data.tar.gz: 4b9b9dc78020adb4eefa32e0ec25b1e846088c41
+  metadata.gz: 2fd7b0dd970c7160af30546770764c5fc2725bc6
+  data.tar.gz: a702ad714aba8bd83308c19905841f0cec380430
 SHA512:
-  metadata.gz: de30e69c5f7d06a9579a28d38f5604038b0227ea0b380541c582107400139862bce47ab59c9be1cc6edd19a2d88533a29a3cb0ff8b26c607f67fe2337fbdc656
-  data.tar.gz: 79ecf41880b13bee8f514aeb9d3aaff18d4b17ad009fdbe336c20a0384ba002db06c99398b9861258f5aef34fe79fa8658d333310f6820f844e8cea6fe01a11a
+  metadata.gz: ef2d9c3392044008379014cfe5a12d22849c5f36a0852a3f36dccc6a6677f8b1d65f3c51143dacf9607077164abfb10874f369dc0f62246edc5a2b04fb5c757c
+  data.tar.gz: 687e95ee42e0e32aaf9290669ed11ba2e68f3b1b1acf43195990158a2724e285f708028e1ab43d20cfd5d5953c5bd1fbae4a98c75c31a33bc34f0c2123d64c98

data/Gemfile

@@ -2,4 +2,3 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in hyperion.gemspec
 gemspec
-

data/lib/mercury/mercury.rb

@@ -26,16 +26,22 @@ class Mercury
                  password: 'guest',
                  parallelism: 1,
                  on_error: nil,
+                 wait_for_publisher_confirms: true,
                  &k)
     @on_error = on_error
     AMQP.connect(host: host, port: port, vhost: vhost, username: username, password: password,
                  on_tcp_connection_failure: server_down_error_handler) do |amqp|
       @amqp = amqp
       @channel = AMQP::Channel.new(amqp, prefetch: parallelism) do
-        @channel.confirm_select
         install_channel_error_handler
         install_lost_connection_error_handler
-        k.call(self)
+        if wait_for_publisher_confirms
+          enable_publisher_confirms do
+            k.call(self)
+          end
+        else
+          k.call(self)
+        end
       end
     end
   end
@@ -44,10 +50,14 @@ class Mercury
   def publish(source_name, msg, tag: '', headers: {}, &k)
     # The amqp gem caches exchange objects, so it's fine to
     # redeclare the exchange every time we publish.
-    # TODO: wait for publish confirmations (@channel.on_ack)
     with_source(source_name) do |exchange|
-      exchange.publish(write(msg), **Mercury.publish_opts(tag, headers)) do
-        k.call
+      payload = write(msg)
+      pub_opts = Mercury.publish_opts(tag, headers)
+      if publisher_confirms_enabled
+        expect_publisher_confirm(k)
+        exchange.publish(payload, **pub_opts)
+      else
+        exchange.publish(payload, **pub_opts, &k)
       end
     end
   end
@@ -112,6 +122,58 @@ class Mercury
 
   private
 
+  # In AMQP, queue consumers ack requests after handling them. Unacked messages
+  # are automatically returned to the queue, guaranteeing they are eventually handled.
+  # Services often ack one request while publishing related messages. Ideally, these
+  # operations would be transactional. If the ack succeeds but the publish does not,
+  # the line of processing is abandoned, resulting in processing getting "stuck".
+  # The best we can do in AMQP is to use "publisher confirms" to confirm that the publish
+  # succeeded before acking the originating request. Since the ack can still fail in this
+  # scenario, the system should employ idempotent design, which makes request redelivery
+  # harmless.
+  #
+  # see https://www.rabbitmq.com/confirms.html
+  # see http://rubyamqp.info/articles/durability/
+  def enable_publisher_confirms(&k)
+    @confirm_handlers = {}
+    @channel.confirm_select do
+      @last_published_delivery_tag = 0
+      @channel.on_ack do |basic_ack|
+        tag = basic_ack.delivery_tag
+        if @confirm_handlers.keys.exclude?(tag)
+          raise "Got an unexpected publish confirmation ACK for delivery-tag: #{tag}. Was expecting one of: #{@confirm_handlers.keys.inspect}"
+        end
+        dispatch_publisher_confirm(basic_ack)
+      end
+      @channel.on_nack do |basic_nack|
+        raise "Delivery failed for message with delivery-tag: #{basic_nack.delivery_tag}"
+      end
+      k.call
+    end
+  end
+
+  def publisher_confirms_enabled
+    @confirm_handlers.is_a?(Hash)
+  end
+
+  def expect_publisher_confirm(k)
+    expected_delivery_tag = (@last_published_delivery_tag += 1)
+    @confirm_handlers[expected_delivery_tag] = k
+    expected_delivery_tag
+  end
+
+  def dispatch_publisher_confirm(basic_ack)
+    confirmed_tags =
+      if basic_ack.multiple
+        @confirm_handlers.keys.select { |tag| tag <= basic_ack.delivery_tag }.sort # sort just to be deterministic
+      else
+        [basic_ack.delivery_tag]
+      end
+    confirmed_tags.each do |tag|
+      @confirm_handlers.delete(tag).call
+    end
+  end
+
   def make_received_message(payload, metadata, is_ackable)
     msg = ReceivedMessage.new(read(payload), metadata, is_ackable: is_ackable)
     Logatron.msg_id = msg.headers['X-Ascent-Log-Id']
@@ -157,11 +219,19 @@ class Mercury
 
   def make_error_handler(msg)
     proc do
-      Logatron.error(msg)
-      if @on_error.respond_to?(:call)
-        @on_error.call(msg)
-      else
-        raise msg
+      # If an error is already being raised, don't interfere with it.
+      # This is actually essential since some versions of EventMachine (notably 1.2.0.1)
+      # fail to clean up properly if an error is raised during the `ensure` clean up
+      # phase (in EventMachine::run), which zombifies subsequent reactors. (AMQP connection
+      # failure handlers are invoked from EventMachine's `ensure`.)
+      current_exception = $!
+      unless current_exception
+        Logatron.error(msg)
+        if @on_error.respond_to?(:call)
+          @on_error.call(msg)
+        else
+          raise msg
+        end
       end
     end
   end

data/lib/mercury/sync.rb

@@ -4,14 +4,17 @@ require 'bunny'
 class Mercury
   class Sync
     class << self
-      def publish(source_name, msg, tag: '', amqp_opts: {})
+      def publish(source_name, msg, tag: '', amqp_opts: {}, wait_for_publisher_confirms: true)
         conn = Bunny.new(amqp_opts)
         conn.start
         ch = conn.create_channel
-        ch.confirm_select
+
+        ch.confirm_select if wait_for_publisher_confirms # see http://rubybunny.info/articles/extensions.html and Mercury#enable_publisher_confirms
         ex = ch.topic(source_name, Mercury.source_opts)
         ex.publish(WireSerializer.new.write(msg), **Mercury.publish_opts(tag, {}))
-        ch.wait_for_confirms or raise 'failed to confirm publication'
+        if wait_for_publisher_confirms
+          ch.wait_for_confirms or raise 'failed to confirm publication'
+        end
       ensure
         conn.close
       end

data/lib/mercury/test_utils.rb

@@ -6,9 +6,9 @@ class Mercury
   module TestUtils
     include Cps::Methods
 
-    def em
+    def em(timeout_seconds: 3)
       EM.run do
-        EM.add_timer(in_debug_mode? ? 999999 : 3) { raise 'EM spec timed out' }
+        EM.add_timer(in_debug_mode? ? 999999 : timeout_seconds) { raise 'EM spec timed out' }
         yield
       end
     end

data/lib/mercury/version.rb

@@ -1,3 +1,3 @@
 class Mercury
-  VERSION = '0.1.9'
+  VERSION = '0.2.0'
 end

data/spec/lib/mercury/mercury_spec.rb

@@ -75,12 +75,85 @@ describe Mercury do
     end
   end
 
-  def with_mercury(&block)
+  # Commented out until this gets fixed: https://github.com/eventmachine/eventmachine/issues/670
+  # it 'raises an error when a connection cannot be established' do
+  #   expect { em { Mercury.open(port: 31999) { done } } }.to raise_error /Failed to establish connection/
+  #   expect(EM.reactor_running?).to be false
+  # end
+
+  it 'raises an error when the connection breaks' do
+    expect { em { Mercury.open { done } } }.to raise_error /Lost connection/
+    expect(EM.reactor_running?).to be false   # make sure we're not triggering EventMachine cleanup bugs
+  end
+
+  it 'does not obscure exceptions thrown inside the reactor' do
+    expect { em { Mercury.open { raise 'oops' } } }.to raise_error 'oops'
+    expect(EM.reactor_running?).to be false   # make sure we're not triggering EventMachine cleanup bugs
+  end
+
+  describe '#publish' do
+    context 'docker assumptions' do
+      after(:each) { start_rabbitmq_server }
+      # This test is commented out because it make assumptions about, and manipulates, a local docker orchestration.
+      # Uncomment and run it in a suitable environment to do semi-automated testing on publisher confirms.
+      # it 'waits to invoke its continuation until after the message is confirmed' do
+      #   got_confirmation = false
+      #   expect {
+      #     with_mercury(timeout_seconds: 15) do |m|
+      #       m.publish(source, 'hello') do # cause `source` to be declared so subsequent publishes simply publish
+      #         stop_rabbitmq_server
+      #         m.publish(source, 'hello') do
+      #           got_confirmation = true
+      #           m.close { done }
+      #         end
+      #       end
+      #     end
+      #   }.to raise_error /Lost connection to AMQP server/
+      #   expect(got_confirmation).to be false
+      # end
+    end
+    it 'allows multiple outstanding confirmations' do
+      log = []
+      with_mercury do |m|
+        m.publish(source, 'hello') do # cause `source` to be declared so subsequent publishes simply publish
+          publish_and_confirm(m, 'a', log)
+          publish_and_confirm(m, 'b', log)
+          em_wait_until(proc { log.size == 4 }) do
+            expect(log).to eql ['publish a', 'publish b', 'confirm a', 'confirm b']
+            m.close { done }
+          end
+        end
+      end
+    end
+
+    def publish_and_confirm(m, name, log)
+      m.publish(source, name) do
+        log << "confirm #{name}"
+      end
+      log << "publish #{name}"
+    end
+  end
+
+  def start_rabbitmq_server
+    Dir.chdir(File.expand_path('~/git/docker/local_orchestration')) do
+      system('docker-compose start rabbitmq')
+    end
+    puts 'done.'
+  end
+
+  def stop_rabbitmq_server
+    Dir.chdir(File.expand_path('~/git/docker/local_orchestration')) do
+      system('docker-compose stop rabbitmq')
+    end
+    puts 'done.'
+  end
+
+  def with_mercury(timeout_seconds: 3, &block)
     sources = [source]
     queues = [queue]
-    em { delete_sources_and_queues_cps(sources, queues).run{done} }
-    em { Mercury.open(&block) }
-    em { delete_sources_and_queues_cps(sources, queues).run{done} }
+    em(timeout_seconds: timeout_seconds) { delete_sources_and_queues_cps(sources, queues).run{done} }
+    em(timeout_seconds: timeout_seconds) { Mercury.open(&block) }
+    em(timeout_seconds: timeout_seconds) { delete_sources_and_queues_cps(sources, queues).run{done} }
   end
 
 end

data/spec/lib/mercury/monadic_spec.rb

@@ -78,8 +78,8 @@ describe Mercury::Monadic do
       end
       seql do
         and_then { m.start_worker('worker1', source1, handle_msg) }
-        and_then { m.publish(source1, {'id' => 1, 'sleep_seconds' => 0.01}) }
-        and_then { m.publish(source1, {'id' => 2, 'sleep_seconds' => 0.01}) }
+        and_then { m.publish(source1, {'id' => 1, 'sleep_seconds' => 0.1}) }
+        and_then { m.publish(source1, {'id' => 2, 'sleep_seconds' => 0.1}) }
         and_then { wait_until { events.size == 4 } }
         and_lift do
           expect(events).to eql ['received 1', 'received 2', 'finished 1', 'finished 2']
@@ -128,8 +128,8 @@ describe Mercury::Monadic do
       msgs = []
       seql do
         and_then { m.start_listener(source, &msgs.method(:push)) }
+        and_lift { EM.next_tick { Logatron.msg_id = 'fake_msg_id' } } # we want this to happen right after publishing but before getting the response
         and_then { m.publish(source, msg) }
-        and_lift { Logatron.msg_id = 'fake_msg_id' }
         and_then { wait_until { msgs.size == 1 } }
         and_lift do
           expect(msgs[0].headers['X-Ascent-Log-Id']).to eql real_msg_id
@@ -188,14 +188,21 @@ describe Mercury::Monadic do
     end
   end
 
+  def push_and_ack(array)
+    proc do |msg|
+      array.push(msg)
+      msg.ack
+    end
+  end
+
   itt 'workers can specify tag filters' do
     test_with_mercury do |m|
       seql do
         let(:m2) { Mercury::Monadic.open }
         work1 = []
         work2 = []
-        and_then { m.start_worker(worker, source, tag_filter: 'success', &push_and_ack(work1)) }
-        and_then { m2.start_worker(worker, source, tag_filter: 'failure', &push_and_ack(work2)) }
+        and_then { m.start_worker(worker, source, tag_filter: 'success', &work1.method(:push)) }
+        and_then { m2.start_worker(worker, source, tag_filter: 'failure', &work2.method(:push)) }
         and_then { m.publish(source, msg1, tag: 'success') }
         and_then { m.publish(source, msg2, tag: 'failure') }
         and_then { wait_until { work1.size == 1 && work2.size == 1 } }
@@ -208,13 +215,6 @@ describe Mercury::Monadic do
     end
   end
 
-  def push_and_ack(array)
-    proc do |msg|
-      array.push(msg)
-      msg.ack
-    end
-  end
-
   itt 'a worker must ack before receiving another message' do
     test_with_mercury do |m|
       msgs = []

data/spec/lib/mercury/sync_spec.rb

@@ -7,17 +7,20 @@ describe Mercury::Sync do
   let!(:source) { 'test-exchange1' }
   let!(:queue) { 'test-queue1' }
   describe '::publish' do
-    it 'publishes synchronously' do
-      sent = {'a' => 1}
-      received = []
-      test_with_mercury do |m|
-        seql do
-          and_then { m.start_listener(source, received.method(:push)) }
-          and_lift { Mercury::Sync.publish(source, sent) }
-          and_then { wait_until { received.any? } }
-          and_lift do
-            expect(received.size).to eql 1
-            expect(received[0].content).to eql sent
+    %w{with without}.each do |w|
+      it "publishes synchronously (#{w} publisher confirms)" do
+        use_publisher_confirms = w == 'with'
+        sent = {'a' => 1}
+        received = []
+        test_with_mercury(wait_for_publisher_confirms: use_publisher_confirms) do |m|
+          seql do
+            and_then { m.start_listener(source, received.method(:push)) }
+            and_lift { Mercury::Sync.publish(source, sent) }
+            and_then { wait_until { received.any? } }
+            and_lift do
+              expect(received.size).to eql 1
+              expect(received[0].content).to eql sent
+            end
           end
         end
       end
@@ -25,9 +28,9 @@ describe Mercury::Sync do
   end
 
   # the block must return a Cps
-  def test_with_mercury(&block)
+  def test_with_mercury(**kws, &block)
     sources = [source]
     queues = [queue]
-    test_with_mercury_cps(sources, queues, &block)
+    test_with_mercury_cps(sources, queues, **kws, &block)
   end
 end

data/spec/spec_helper.rb

@@ -4,10 +4,10 @@ require 'mercury/fake'
 include Mercury::TestUtils
 
 # the block must return a Cps
-def test_with_mercury_cps(sources, queues, parallelism: 1, &block)
+def test_with_mercury_cps(sources, queues, **kws, &block)
   em do
     seql do
-      let(:m)  { Mercury::Monadic.open(parallelism: parallelism) }
+      let(:m)  { Mercury::Monadic.open(**kws) }
       and_then { delete_sources_and_queues_cps(sources, queues) }
       and_then { block.call(m) }
       and_then { delete_sources_and_queues_cps(sources, queues) }
@@ -26,6 +26,15 @@ module MercuryFakeSpec
     # runs a test once with real mercury and once with Mercury::Fake
     def itt(name, &block)
       it(name, &block)
+      context 'without publisher confirms' do
+        before :each do
+          real_open = Mercury.method(:open)
+          allow(Mercury).to receive(:open) do |**kws, &k|
+            real_open.call(**kws.merge(wait_for_publisher_confirms: false), &k)
+          end
+        end
+        it(name, &block)
+      end
       context 'with Mercury::Fake' do
         before :each do
           allow(Mercury).to receive(:open) do |parallelism:1, &k|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mercury_amqp
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.0
 platform: ruby
 authors:
 - Peter Winton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-12 00:00:00.000000000 Z
+date: 2016-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler