mercury_amqp 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 22461cdd32d965f465191c9f4270ea23724d9d14
+  data.tar.gz: 4934514f73a40ee799e174eba6dabe11add998c4
+SHA512:
+  metadata.gz: de0a9fa90882b011bdd0011055f2546ad1a7b6d0ed9deadc5e0c7c3c7f759a5824267fa9e6c9c75a834b23f3413be306de6b303766c5e024af32a85f4c750225
+  data.tar.gz: 639003755c8e26bbdcf98b66178d1d32cd91a2c8b7ba4b6efb1a98996915a7715381077d06f895d892358edf719e5975da7ae34e77401cb531d62b85a4447a8b

data/.gitignore

@@ -0,0 +1,4 @@
+.idea
+.bundle
+Gemfile.lock
+*~

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hyperion.gemspec
+gemspec
+

data/README.md

@@ -0,0 +1,85 @@
+mercury
+=======
+
+Mercury is a messaging layer intended to hide complexity for typical
+messaging scenarios. It is backed by the AMQP gem and consequently
+runs in an EventMachine reactor and has an asynchronous API. Mercury
+consists of _sources_, _work queues_, and _listeners_. A message is
+published to a source, to which one or more work queues and/or
+listeners are attached. These map roughly to AMQP constructs:
+
+- **source**: topic exchange
+- **work queue**: durable named queue
+- **listener**: temporary anonymous queue
+- **tag**: routing key
+
+At the moment, mercury is backed by AMQP and serializes messages as
+JSON. In the future, additional transports and message formats may be
+supported.
+
+Mercury writes string messages directly without encoding; this allows
+a client to pre-encode a message using an arbitrary encoding. The
+receiving client receives the encoded bytes as the message content
+(assuming the encoded message fails to parse as JSON).
+
+
+```ruby
+require 'mercury'
+
+def run
+  EventMachine.run do
+    Mercury.open do |m|
+      m.start_worker('cooks', 'orders', method(:handle_message)) do
+        m.publish('orders', {'table' => 5, 'items' => ['salad', 'steak', 'cake']})
+      end
+    end
+  end
+end
+
+def handle_message(msg)
+  order = msg.content
+  cook(order)
+  msg.ack
+end
+```
+
+Notably, mercury also has a monadic interface that hides the explicit
+continuation passing introduced by asynchrony, which has the effect of
+flattening chained calls. This is particularly useful for testing,
+where the same code plays both sides of a conversation. Compare:
+
+```ruby
+require 'mercury'
+
+Mercury.open do |m|
+  m.start_listener(source, proc{}) do
+    m.source_exists?(source) do |r1|
+      expect(r1).to be true
+      m.delete_source(source) do
+        m.source_exists?(source) do |r2|
+          expect(r2).to be false
+          m.close do
+            done
+          end
+        end
+      end
+    end
+  end
+end
+
+# ... vs ...
+
+require 'mercury/monadic'
+
+seq do
+  let(:m)  { Mercury::Monadic.open    }
+  and_then { m.start_listener(source) }
+  let(:r1) { m.source_exists?(source) }
+  and_lift { expect(r1).to be true    }
+  and_then { m.delete_source(source)  }
+  let(:r2) { m.source_exists?(source) }
+  and_lift { expect(r2).to be false   }
+  and_then { m.close                  }
+  and_lift { done                     }
+end
+```

data/Rakefile

@@ -0,0 +1,9 @@
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  task :default => :spec
+rescue LoadError
+  # no rspec available
+end

data/lib/mercury/cps/methods.rb

@@ -0,0 +1,23 @@
+class Cps
+  module Methods
+    def lift(&block)
+      Cps.lift(&block)
+    end
+
+    def seq(&block)
+      Cps.seq(&block)
+    end
+
+    def seql(&block)
+      Cps.seql(2, &block)
+    end
+
+    def seqp(&block)
+      Cps.seqp(&block)
+    end
+
+    def cps(&block)
+      Cps.new(&block)
+    end
+  end
+end

data/lib/mercury/cps/seq.rb

@@ -0,0 +1,23 @@
+class Cps
+
+  # Syntactic sugar for and_then chains.
+  def self.seq(&block)
+    s = Seq.new
+    block.call(s.method(:chain))
+    s.m
+  end
+
+  class Seq
+    def m
+      @m ||= Cps.identity # we need an initial Cps to chain onto
+    end
+
+    def chain(proc=nil, &block)
+      @m = m.and_then(&(proc || block))
+    end
+  end
+end
+
+class Method
+  alias_method :en, :call # to be called `th.en`
+end

data/lib/mercury/cps/seq_with_let.rb

@@ -0,0 +1,67 @@
+require 'binding_of_caller'
+
+class Cps
+  # Syntactic sugar for and_then chains.
+  def self.seql(depth=1, &block)
+    # EXPERIMENTAL
+    # The trick here is to execute the block in a context where
+    # 1. we can simulate local let-bound variables, and
+    # 2. the block can access variables and methods available
+    #    outside the call to seql.
+    #
+    # To achieve this, we instance_exec the block in a SeqWithLet
+    # object, which provides the let bound variables (as methods)
+    # and uses method_missing to proxy other methods to the parent
+    # binding.
+    #
+    # Note: parent instance variables are not available inside the block.
+    # Note: keyword arguments are not proxied to methods called in the parent binding
+    context = SeqWithLet.new(binding.of_caller(depth))
+    context.instance_exec(&block)
+    context.__chain
+  end
+
+  class SeqWithLet
+
+    def and_then(&block)
+      @__chain = __chain.and_then(&block)
+    end
+
+    def and_lift(&block)
+      @__chain = __chain.and_lift(&block)
+    end
+
+    def let(name, &block)
+      and_then(&block)
+      and_then do |value|
+        __values[name] = value
+        Cps.lift{value}
+      end
+    end
+
+    def initialize(parent_binding)
+      @__parent_binding = parent_binding
+    end
+
+    def __chain
+      @__chain ||= Cps.identity
+    end
+
+    def method_missing(name, *args, &block)
+      __values.fetch(name) { __parent_call(name.to_s, *args, &block) }
+    end
+
+    def __parent_call(name, *args, &block)
+      @__parent_caller ||= @__parent_binding.eval <<-EOD
+      proc do |name, *args, &block|
+        send(name, *args, &block)
+      end
+      EOD
+      @__parent_caller.call(name, *args, &block)
+    end
+
+    def __values
+      @__values ||= {}
+    end
+  end
+end

data/lib/mercury/cps.rb

@@ -0,0 +1,97 @@
+require 'mercury/cps/seq'
+require 'mercury/cps/seq_with_let'
+require 'mercury/cps/methods'
+require 'mercury/utils'
+
+# Async IO often involves CPS (continuation-passing style)
+# code, where the continuation (a.k.a. "callback") is passed
+# to a function to be invoked at a later time. CPS style
+# can result in deep lexical nesting making code difficult
+# to read.
+# This monad hides the CPS plumbing, which allows code
+# to be written in a flat style with no visible continuation
+# passing. It basically wraps a CPS proc with methods for
+# composing them.
+# See http://codon.com/refactoring-ruby-with-monads
+class Cps
+  attr_reader :cps
+
+  # @param [Proc] cps a CPS proc (signature: *args, &k)
+  def initialize(&cps)
+    @cps = cps
+  end
+
+  # Applies the wrapped proc. If the CPS return value
+  # is not needed, the continuation k may be omitted.
+  # Returns the return value of the continuation.
+  def run(*args, &k)
+    k ||= proc { |x| x }
+    cps.call(*args, &k)
+  end
+
+  # The "bind" operation; composes two Cps
+  # @param [Proc] pm a proc that takes the output of this
+  #   Cps and returns a Cps
+  def and_then(&pm)
+    Cps.new do |*args, &k|
+      self.run(*args) do |*args2|
+        next_cps = pm.call(*args2)
+        next_cps.is_a?(Cps) or raise "'and_then' block did not return a Cps. Did you want 'and_lift'? at #{pm.source_location}"
+        next_cps.run(&k)
+      end
+    end
+  end
+
+  # equivalent to: and_then { lift { ... } }
+  def and_lift(&p)
+    and_then do |*args|
+      Cps.lift { p.call(*args) }
+    end
+  end
+
+  # Returns a Cps for a non-CPS proc.
+  def self.lift(&p)
+    new { |*args, &k| k.call(p.call(*args)) }
+  end
+
+  # The identity function as a Cps.
+  def self.identity
+    new { |*args, &k| k.call(*args) }
+  end
+
+  # Returns a Cps that executes the provided Cpses concurrently.
+  # Once all complete, their return values are passed to the continuation
+  # in an array with positions corresponding to the provided Cpses.
+  def self.concurrently(*cpss)
+    cpss = Utils.unsplat(cpss)
+
+    Cps.new do |*in_args, &k|
+      pending_completions = cpss
+      returned_args = []
+      cpss.each_with_index do |cps, i|
+        cps.run(*in_args) do |*out_args|
+          returned_args[i] = out_args
+          pending_completions.delete(cps)
+          if pending_completions.none?
+            k.call(returned_args)
+          end
+        end
+      end
+    end
+  end
+
+  # Calls and_then for each x.
+  # @yieldparam  x     [Object]  An item from xs
+  # @yieldparam  *args [Objects] The value(s) passed from the last action
+  # @yieldreturn       [Cps]     The next action to add to the chain
+  def inject(xs, &block)
+    xs.inject(self) do |chain, x|
+      chain.and_then { |*args| block.call(x, *args) }
+    end
+  end
+
+  # equivalent to Cps.identity.inject(...)
+  def self.inject(xs, &block)
+    Cps.identity.inject(xs, &block)
+  end
+end

data/lib/mercury/fake/domain.rb

@@ -0,0 +1,9 @@
+class Mercury
+  class Fake
+    class Domain
+      def queues
+        @queues ||= {}
+      end
+    end
+  end
+end

data/lib/mercury/fake/metadata.rb

@@ -0,0 +1,24 @@
+class Mercury
+  class Fake
+    class Metadata
+      def initialize(tag, dequeue, requeue)
+        @tag = tag
+        @dequeue = dequeue
+        @requeue = requeue
+      end
+
+      def routing_key
+        @tag
+      end
+
+      def ack
+        @dequeue.call
+      end
+
+      def reject(opts)
+        requeue = opts[:requeue]
+        requeue ? @requeue.call : @dequeue.call
+      end
+    end
+  end
+end

data/lib/mercury/fake/queue.rb

@@ -0,0 +1,80 @@
+require 'eventmachine'
+require 'mercury/fake/queued_message'
+
+class Mercury
+  class Fake
+    class Queue
+      attr_reader :source, :worker
+
+      def initialize(source, tag_filter, worker, require_ack)
+        @source = source
+        @tag_filter = tag_filter
+        @worker = worker
+        @require_ack = require_ack
+        @msgs = []
+        @subscribers = []
+      end
+
+      def add_subscriber(s)
+        subscribers << s
+        deliver # new subscriber probably wants a message
+      end
+
+      def enqueue(msg, tag)
+        msgs.push(QueuedMessage.new(self, msg, tag, @require_ack))
+        deliver # new message. someone probably wants it.
+      end
+
+      def ack_or_reject_message(msg)
+        msgs.delete(msg) or raise 'tried to delete message that was not in queue!!'
+        msg.subscriber.busy = false
+        deliver # a subscriber just freed up
+      end
+
+      def nack(msg)
+        msg.delivered = false
+        msg.subscriber.busy = false
+        deliver
+      end
+
+      def binds?(source_name, tag)
+        source_name == source && tag_match?(tag_filter, tag)
+      end
+
+      private
+      attr_reader :msgs, :subscribers, :tag_filter
+
+      def tag_match?(filter, tag)
+        # for wildcard description, see https://www.rabbitmq.com/tutorials/tutorial-five-python.html
+        pattern = Regexp.new(filter.gsub('*', '[^\.]+').gsub('#', '.*?'))
+        pattern.match(tag)
+      end
+
+      def deliver
+        EM.next_tick do
+          if idle_subscribers.any? && undelivered.any?
+            msg = undelivered.first
+            subscriber = idle_subscribers.sample
+            if @require_ack
+              msg.delivered = true
+              subscriber.busy = true
+            else
+              msgs.delete(msg)
+            end
+            msg.subscriber = subscriber
+            subscriber.handler.call(msg.received_msg)
+            deliver # continue delivering
+          end
+        end
+      end
+
+      def undelivered
+        msgs.reject(&:delivered)
+      end
+
+      def idle_subscribers
+        subscribers.reject(&:busy)
+      end
+    end
+  end
+end

data/lib/mercury/fake/queued_message.rb

@@ -0,0 +1,17 @@
+require 'mercury/fake/metadata'
+require 'mercury/received_message'
+
+class Mercury
+  class Fake
+    class QueuedMessage
+      attr_reader :received_msg
+      attr_accessor :delivered, :subscriber
+
+      def initialize(queue, msg, tag, is_ackable)
+        metadata = Metadata.new(tag, proc{queue.ack_or_reject_message(self)}, proc{queue.nack(self)})
+        @received_msg = ReceivedMessage.new(msg, metadata, is_ackable: is_ackable)
+        @delivered = false
+      end
+    end
+  end
+end

data/lib/mercury/fake/subscriber.rb

@@ -0,0 +1,13 @@
+class Mercury
+  class Fake
+    class Subscriber
+      attr_reader :handler
+      attr_accessor :busy
+
+      def initialize(handler)
+        @handler = handler
+        @busy = false
+      end
+    end
+  end
+end

data/lib/mercury/fake.rb

@@ -0,0 +1,104 @@
+require 'securerandom'
+require 'delegate'
+require 'mercury/received_message'
+require 'mercury/fake/domain'
+require 'mercury/fake/metadata'
+require 'mercury/fake/queue'
+require 'mercury/fake/queued_message'
+require 'mercury/fake/subscriber'
+
+# This class simulates Mercury without using the AMQP gem.
+# It can be useful for unit testing code that uses Mercury.
+# The domain concept allows different mercury instances to
+# hit different virtual servers; this should rarely be needed.
+# This class cannot simulate behavior of server disconnections,
+# broken sockets, etc.
+class Mercury
+  class Fake
+    def initialize(domain=:default)
+      @domain = Fake.domains[domain]
+    end
+
+    def self.domains
+      @domains ||= Hash.new { |h, k| h[k] = Domain.new }
+    end
+
+    def close(&k)
+      @closed = true
+      ret(k)
+    end
+
+    def publish(source_name, msg, tag: '', &k)
+      assert_not_closed
+      queues.values.select{|q| q.binds?(source_name, tag)}.each{|q| q.enqueue(roundtrip(msg), tag)}
+      ret(k)
+    end
+
+    def start_listener(source_name, handler, tag_filter: '#', &k)
+      start_worker_or_listener(source_name, handler, tag_filter, &k)
+    end
+
+    def start_worker(worker_group, source_name, handler, tag_filter: '#', &k)
+      start_worker_or_listener(source_name, handler, tag_filter, worker_group, &k)
+    end
+
+    def start_worker_or_listener(source_name, handler, tag_filter, worker_group=nil, &k)
+      assert_not_closed
+      q = ensure_queue(source_name, tag_filter, !!worker_group, worker_group)
+      ret(k) # it's important we show the "start" operation finishing before delivery starts (in add_subscriber)
+      q.add_subscriber(Subscriber.new(handler))
+    end
+    private :start_worker_or_listener
+
+    def delete_source(source_name, &k)
+      assert_not_closed
+      queues.delete_if{|_k, v| v.source == source_name}
+      ret(k)
+    end
+
+    def delete_work_queue(worker_group, &k)
+      assert_not_closed
+      queues.delete_if{|_k, v| v.worker == worker_group}
+      ret(k)
+    end
+
+    def source_exists?(source, &k)
+      built_in_sources = %w(direct topic fanout headers match rabbitmq.log rabbitmq.trace).map{|x| "amq.#{x}"}
+      ret(k, (queues.values.map(&:source) + built_in_sources).include?(source))
+    end
+
+    def queue_exists?(worker, &k)
+      ret(k, queues.values.map(&:worker).include?(worker))
+    end
+
+    private
+
+    def queues
+      @domain.queues
+    end
+
+    def ret(k, value=nil)
+      EM.next_tick{k.call(value)} if k
+    end
+
+    def roundtrip(msg)
+      ws = WireSerializer.new
+      ws.read(ws.write(msg))
+    end
+
+    def ensure_queue(source, tag_filter, require_ack, worker=nil)
+      worker ||= SecureRandom.uuid
+      queues.fetch(unique_queue_name(source, tag_filter, worker)) do |k|
+        queues[k] = Queue.new(source, tag_filter, worker, require_ack)
+      end
+    end
+
+    def unique_queue_name(source, tag_filter, worker)
+      [source, tag_filter, worker].join('^')
+    end
+
+    def assert_not_closed
+      raise 'connection is closed' if @closed
+    end
+  end
+end