mercury_amqp 0.1.0

data/lib/mercury/mercury.rb

@@ -0,0 +1,186 @@
+require 'amqp'
+require 'securerandom'
+require 'mercury/wire_serializer'
+require 'mercury/received_message'
+require 'logger'
+
+class Mercury
+  attr_reader :amqp, :channel, :logger
+
+  def self.open(logger: Logger.new(STDOUT), **kws, &k)
+    @logger = logger
+    new(**kws, &k)
+    nil
+  end
+
+  def close(&k)
+    @amqp.close do
+      k.call
+    end
+  end
+
+  def initialize(host: 'localhost',
+                 port: 5672,
+                 vhost: '/',
+                 username: 'guest',
+                 password: 'guest',
+                 &k)
+    AMQP.connect(host: host, port: port, vhost: vhost, username: username, password: password) do |amqp|
+      @amqp = amqp
+      @channel = AMQP::Channel.new(amqp, prefetch: 1) do
+        @channel.confirm_select
+        install_default_error_handler
+        k.call(self)
+      end
+    end
+  end
+  private_class_method :new
+
+  def publish(source_name, msg, tag: '', &k)
+    # The amqp gem caches exchange objects, so it's fine to
+    # redeclare the exchange every time we publish.
+    # TODO: wait for publish confirmations (@channel.on_ack)
+    with_source(source_name) do |exchange|
+      exchange.publish(write(msg), **Mercury.publish_opts(tag)) do
+        k.call
+      end
+    end
+  end
+
+  def self.publish_opts(tag)
+    { routing_key: tag, persistent: true }
+  end
+
+  def start_listener(source_name, handler, tag_filter: '#', &k)
+    with_source(source_name) do |exchange|
+      with_listener_queue(exchange, tag_filter) do |queue|
+        queue.subscribe(ack: false) do |metadata, payload|
+          handler.call(make_received_message(payload, metadata, false))
+        end
+        k.call
+      end
+    end
+  end
+
+  def start_worker(worker_group, source_name, handler, tag_filter: '#', &k)
+    with_source(source_name) do |exchange|
+      with_work_queue(worker_group, exchange, tag_filter) do |queue|
+        queue.subscribe(ack: true) do |metadata, payload|
+          handler.call(make_received_message(payload, metadata, true))
+        end
+        k.call
+      end
+    end
+  end
+
+  def delete_source(source_name, &k)
+    with_source(source_name) do |exchange|
+      exchange.delete do
+        k.call
+      end
+    end
+  end
+
+  def delete_work_queue(worker_group, &k)
+    @channel.queue(worker_group, work_queue_opts) do |queue|
+      queue.delete do
+        k.call
+      end
+    end
+  end
+
+  def source_exists?(source_name, &k)
+    existence_check(k) do |ch, &ret|
+      with_source_no_cache(ch, source_name, passive: true) do
+        ret.call(true)
+      end
+    end
+  end
+
+  def queue_exists?(queue_name, &k)
+    existence_check(k) do |ch, &ret|
+      ch.queue(queue_name, passive: true) do
+        ret.call(true)
+      end
+    end
+  end
+
+  private
+
+  def make_received_message(payload, metadata, is_ackable)
+    ReceivedMessage.new(read(payload), metadata, is_ackable: is_ackable)
+  end
+
+  def existence_check(k, &check)
+    AMQP::Channel.new(@amqp) do |ch|
+      ch.on_error do |_, info|
+        if info.reply_code == 404
+          # our request failed because it does not exist
+          k.call(false)
+        else
+          # failed for unknown reason
+          default_error_handler(ch, info)
+        end
+      end
+      check.call(ch) do |result|
+        ch.close do
+          k.call(result)
+        end
+      end
+    end
+  end
+
+  def install_default_error_handler
+    @channel.on_error(&method(:default_error_handler))
+  end
+
+  def default_error_handler(_ch, info)
+    @amqp.close do
+      raise "An error occurred: #{info.reply_code} - #{info.reply_text}"
+    end
+  end
+
+  def write(msg)
+    WireSerializer.new.write(msg)
+  end
+
+  def read(bytes)
+    WireSerializer.new.read(bytes)
+  end
+
+  def with_source(source_name, &k)
+    with_source_no_cache(@channel, source_name, Mercury.source_opts) do |exchange|
+      k.call(exchange)
+    end
+  end
+
+  def with_source_no_cache(channel, source_name, opts, &k)
+    channel.topic(source_name, opts) do |*args|
+      k.call(*args)
+    end
+  end
+
+  def with_work_queue(worker_group, source_exchange, tag_filter, &k)
+    bind_queue(source_exchange, worker_group, tag_filter, work_queue_opts, &k)
+  end
+
+  def self.source_opts
+    { durable: true, auto_delete: false }
+  end
+
+  def work_queue_opts
+    { durable: true, auto_delete: false }
+  end
+
+  def with_listener_queue(source_exchange, tag_filter, &k)
+    bind_queue(source_exchange, '', tag_filter, exclusive: true, auto_delete: true, durable: false, &k)
+  end
+
+  def bind_queue(exchange, queue_name, tag_filter, opts, &k)
+    queue = @channel.queue(queue_name, opts)
+    queue.bind(exchange, routing_key: tag_filter) do
+      k.call(queue)
+    end
+  end
+
+end

data/lib/mercury/monadic.rb

@@ -0,0 +1,41 @@
+require 'mercury'
+require 'mercury/cps'
+
+class Mercury
+  class Monadic
+
+    def self.open(**kws)
+      Cps.new do |&k|
+        Mercury.open(**kws) do |m|
+          k.call(new(m))
+        end
+      end
+    end
+
+    def self.wrap(method_name)
+      define_method(method_name) do |*args, **kws, &block|
+        Cps.new do |&k|
+          if @mercury.method(method_name).parameters.map(&:first).include?(:key)
+            @mercury.send(method_name, *[*args, *block], **kws, &k)
+          else
+            @mercury.send(method_name, *[*args, *block], &k)
+          end
+        end
+      end
+    end
+
+    wrap(:publish)
+    wrap(:start_listener)
+    wrap(:start_worker)
+    wrap(:delete_source)
+    wrap(:delete_work_queue)
+    wrap(:source_exists?)
+    wrap(:queue_exists?)
+    wrap(:close)
+
+    private
+    def initialize(mercury)
+      @mercury = mercury
+    end
+  end
+end

data/lib/mercury/received_message.rb

@@ -0,0 +1,30 @@
+class Mercury
+  class ReceivedMessage
+    attr_reader :content, :metadata
+
+    def initialize(content, metadata, is_ackable: false)
+      @content = content
+      @metadata = metadata
+      @is_ackable = is_ackable
+    end
+
+    def tag
+      metadata.routing_key
+    end
+
+    def ack
+      @is_ackable or raise 'This message is not ackable'
+      metadata.ack
+    end
+
+    def reject
+      @is_ackable or raise 'This message is not rejectable'
+      metadata.reject(requeue: false)
+    end
+
+    def nack
+      @is_ackable or raise 'This message is not nackable'
+      metadata.reject(requeue: true)
+    end
+  end
+end

data/lib/mercury/sync.rb

@@ -0,0 +1,20 @@
+require 'mercury/mercury'
+require 'bunny'
+
+class Mercury
+  class Sync
+    class << self
+      def publish(source_name, msg, tag: '', amqp_opts: {})
+        conn = Bunny.new(amqp_opts)
+        conn.start
+        ch = conn.create_channel
+        ch.confirm_select
+        ex = ch.topic(source_name, Mercury.source_opts)
+        ex.publish(WireSerializer.new.write(msg), **Mercury.publish_opts(tag))
+        ch.wait_for_confirms or raise 'failed to confirm publication'
+      ensure
+        conn.close
+      end
+    end
+  end
+end

data/lib/mercury/test_utils.rb

@@ -0,0 +1,59 @@
+require 'eventmachine'
+require 'mercury/cps'
+require 'mercury/monadic'
+
+class Mercury
+  module TestUtils
+    def em
+      EM.run do
+        EM.add_timer(in_debug_mode? ? 999999 : 3) { raise 'EM spec timed out' }
+        yield
+      end
+    end
+
+    def in_debug_mode?
+      ENV['RUBYLIB'] =~ /ruby-debug-ide/ # http://stackoverflow.com/questions/22039807/determine-if-a-program-is-running-in-debug-mode
+    end
+
+    def done
+      EM.stop
+    end
+
+    def em_wait_until(pred, &k)
+      try_again = proc do
+        if pred.call
+          k.call
+        else
+          EM.add_timer(1.0 / 50, try_again)
+        end
+      end
+      try_again.call
+    end
+
+    def wait_until(&pred)
+      cps do |&k|
+        em_wait_until(pred, &k)
+      end
+    end
+
+    def wait_for(seconds)
+      cps do |&k|
+        EM.add_timer(seconds, &k)
+      end
+    end
+
+    def delete_sources_and_queues_cps(source_names, queue_names)
+      # We must create a new mercury. The AMQP gem doesn't let you redeclare
+      # a construct with the same instance you deleted it with.
+      Mercury::Monadic.open.and_then do |m|
+        Cps.inject(amq_filter(source_names)) { |s| m.delete_source(s) }.
+          inject(amq_filter(queue_names)) { |q| m.delete_work_queue(q) }.
+          and_then { m.close }
+      end
+    end
+
+    def amq_filter(xs)
+      xs.reject{|x| x.start_with?('amq.')}
+    end
+  end
+end

data/lib/mercury/utils.rb

@@ -0,0 +1,9 @@
+class Utils
+  def self.unsplat(args)
+    if args.size == 1 and args.first.is_a?(Array)
+      args.first
+    else
+      args
+    end
+  end
+end

data/lib/mercury/version.rb

@@ -0,0 +1,3 @@
+class Mercury
+  VERSION = '0.1.0'
+end

data/lib/mercury/wire_serializer.rb

@@ -0,0 +1,54 @@
+require 'oj'
+
+class Mercury
+  class WireSerializer
+    # TODO: DRY with hyperion once we know more
+
+    def write(struct_or_hash)
+      write_json(struct_or_hash)
+    end
+
+    def read(bytes)
+      read_json(bytes)
+    end
+
+    private
+
+    def write_json(obj)
+      if obj.is_a?(String)
+        obj
+      else
+        Oj.dump(hashify(obj), oj_options)
+      end
+    end
+
+    def read_json(bytes)
+      begin
+        Oj.compat_load(bytes, oj_options)
+      rescue Oj::ParseError => e
+        bytes
+      end
+    end
+
+    def oj_options
+      {
+        mode: :compat,
+        time_format: :xmlschema,  # xmlschema == iso8601
+        use_to_json: false,
+        second_precision: 3
+      }
+    end
+
+    def hashify(x)
+      case x
+      when Hash
+        x
+      when Struct
+        x.to_h
+      else
+        raise "Could not convert to hash: #{x.inspect}"
+      end
+    end
+
+  end
+end

data/lib/mercury.rb

@@ -0,0 +1 @@
+require 'mercury/mercury'

data/mercury_amqp.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mercury/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'mercury_amqp'
+  spec.version       = Mercury::VERSION
+  spec.authors       = ['Peter Winton']
+  spec.email         = ['wintonpc@gmail.com']
+  spec.summary       = 'AMQP-backed messaging layer'
+  spec.description   = 'Abstracts common patterns used with AMQP'
+  spec.homepage      = 'https://github.com/wintonpc/mercury_amqp'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'yard'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'json_spec'
+  spec.add_development_dependency 'evented-spec'
+  spec.add_development_dependency 'rspec_junit_formatter'
+
+  spec.add_runtime_dependency 'oj', '~> 2.12'
+  spec.add_runtime_dependency 'amqp', '~> 1.5'
+  spec.add_runtime_dependency 'bunny', '~> 2.1'
+  spec.add_runtime_dependency 'binding_of_caller', '~> 0.7'
+end

data/spec/lib/mercury/cps_spec.rb

@@ -0,0 +1,225 @@
+require 'spec_helper'
+require 'mercury/cps'
+
+describe Cps do
+  include Cps::Methods
+
+  let!(:a) { 123 }
+
+  describe '::lift' do
+    it 'CPS-transforms a non-CPS proc' do
+      expect(Cps.lift{rand}.run).to be_a Numeric
+    end
+  end
+
+  describe '::run' do
+    it 'passes the value to the continuation' do
+      expect{|b| lift{a}.run(&b)}.to yield_with_args(a)
+    end
+    it 'returns the return value of the continuation' do
+      expect(lift{a}.run{456}).to eql 456
+    end
+    it 'feeds its arguments into the Cps' do
+      expect{|b| Cps.identity.run(a, &b)}.to yield_with_args(a)
+    end
+  end
+
+  describe '#and_then' do
+    it 'composes two Cps instances' do
+      expect(lift{a}.and_then{|x| to_string(x)}.run).to eql a.to_s
+    end
+  end
+
+  describe '#and_lift' do
+    it 'composes a Cps instance with a normal proc' do
+      expect(lift{a}.and_lift{|x| x.to_s}.run).to eql a.to_s
+    end
+  end
+
+  describe '::concurrently' do
+    it 'composes Cps instances concurrently' do
+      em do
+        actions = []
+        started1 = false
+        finished2 = false
+
+        task1 = proc do |n, &k|
+          actions << 'start1'
+          started1 = true
+          em_wait_until(proc{finished2}) do
+            actions << 'finish1'
+            k.call(n.to_s)
+          end
+        end
+        task2 = proc do |n, &k|
+          em_wait_until(proc{started1}) do
+            actions << 'start2'
+            actions << 'finish2'
+            finished2 = true
+            k.call(-1 * n)
+          end
+        end
+
+        result = nil
+        Cps.concurrently(Cps.new(&task1), Cps.new(&task2)).run(42) { |r| result = r }
+        em_wait_until(proc{result}) do
+          expect(result).to eql [['42'], [-42]]
+          expect(actions).to eql %w(start1 start2 finish2 finish1)
+          done
+        end
+      end
+    end
+  end
+
+  describe '::seq' do
+    it 'binds a sequence of monadic functions' do
+      result = Cps.seq do |th|
+        th.en { to_string(a) }
+        th.en { |v| twice(v) }
+        th.en { |v| reverse(v) }
+      end.run
+      expect(result).to eql '321321'
+    end
+    it 'can be chained' do
+      result = Cps.seq do |th|
+        th.en { to_string(a) }
+        th.en { |v| twice(v) }
+        th.en do |v|
+          Cps.seq do |th|
+            th.en { reverse(v)  }
+            th.en { |v| surround(v) }
+          end
+        end
+      end.run
+      expect(result).to eql '*321321*'
+    end
+  end
+
+  describe '::seql' do
+    it 'binds a sequence of monadic functions' do
+      result = Cps.seql do
+        let(:v)  { to_string(a) }
+        let(:v)  { twice(v) }
+        and_then { reverse(v) }
+      end.run
+      expect(result).to eql '321321'
+    end
+    it 'can be chained' do
+      result = Cps.seql do
+        let(:v) { to_string(a) }
+        let(:v) { twice(v) }
+        and_then do
+          Cps.seql do
+            let(:v) { reverse(v) }
+            and_then { surround(v) }
+          end
+        end
+      end.run
+      expect(result).to eql '*321321*'
+    end
+    it 'can be used like seq' do
+      result = Cps.seql do
+        and_then { to_string(a) }
+        and_then { |v| twice(v) }
+        and_then { |v| reverse(v) }
+      end.run
+      expect(result).to eql '321321'
+    end
+    it 'passes a bound value to the next function' do
+      result = Cps.seql do
+        let(:v) { to_string(a) }
+        and_then { |x| lift{x} }
+      end.run
+      expect(result).to eql '123'
+    end
+    it 'block can access outer methods' do
+      Cps.seql do
+        expect(foo).to eql 'foo'
+      end
+    end
+    def foo
+      'foo'
+    end
+    it 'block can access outer variables' do
+      bar = 'bar'
+      Cps.seql do
+        expect(bar).to eql 'bar'
+      end
+    end
+    it 'block cannot access outer instance variables' do
+      @baz = 'baz'
+      Cps.seql do
+        expect(@baz).to eql nil
+      end
+    end
+    it 'block can access outer constants' do
+      Cps.seql do
+        expect(FLIP).to eql 'flip'
+      end
+    end
+    FLIP = 'flip'
+  end
+
+  describe '::identity' do
+    it 'passes its arguments to the continuation' do
+      expect{|b| Cps.identity.run(a, &b)}.to yield_with_args(a)
+    end
+  end
+
+  describe '#inject' do
+    it 'creates a Cps chain given an array and a transformation function' do
+      chain = lift{'entity'}.inject(['-ize', '-er']) do |suffix, v|
+        lift { v + suffix }
+      end
+      expect(chain.run).to eql 'entity-ize-er'
+    end
+  end
+
+  # verify Cps obeys the monad laws (https://wiki.haskell.org/Monad_laws)
+  # unit   ::= Cps::lift
+  # bind   ::= Cps#and_then
+
+  it 'obeys the left identity law' do
+    # return a >>= f  ===  f a
+    expect_identical(lift{a}.and_then(&method(:to_string)),
+                     to_string(a),
+                     '123')
+  end
+
+  it 'obeys the right identity law' do
+    # m >>= return  === m
+    m = lift{123}
+    expect_identical(m.and_then{|x| lift{x}},
+                     m,
+                     123)
+  end
+
+  it 'obeys the associativity law' do
+    # (m >>= f) >>= g  ===  m >>= (\x -> f x >>= g)
+    m = lift{123}
+    expect_identical((m.and_then(&method(:to_string))).and_then(&method(:twice)),
+                     m.and_then {|x| to_string(x).and_then(&method(:twice)) },
+                     '123123')
+  end
+
+  def expect_identical(lhs, rhs, expected_value)
+    expect{|b| lhs.run(&b)}.to yield_with_args(expected_value)
+    expect{|b| rhs.run(&b)}.to yield_with_args(expected_value)
+  end
+
+  def to_string(x)
+    lift { x.to_s }
+  end
+
+  def twice(x)
+    lift { x * 2 }
+  end
+
+  def reverse(x)
+    lift { x.reverse }
+  end
+
+  def surround(x)
+    lift { "*#{x}*" }
+  end
+end