govuk_message_queue_consumer 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c236b54beb0a0440047c111caf390c175b9a6a67
+  data.tar.gz: 6064d29e67a5156cf20d111d8704045cb8816fac
+SHA512:
+  metadata.gz: bb3b285c4210e1cc9bd0c42395ab875391aaa1a94d795bcd1a53d005ddcdaea852bf3f0e59c4f846f63abe5a7d74961d6eb5c20d570e2ee4fdc56246c7afade8
+  data.tar.gz: b62511ec795ad7dcda904577f4f3ab2ba2943fbc27a2b2ba4be6c5c292f5f18c146c92d1390022aa3ba9d89096d7bdca6df2da7c198d2b6495ef365a968409d8

data/README.md

@@ -0,0 +1,95 @@
+# GOV.UK Message queue consumer
+
+Standardise the way GOV.UK consumes messages from RabbitMQ.
+
+
+## Nomenclature
+
+- **Message queue**: a method of providing asynchronous interprocess
+  communication
+
+
+## Technical documentation
+
+This is a ruby gem that deals with the boiler plate code of connecting,
+subscribing, etc, to [RabbitMQ](https://www.rabbitmq.com/).
+
+The user of this gem is left the task of supplying their rabbitmq infrastructure
+configuration and an instance of a class that processes messages.
+
+The message format received by the message processor is found in
+`lib/govuk_message_queue_consumer/message.rb`
+
+### Dependencies
+
+- **bunny**: to interact with RabbitMQ
+- **activesupport**: use `with_indifferent_access` for Bunny
+
+
+### Running the application
+
+We recommend creating a rake task like the following example:
+
+```ruby
+namespace :message_queue do
+  desc "Run worker to consume messages from rabbitmq"
+  task consumer: :environment do
+    config = get_rabbitmq_configuration_hash
+    # ^ eg YAML.load_file(Rails.root.join('config', 'rabbitmq.yml'))[Rails.env]
+    GovukMessageQueueConsumer::Consumer.new(config, MyProcessor.new).run
+  end
+end
+```
+
+`govuk_message_queue_consumer` expects configuration and a processor to be supplied:
+
+```ruby
+# example configuration. Could be stored in YAML if preferred
+config = {
+    host: 'localhost',
+    port: 5672,
+    user: rabbitmq_user,
+    pass: rabbitmq_pass,
+    recover_from_connection_close: true,
+    exchange: my_exchange,
+    queue: my_queue,
+}
+
+# example message processor
+class MyProcessor
+  def process(message)
+    message.ack
+  end
+end
+```
+
+#### Testing your processor
+
+This gem provides a test helper for your processor.
+
+```ruby
+require 'test_helper'
+require 'govuk_message_queue_consumer/test_helpers'
+
+describe MyProcessor do
+  it_behaves_like "a message queue processor"
+end
+```
+
+This will verify that your processor class implements the correct methods. You should add your own tests to verify its behaviour.
+
+### Running the test suite
+
+```bash
+bundle exec rake spec
+```
+
+
+## Licence
+
+[MIT License](LICENCE)
+
+
+## Versioning policy
+
+[Semantic versioning](http://semver.org/spec/v2.0.0.html)

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+# Load local tasks
+Dir['tasks/**/*.rake'].each { |file| load file }
+
+task(:default).clear
+task :default => [:spec]

data/lib/govuk_message_queue_consumer.rb

@@ -0,0 +1,4 @@
+require 'govuk_message_queue_consumer/version'
+require 'govuk_message_queue_consumer/heartbeat_processor'
+require 'govuk_message_queue_consumer/message'
+require 'govuk_message_queue_consumer/consumer'

data/lib/govuk_message_queue_consumer/consumer.rb

@@ -0,0 +1,44 @@
+require 'active_support/core_ext/hash/indifferent_access'
+require 'bunny'
+
+module GovukMessageQueueConsumer
+  class Consumer
+    def initialize(config, processor)
+      @processor = HeartbeatProcessor.new(processor)
+
+      @config = config.with_indifferent_access
+      @queue_name = @config.fetch(:queue)
+      @bindings = { @config.fetch(:exchange) => "#" }
+      @connection = Bunny.new(@config[:connection].symbolize_keys)
+      @connection.start
+    end
+
+    def run
+      queue.subscribe(:block => true, :manual_ack => true) do |delivery_info, headers, payload|
+        begin
+          @processor.process(Message.new(delivery_info, headers, payload))
+        rescue Exception => e
+          $stderr.puts "rabbitmq_consumer: aborting due to unhandled exception in processor #{e.class}: #{e.message}"
+          exit(1) # ensure rabbitmq requeues outstanding messages
+        end
+      end
+    end
+
+    private
+
+    def queue
+      @queue ||= setup_queue
+    end
+
+    def setup_queue
+      @channel = @connection.create_channel
+      @channel.prefetch(1) # only one unacked message at a time
+      queue = @channel.queue(@queue_name, :durable => true)
+      @bindings.each do |exchange_name, routing_key|
+        exchange = @channel.topic(exchange_name, :passive => true)
+        queue.bind(exchange, :routing_key => routing_key)
+      end
+      queue
+    end
+  end
+end

data/lib/govuk_message_queue_consumer/heartbeat_processor.rb

@@ -0,0 +1,16 @@
+module GovukMessageQueueConsumer
+  class HeartbeatProcessor
+    def initialize(next_processor)
+      @next_processor = next_processor
+    end
+
+    def process(message)
+      # Ignore heartbeat messages
+      if message.headers.content_type == "application/x-heartbeat"
+        message.ack
+      else
+        @next_processor.process(message)
+      end
+    end
+  end
+end

data/lib/govuk_message_queue_consumer/message.rb

@@ -0,0 +1,30 @@
+require 'json'
+
+module GovukMessageQueueConsumer
+  # Client code will receive an instance of this
+  class Message
+    def initialize(delivery_info, headers, payload)
+      @delivery_info = delivery_info
+      @headers = headers
+      @body = payload
+    end
+
+    attr_reader :delivery_info, :headers, :body
+
+    def body_data
+      @body_data ||= JSON.parse(@body)
+    end
+
+    def ack
+      @delivery_info.channel.ack(@delivery_info.delivery_tag)
+    end
+
+    def retry
+      @delivery_info.channel.reject(@delivery_info.delivery_tag, true)
+    end
+
+    def discard
+      @delivery_info.channel.reject(@delivery_info.delivery_tag, false)
+    end
+  end
+end

data/lib/govuk_message_queue_consumer/test_helpers.rb

@@ -0,0 +1 @@
+require 'govuk_message_queue_consumer/test_helpers/shared_examples'

data/lib/govuk_message_queue_consumer/test_helpers/shared_examples.rb

@@ -0,0 +1,9 @@
+RSpec.shared_examples "a message queue processor" do
+  it "implements #process" do
+    expect(subject).to respond_to(:process)
+  end
+
+  it "accepts 1 argument for #process" do
+    expect(subject.method(:process).arity).to eq(1)
+  end
+end

data/lib/govuk_message_queue_consumer/version.rb

@@ -0,0 +1,3 @@
+module GovukMessageQueueConsumer
+  VERSION = '1.0.0'
+end

data/spec/consumer_spec.rb

@@ -0,0 +1,48 @@
+require_relative 'spec_helper'
+
+describe Consumer do
+
+  let(:queue) { instance_double('Bunny::Queue', bind: nil, subscribe: message_values) }
+  let(:channel) { instance_double('Bunny::Channel', queue: queue, prefetch: nil, topic: nil) }
+  let(:rabbitmq_connecton) { instance_double("Bunny::Session", start: nil, create_channel: channel) }
+  let(:client_processor) { instance_double('Client::Processor') }
+
+  before do
+    allow(Bunny).to receive(:new).and_return(rabbitmq_connecton)
+  end
+
+  describe "constructing an instance" do
+    let(:rabbitmq_connecton) { instance_double("Bunny::Session", start: nil, create_channel: channel) }
+    let(:client_processor) { instance_double('Client::Processor') }
+
+    it "passes the client processor to the Heartbeat Processor" do
+      expect(HeartbeatProcessor).to receive(:new).with(client_processor)
+
+      Consumer.new(rabbitmq_config, client_processor)
+    end
+
+    it "connects to rabbitmq" do
+      expected_options = rabbitmq_config['connection'].symbolize_keys # Bunny requires the keys to be symbols
+      expect(Bunny).to receive(:new).with(expected_options).and_return(rabbitmq_connecton)
+      expect(rabbitmq_connecton).to receive(:start)
+
+      Consumer.new(rabbitmq_config, client_processor)
+    end
+  end
+
+  describe "running the consumer" do
+    it "binds the queue" do
+      expect(queue).to receive(:bind)
+
+      Consumer.new(rabbitmq_config, client_processor).run
+    end
+
+    it "calls the heartbeat processor when subscribing to messages" do
+      expect(queue).to receive(:subscribe).and_yield(*message_values)
+      expect(Message).to receive(:new).with(*message_values)
+      expect_any_instance_of(HeartbeatProcessor).to receive(:process)
+
+      Consumer.new(rabbitmq_config, client_processor).run
+    end
+  end
+end

data/spec/heartbeat_processor_spec.rb

@@ -0,0 +1,43 @@
+require_relative 'spec_helper'
+
+describe HeartbeatProcessor do
+  let(:heartbeat_headers) { instance_double("Heartbeat Headers", :content_type => "application/x-heartbeat") }
+  let(:heartbeat_message) { instance_double("Heartbeat Message", :headers => heartbeat_headers, :ack => nil) }
+  let(:standard_headers) { instance_double("Standard Headers", :content_type => nil) }
+  let(:standard_message) { instance_double("Standard Message", :headers => standard_headers, :ack => nil) }
+
+  let(:next_processor) { instance_double("Client::Processor") }
+
+  subject {
+    HeartbeatProcessor.new(next_processor)
+  }
+
+  context "for a heartbeat message" do
+    it "doesn't call the next processor" do
+      expect(next_processor).not_to receive(:process)
+
+      subject.process(heartbeat_message)
+    end
+
+    it "acks the message" do
+      expect(heartbeat_message).to receive(:ack)
+
+      subject.process(heartbeat_message)
+    end
+  end
+
+  context "for a content message" do
+    it "calls the next processor" do
+      expect(next_processor).to receive(:process).with(standard_message)
+
+      subject.process(standard_message)
+    end
+
+    it "doesn't ack the message" do
+      expect(standard_message).not_to receive(:ack)
+      expect(next_processor).to receive(:process)
+
+      subject.process(standard_message)
+    end
+  end
+end

data/spec/message_spec.rb

@@ -0,0 +1,28 @@
+require_relative 'spec_helper'
+
+describe Message do
+  let(:mock_channel) { instance_double("Channel") }
+  let(:delivery_info) { instance_double("DeliveryInfo", :channel => mock_channel, :delivery_tag => "a_tag") }
+  let(:headers) { instance_double("Headers") }
+  let(:body) { {"foo" => "bar"}.to_json }
+  let(:message) { Message.new(delivery_info, headers, body) }
+
+  it "json decodes the body" do
+    expect(message.body_data).to eq("foo" => "bar")
+  end
+
+  it "ack sends an ack to the channel" do
+    expect(mock_channel).to receive(:ack).with("a_tag")
+    message.ack
+  end
+
+  it "retry sends a reject to the channel with requeue set" do
+    expect(mock_channel).to receive(:reject).with("a_tag", true)
+    message.retry
+  end
+
+  it "reject sends a reject to the channel without requeue set" do
+    expect(mock_channel).to receive(:reject).with("a_tag", false)
+    message.discard
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+require_relative '../lib/govuk_message_queue_consumer'
+
+include GovukMessageQueueConsumer
+
+module TestHelpers
+  def rabbitmq_config
+    {
+      "connection" => {
+        "hosts" => ["rabbitmq1.example.com", "rabbitmq2.example.com"],
+        "port" => 5672,
+        "vhost" => "/",
+        "user" => "a_user",
+        "pass" => "super secret",
+        "recover_from_connection_close" => true,
+      },
+      "queue" => "content_register",
+      "exchange" => "published_documents",
+    }
+  end
+
+  def message_values
+    [:delivery_info1, :headers1, "message1_body"]
+  end
+end
+
+RSpec.configure do |c|
+  c.include TestHelpers
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: govuk_message_queue_consumer
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- GOV.UK Dev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: gem_publisher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.4.2
+description: Avoid writing boilerplate code in order to consume messages from an AMQP
+  message queue. Plug in queue configuration, and how to process each message.
+email:
+- govuk-dev@digital.cabinet-office.gov.uk
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/govuk_message_queue_consumer.rb
+- lib/govuk_message_queue_consumer/consumer.rb
+- lib/govuk_message_queue_consumer/heartbeat_processor.rb
+- lib/govuk_message_queue_consumer/message.rb
+- lib/govuk_message_queue_consumer/test_helpers.rb
+- lib/govuk_message_queue_consumer/test_helpers/shared_examples.rb
+- lib/govuk_message_queue_consumer/version.rb
+- spec/consumer_spec.rb
+- spec/heartbeat_processor_spec.rb
+- spec/message_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/alphagov/govuk_message_queue_consumer
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: AMQP message queue consumption with GOV.UK conventions
+test_files:
+- spec/heartbeat_processor_spec.rb
+- spec/spec_helper.rb
+- spec/message_spec.rb
+- spec/consumer_spec.rb