govuk_message_queue_consumer 2.1.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ea5c6560ba396d9f32d22371cd8f94240bdfbcd
-  data.tar.gz: fdc543d6ee7d1290e3df4db3628f497aa3151e37
+  metadata.gz: d4a095426507f671b2ed962c5fcd7fd2a8b8562a
+  data.tar.gz: 7b5611a301ea24405a3b2fc83e18e436480f6123
 SHA512:
-  metadata.gz: 760eb4893ce343d4e5dc439bc05a80ef49ceb8fc9c4538395079f6644b05ad555e435e32415be004bb0116dd1d9ec8b54bedf62819bf8eceaf7ac1caa3ce8451
-  data.tar.gz: 98f3ff7cb41cb8f20233114121e53967d180462ecd1118c07d0abf409456ca3c663127c6e2692f8a7cbadaa250bf2e36ba65e4865a45edf160a3d6f6d9e71374
+  metadata.gz: cf97eb5b97d6a202224689afc7d4b8a79e21bbbdba16337d54f2ad52b5404a0bae7fea90dde0865230fecb40419cab9555b027db06d561089a1862321c737789
+  data.tar.gz: 6dd646bfd603d17f827b91ff5cf19c48bf24c0f732bc9bfc6e7f8ade85cf5ea3954f0f002c12e3b901b2f288893de3b786e03d0f26cabc9f9bb753f129075871

data/CHANGELOG.md

@@ -0,0 +1,42 @@
+# 3.0.0
+
+- Updated README to conform changes on [PR #32](https://github.com/alphagov/govuk_message_queue_consumer/pull/32)
+- Remove `exchange_name` parameter [PR #34](https://github.com/alphagov/govuk_message_queue_consumer/pull/34)
+- Don't build test files in the gem [PR #33](https://github.com/alphagov/govuk_message_queue_consumer/pull/33)
+- Prevent consumer from creating rabbitmq queues or bindings [PR #32](https://github.com/alphagov/govuk_message_queue_consumer/pull/32)
+
+# 2.1.0
+
+- Add support for sending stats to Statsd
+
+# 2.0.1
+
+* Add support for Airbrake.
+
+# 2.0.0
+
+- README updates making it clearer how to use the gem
+- Use environment variables for RabbitMQ configuration
+- Use keyword arguments for the `Consumer` setup
+- Add rspec shared examples for testing a message processor
+- Add `GovukMessageQueueConsumer::MockMessage` for easy testing
+- Add `GovukMessageQueueConsumer::JSONProcessor` as an intermediate processor for JSON payloads
+- Add Airbrake notification for gem errors
+- Remove active_support dependency
+
+# 1.0.0
+
+- Rename the gem to `govuk_message_queue_consumer`
+- Make test helpers easier to use
+- Readme improvements
+- Initial release!
+
+# 0.9.1
+
+Bug fix:
+- relax the ruby version as it's causing problems in projects trying to use it
+
+
+# 0.9.0
+
+- Initial implementation of the gem

data/LICENCE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+Copyright (C) 2015 Crown copyright (Government Digital Service)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -65,8 +65,7 @@ namespace :message_queue do
   task consumer: :environment do
     GovukMessageQueueConsumer::Consumer.new(
       queue_name: "some-queue",
-      exchange_name: "some-exchange",
-      processor: MyProcessor.new
+      processor: MyProcessor.new,
     ).run
   end
 end

data/lib/govuk_message_queue_consumer/consumer.rb

@@ -1,6 +1,3 @@
-require 'bunny'
-require_relative 'null_statsd'
-
 module GovukMessageQueueConsumer
   class Consumer
     # Only fetch one message at a time on the channel.
@@ -14,20 +11,23 @@ module GovukMessageQueueConsumer
 
     # Create a new consumer
     #
-    # @param queue_name [String] Your queue name. This is specific to your application.
-    # @param exchange_name [String] Name of the exchange to bind to, for example `published_documents`
+    # @param queue_name [String] Your queue name. This is specific to your application,
+    #                            and should already exist and have a binding via puppet
     # @param processor [Object] An object that responds to `process`
-    # @param routing_key [String] The RabbitMQ routing key to bind the queue to
+    # @param rabbitmq_connection [Object] A Bunny connection object derived from `Bunny.new`
     # @param statsd_client [Statsd] An instance of the Statsd class
-    def initialize(queue_name:, exchange_name:, processor:, routing_key: '#', statsd_client: NullStatsd.new)
+    # @param logger [Object] A Logger object for emitting errors (to stderr by default)
+    def initialize(queue_name:, processor:, rabbitmq_connection: Consumer.default_connection_from_env, statsd_client: NullStatsd.new, logger: Logger.new(STDERR))
       @queue_name = queue_name
-      @exchange_name = exchange_name
       @processor = processor
-      @routing_key = routing_key
+      @rabbitmq_connection = rabbitmq_connection
       @statsd_client = statsd_client
+      @logger = logger
     end
 
     def run
+      @rabbitmq_connection.start
+
       queue.subscribe(block: true, manual_ack: true) do |delivery_info, headers, payload|
         begin
           message = Message.new(payload, headers, delivery_info)
@@ -37,7 +37,7 @@ module GovukMessageQueueConsumer
         rescue Exception => e
           @statsd_client.increment("#{@queue_name}.uncaught_exception")
           Airbrake.notify_or_ignore(e) if defined?(Airbrake)
-          $stderr.puts "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
+          @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
           exit(1) # Ensure rabbitmq requeues outstanding messages
         end
       end
@@ -45,6 +45,11 @@ module GovukMessageQueueConsumer
 
   private
 
+    class NullStatsd
+      def increment(_key)
+      end
+    end
+
     def processor_chain
       @processor_chain ||= HeartbeatProcessor.new(JSONProcessor.new(@processor))
     end
@@ -52,26 +57,16 @@ module GovukMessageQueueConsumer
     def queue
       @queue ||= begin
         channel.prefetch(NUMBER_OF_MESSAGES_TO_PREFETCH)
-        queue = channel.queue(@queue_name, durable: true)
-        queue.bind(exchange, routing_key: @routing_key)
-        queue
+        channel.queue(@queue_name, no_declare: true)
       end
     end
 
-    def exchange
-      @exchange ||= channel.topic(@exchange_name, passive: true)
-    end
-
     def channel
-      @channel ||= connection.create_channel
+      @channel ||= @rabbitmq_connection.create_channel
     end
 
-    def connection
-      @connection ||= begin
-        new_connection = Bunny.new(RabbitMQConfig.new.from_environment)
-        new_connection.start
-        new_connection
-      end
+    def self.default_connection_from_env
+      Bunny.new(GovukMessageQueueConsumer::RabbitMQConfig.from_environment(ENV))
     end
   end
 end

data/lib/govuk_message_queue_consumer/message.rb

@@ -1,5 +1,3 @@
-require 'json'
-
 module GovukMessageQueueConsumer
   # Client code will receive an instance of this
   class Message

data/lib/govuk_message_queue_consumer/rabbitmq_config.rb

@@ -1,29 +1,31 @@
-class RabbitMQConfig
-  class ConfigurationError < StandardError
-  end
+module GovukMessageQueueConsumer
+  module RabbitMQConfig
+    class ConfigurationError < StandardError
+    end
 
-  def from_environment
-    {
-      hosts: fetch("RABBITMQ_HOSTS").split(','),
-      vhost: fetch("RABBITMQ_VHOST"),
-      user: fetch("RABBITMQ_USER"),
-      pass: fetch("RABBITMQ_PASSWORD"),
-      recover_from_connection_close: true,
-    }
-  end
+    def self.from_environment(env)
+      {
+        hosts: fetch(env, "RABBITMQ_HOSTS").split(','),
+        vhost: fetch(env, "RABBITMQ_VHOST"),
+        user: fetch(env, "RABBITMQ_USER"),
+        pass: fetch(env, "RABBITMQ_PASSWORD"),
+        recover_from_connection_close: true,
+      }
+    end
 
-private
+  private
 
-  def fetch(variable_name)
-    ENV[variable_name] || raise_error(variable_name)
-  end
+    def self.fetch(env, variable_name)
+      env[variable_name] || raise_error(variable_name)
+    end
 
-  def raise_error(variable_name)
-    raise ConfigurationError, <<-err
-      The environment variable #{variable_name} is not set. If you are in test
-      mode, make sure you set the correct vars in your helpers. If you get this
-      error in development, make sure you run rails or rake with `govuk_setenv`
-      and puppet is up to date.
-    err
+    def self.raise_error(variable_name)
+      raise ConfigurationError, <<-err
+        The environment variable #{variable_name} is not set. If you are in test
+        mode, make sure you set the correct vars in your helpers. If you get this
+        error in development, make sure you run rails or rake with `govuk_setenv`
+        and puppet is up to date.
+      err
+    end
   end
 end

data/lib/govuk_message_queue_consumer/version.rb

@@ -1,3 +1,3 @@
 module GovukMessageQueueConsumer
-  VERSION = '2.1.0'
+  VERSION = '3.0.0'
 end

data/lib/govuk_message_queue_consumer.rb

@@ -1,3 +1,6 @@
+require 'bunny'
+require 'json'
+
 require 'govuk_message_queue_consumer/version'
 require 'govuk_message_queue_consumer/heartbeat_processor'
 require 'govuk_message_queue_consumer/json_processor'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: govuk_message_queue_consumer
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 3.0.0
 platform: ruby
 authors:
 - GOV.UK Dev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-06 00:00:00.000000000 Z
+date: 2016-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -88,28 +88,19 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- CHANGELOG.md
+- LICENCE
 - README.md
-- Rakefile
 - lib/govuk_message_queue_consumer.rb
 - lib/govuk_message_queue_consumer/consumer.rb
 - lib/govuk_message_queue_consumer/heartbeat_processor.rb
 - lib/govuk_message_queue_consumer/json_processor.rb
 - lib/govuk_message_queue_consumer/message.rb
-- lib/govuk_message_queue_consumer/null_statsd.rb
 - lib/govuk_message_queue_consumer/rabbitmq_config.rb
 - lib/govuk_message_queue_consumer/test_helpers.rb
 - lib/govuk_message_queue_consumer/test_helpers/mock_message.rb
 - lib/govuk_message_queue_consumer/test_helpers/shared_examples.rb
-- lib/govuk_message_queue_consumer/test_helpers/test_consumer.rb
 - lib/govuk_message_queue_consumer/version.rb
-- spec/consumer_spec.rb
-- spec/consumer_with_statsd_spec.rb
-- spec/heartbeat_processor_spec.rb
-- spec/json_processor_spec.rb
-- spec/message_spec.rb
-- spec/mock_message_spec.rb
-- spec/rabbitmq_config_spec.rb
-- spec/spec_helper.rb
 homepage: https://github.com/alphagov/govuk_message_queue_consumer
 licenses: []
 metadata: {}
@@ -133,13 +124,5 @@ rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: AMQP message queue consumption with GOV.UK conventions
-test_files:
-- spec/heartbeat_processor_spec.rb
-- spec/spec_helper.rb
-- spec/message_spec.rb
-- spec/consumer_spec.rb
-- spec/consumer_with_statsd_spec.rb
-- spec/mock_message_spec.rb
-- spec/json_processor_spec.rb
-- spec/rabbitmq_config_spec.rb
+test_files: []
 has_rdoc: 

data/Rakefile

@@ -1,8 +0,0 @@
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec)
-
-# Load local tasks
-Dir['tasks/**/*.rake'].each { |file| load file }
-
-task(:default).clear
-task :default => [:spec]

data/lib/govuk_message_queue_consumer/null_statsd.rb

@@ -1,4 +0,0 @@
-class NullStatsd
-  def increment(_key)
-  end
-end

data/lib/govuk_message_queue_consumer/test_helpers/test_consumer.rb

@@ -1,12 +0,0 @@
-module GovukMessageQueueConsumer
-  class TestConsumer < Consumer
-    def publish_message(payload, options)
-      exchange.publish(payload, options)
-    end
-
-    # call after integration tests finish
-    def delete_queue
-      queue.delete
-    end
-  end
-end

data/spec/consumer_spec.rb

@@ -1,36 +0,0 @@
-require_relative 'spec_helper'
-require_relative 'support/queue_helpers'
-
-describe Consumer do
-  include QueueHelpers
-
-  let(:client_processor) { instance_double('Client::Processor') }
-
-  describe "#run" do
-    it "binds the queue to the all-routing key" do
-      queue = create_stubbed_queue
-
-      expect(queue).to receive(:bind).with(nil, { routing_key: "#" })
-
-      Consumer.new(queue_name: "some-queue", exchange_name: "my-exchange", processor: client_processor).run
-    end
-
-    it "binds the queue to a custom routing key" do
-      queue = create_stubbed_queue
-
-      expect(queue).to receive(:bind).with(nil, { routing_key: "*.major" })
-
-      Consumer.new(queue_name: "some-queue", exchange_name: "my-exchange", processor: client_processor, routing_key: "*.major").run
-    end
-
-    it "calls the heartbeat processor when subscribing to messages" do
-      queue = create_stubbed_queue
-
-      expect(queue).to receive(:subscribe).and_yield(:delivery_info_object, :headers, "payload")
-
-      expect_any_instance_of(HeartbeatProcessor).to receive(:process).with(kind_of(Message))
-
-      Consumer.new(queue_name: "some-queue", exchange_name: "my-exchange", processor: client_processor).run
-    end
-  end
-end

data/spec/consumer_with_statsd_spec.rb

@@ -1,54 +0,0 @@
-require_relative 'spec_helper'
-require_relative 'support/queue_helpers'
-
-describe Consumer do
-  include QueueHelpers
-
-  describe "#run" do
-    it "increments the counters on the statsd client" do
-      statsd_client = StatsdClientMock.new
-      queue = create_stubbed_queue
-
-      expect(queue).to receive(:subscribe).and_yield(
-        double(:delivery_info, channel: double(:channel, reject: double), delivery_tag: double),
-        double(:headers, content_type: 'application/json'),
-        "message_payload"
-      )
-
-      Consumer.new(queue_name: "some-queue", exchange_name: "my-exchange", processor: double, statsd_client: statsd_client).run
-
-      expect(statsd_client.incremented_keys).to eql(['some-queue.started', 'some-queue.discarded'])
-    end
-
-    it "increments the uncaught_exception counter for uncaught exceptions" do
-      statsd_client = StatsdClientMock.new
-      queue = create_stubbed_queue
-
-      expect(queue).to receive(:subscribe).and_yield(
-        double(:delivery_info, channel: double(:channel, reject: double), delivery_tag: double),
-        double(:headers, content_type: 'application/json'),
-        {}.to_json
-      )
-
-      processor = double
-      expect(processor).to receive(:process).and_raise("An exception")
-
-      expect do
-        Consumer.new(queue_name: "some-queue", exchange_name: "my-exchange", processor: processor, statsd_client: statsd_client).run
-      end.to raise_error(SystemExit)
-
-      expect(statsd_client.incremented_keys).to eql(['some-queue.started', 'some-queue.uncaught_exception'])
-    end
-  end
-
-  class StatsdClientMock
-    attr_reader :incremented_keys
-    def initialize
-      @incremented_keys = []
-    end
-
-    def increment(key)
-      @incremented_keys << key
-    end
-  end
-end

data/spec/heartbeat_processor_spec.rb

@@ -1,43 +0,0 @@
-require_relative 'spec_helper'
-
-describe HeartbeatProcessor do
-  let(:heartbeat_headers) { instance_double("Heartbeat Headers", :content_type => "application/x-heartbeat") }
-  let(:heartbeat_message) { instance_double("Heartbeat Message", :headers => heartbeat_headers, :ack => nil) }
-  let(:standard_headers) { instance_double("Standard Headers", :content_type => nil) }
-  let(:standard_message) { instance_double("Standard Message", :headers => standard_headers, :ack => nil) }
-
-  let(:next_processor) { instance_double("Client::Processor") }
-
-  subject {
-    HeartbeatProcessor.new(next_processor)
-  }
-
-  context "for a heartbeat message" do
-    it "doesn't call the next processor" do
-      expect(next_processor).not_to receive(:process)
-
-      subject.process(heartbeat_message)
-    end
-
-    it "acks the message" do
-      expect(heartbeat_message).to receive(:ack)
-
-      subject.process(heartbeat_message)
-    end
-  end
-
-  context "for a content message" do
-    it "calls the next processor" do
-      expect(next_processor).to receive(:process).with(standard_message)
-
-      subject.process(standard_message)
-    end
-
-    it "doesn't ack the message" do
-      expect(standard_message).not_to receive(:ack)
-      expect(next_processor).to receive(:process)
-
-      subject.process(standard_message)
-    end
-  end
-end

data/spec/json_processor_spec.rb

@@ -1,33 +0,0 @@
-require_relative 'spec_helper'
-
-describe JSONProcessor do
-  describe "#process" do
-    it "parses the payload string" do
-      next_processor = double("next_processor", process: "ha")
-      message = MockMessage.new('{"some":"json"}', { content_type: "application/json" })
-
-      JSONProcessor.new(next_processor).process(message)
-
-      expect(message.payload).to eql({ "some" => "json" })
-      expect(next_processor).to have_received(:process)
-    end
-
-    it "discards messages with JSON errors" do
-      message = MockMessage.new('{"some" "json"}', { content_type: "application/json" })
-
-      JSONProcessor.new(double).process(message)
-
-      expect(message).to be_discarded
-    end
-
-    it "doesn't parse non-JSON message" do
-      next_processor = double("next_processor", process: "ha")
-      message = MockMessage.new('<SomeXML></SomeXML>', { content_type: "application/xml" })
-
-      JSONProcessor.new(next_processor).process(message)
-
-      expect(message.payload).to eql('<SomeXML></SomeXML>')
-      expect(next_processor).to have_received(:process)
-    end
-  end
-end

data/spec/message_spec.rb

@@ -1,32 +0,0 @@
-require_relative 'spec_helper'
-
-describe Message do
-  let(:mock_channel) { instance_double("Channel") }
-  let(:delivery_info) { instance_double("DeliveryInfo", :channel => mock_channel, :delivery_tag => "a_tag") }
-  let(:headers) { instance_double("Headers") }
-  let(:message) { Message.new({ "a" => "payload" }, headers, delivery_info) }
-
-  it "ack sends an ack to the channel" do
-    expect(mock_channel).to receive(:ack).with("a_tag")
-
-    message.ack
-
-    expect(message.status).to eql(:acked)
-  end
-
-  it "retry sends a reject to the channel with requeue set" do
-    expect(mock_channel).to receive(:reject).with("a_tag", true)
-
-    message.retry
-
-    expect(message.status).to eql(:retried)
-  end
-
-  it "reject sends a reject to the channel without requeue set" do
-    expect(mock_channel).to receive(:reject).with("a_tag", false)
-
-    message.discard
-
-    expect(message.status).to eql(:discarded)
-  end
-end

data/spec/mock_message_spec.rb

@@ -1,43 +0,0 @@
-require_relative 'spec_helper'
-require_relative '../lib/govuk_message_queue_consumer/test_helpers/mock_message'
-
-describe GovukMessageQueueConsumer::MockMessage do
-  describe '#methods' do
-    it "implements the same methods as Message" do
-      mock = MockMessage.new
-      real = Message.new(double, double, double)
-
-      expect(real.methods - mock.methods).to be_empty
-    end
-  end
-
-  describe '#ack' do
-    it "marks the message as acked" do
-      message = MockMessage.new
-
-      message.ack
-
-      expect(message).to be_acked
-    end
-  end
-
-  describe '#retry' do
-    it "marks the message as retried" do
-      message = MockMessage.new
-
-      message.retry
-
-      expect(message).to be_retried
-    end
-  end
-
-  describe '#discard' do
-    it "marks the message as discarded" do
-      message = MockMessage.new
-
-      message.discard
-
-      expect(message).to be_discarded
-    end
-  end
-end

data/spec/rabbitmq_config_spec.rb

@@ -1,26 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe RabbitMQConfig do
-  describe ".from_environment" do
-    it "connects to rabbitmq with the correct environment variables" do
-      ENV["RABBITMQ_HOSTS"] = "server-one,server-two"
-      ENV["RABBITMQ_VHOST"] = "/"
-      ENV["RABBITMQ_USER"] = "my_user"
-      ENV["RABBITMQ_PASSWORD"] = "my_pass"
-
-      expect(RabbitMQConfig.new.from_environment).to eql({
-        hosts: ["server-one", "server-two"],
-        vhost: "/",
-        user: "my_user",
-        pass: "my_pass",
-        recover_from_connection_close: true,
-      })
-    end
-
-    it "provides a friendly error message when a variable is missing" do
-      ENV["RABBITMQ_HOSTS"] = nil
-
-      expect { RabbitMQConfig.new.from_environment }.to raise_error(RabbitMQConfig::ConfigurationError)
-    end
-  end
-end

data/spec/spec_helper.rb

@@ -1,16 +0,0 @@
-require_relative '../lib/govuk_message_queue_consumer'
-
-include GovukMessageQueueConsumer
-
-module TestHelpers
-  def stub_environment_variables!
-    ENV["RABBITMQ_HOSTS"] ||= ""
-    ENV["RABBITMQ_VHOST"] ||= "/"
-    ENV["RABBITMQ_USER"] ||= "/"
-    ENV["RABBITMQ_PASSWORD"] ||= "/"
-  end
-end
-
-RSpec.configure do |c|
-  c.include TestHelpers
-end