govuk_message_queue_consumer 3.5.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a3ef3f89aa60ea51f6335665bf01b9ce01f5d313be4770fa72452f234fd626c
-  data.tar.gz: 2c3ba684115314f3329705c52e670d1055c9109e32e7d56ac2e59d9893ba9510
+  metadata.gz: 2ff831021fdec8b26b02d2a1b9c36d5b35499a617e6c43995bb1dea97710d62e
+  data.tar.gz: 68fb8c56450889fd41def5ce02fd520a6e5d4ead28f738688ebb26a351769e05
 SHA512:
-  metadata.gz: 42c2a962a238605044afaa5fb3f84c494324033bf2426317ad66d730595bfe26060811cc6c9c209cfe0017f60769446d41de1a234dd4042c9932a65531f60aab
-  data.tar.gz: 4bb19be3b6309721256772f0570f1a63095067d6f5ce3c1ba612c12706d96ad7a4626eba6794563d17016935648ebfd8232be8a9cbbc009f7b5ef92dcda3f02a
+  metadata.gz: ccd4008cf70e0a1edb4ff3e1712233518bed5321c7ed5531853a8336d596ecf06ba645a1b65b4071a0fcf7b77885f0c6ed4200f26eb2c8284e7f4d14834d18cb
+  data.tar.gz: 0dc0f62ca439285e225f0cf3b0261158ffc3444339c765937077759e4565134618488826b8547b255bdbc92d41a04c392154ff50b9919ede2a8405f1f26c9862

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 4.0.0
+
+- Breaking: remove batch consumer ([#73](https://github.com/alphagov/govuk_message_queue_consumer/pull/73))
+
 # 3.5.0
 
 - Stop reporting `SignalExceptions` to `GovukError`.

data/README.md

@@ -17,26 +17,11 @@ This gem is used by:
 - [Cache clearing service](https://github.com/alphagov/cache-clearing-service).
 - [Content Data API](https://github.com/alphagov/content-data-api).
 - [Email Alert Service](https://github.com/alphagov/email-alert-service/).
-- [Rummager](https://github.com/alphagov/rummager).
-
-## Nomenclature
-
-![A graph showing the message flow](docs/graph.png)
-
-- **Producer**: an application that sends messages RabbitMQ. On GOV.UK this could
-  be [publishing-api](https://github.com/alphagov/publishing-api).
-- **Message**: an object sent over RabbitMQ. It consists of a _payload_ and
-  _headers_. In the case of the publishing-api the payload is a
-  [content item](https://github.com/alphagov/govuk-content-schemas).
-- **Consumer**: the app that receives the messages and does something with them.
-  On GOV.UK, this is [email-alert-service](https://github.com/alphagov/email-alert-service).
-- **Exchange**: in RabbitMQ's model, producers send messages to an _exchange_.
-  Consumers can create a Queue that listens to the exchange, instead of
-  subscribing to the exchange directly. This is done so that the queue can buffer
-  any messages and we can make sure all messages get delivered to the consumer.
-- **Queue**: a queue listens to an exchange. In most cases the queue will listen
-  to all messages, but it's also possible to listen to a specific pattern.
-- **Processor**: the specific class that processes a message.
+- [Search API](https://github.com/alphagov/search-api).
+
+## Overview of RabbitMQ
+
+To see an overview of RabbitMQ and how we use it, see [here](https://docs.publishing.service.gov.uk/manual/rabbitmq.html#overview).
 
 ## Technical documentation
 

data/lib/govuk_message_queue_consumer/consumer.rb

@@ -10,6 +10,10 @@ module GovukMessageQueueConsumer
     # time to share the work evenly.
     NUMBER_OF_MESSAGES_TO_PREFETCH = 1
 
+    def self.default_connection_from_env
+      Bunny.new(GovukMessageQueueConsumer::RabbitMQConfig.from_environment(ENV))
+    end
+
     # Create a new consumer
     #
     # @param queue_name [String] Your queue name. This is specific to your application,
@@ -18,7 +22,7 @@ module GovukMessageQueueConsumer
     # @param rabbitmq_connection [Object] A Bunny connection object derived from `Bunny.new`
     # @param statsd_client [Statsd] An instance of the Statsd class
     # @param logger [Object] A Logger object for emitting errors (to stderr by default)
-    def initialize(queue_name:, processor:, rabbitmq_connection: Consumer.default_connection_from_env, statsd_client: NullStatsd.new, logger: Logger.new(STDERR))
+    def initialize(queue_name:, processor:, rabbitmq_connection: Consumer.default_connection_from_env, statsd_client: NullStatsd.new, logger: Logger.new($stderr))
       @queue_name = queue_name
       @processor = processor
       @rabbitmq_connection = rabbitmq_connection
@@ -29,33 +33,29 @@ module GovukMessageQueueConsumer
     def run(subscribe_opts: {})
       @rabbitmq_connection.start
 
-      subscribe_opts = { block: true, manual_ack: true}.merge(subscribe_opts)
+      subscribe_opts = { block: true, manual_ack: true }.merge(subscribe_opts)
       queue.subscribe(subscribe_opts) do |delivery_info, headers, payload|
-        begin
-          message = Message.new(payload, headers, delivery_info)
-          @statsd_client.increment("#{@queue_name}.started")
-          message_consumer.process(message)
-          @statsd_client.increment("#{@queue_name}.#{message.status}")
-        rescue SignalException => e
-          @logger.error "SignalException in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
-          exit(1) # Ensure rabbitmq requeues outstanding messages
-        rescue Exception => e
-          @statsd_client.increment("#{@queue_name}.uncaught_exception")
-          GovukError.notify(e) if defined?(GovukError)
-          @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
-          exit(1) # Ensure rabbitmq requeues outstanding messages
-        end
+        message = Message.new(payload, headers, delivery_info)
+        @statsd_client.increment("#{@queue_name}.started")
+        message_consumer.process(message)
+        @statsd_client.increment("#{@queue_name}.#{message.status}")
+      rescue SignalException => e
+        @logger.error "SignalException in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
+        exit(1) # Ensure rabbitmq requeues outstanding messages
+      rescue StandardError => e
+        @statsd_client.increment("#{@queue_name}.uncaught_exception")
+        GovukError.notify(e) if defined?(GovukError)
+        @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
+        exit(1) # Ensure rabbitmq requeues outstanding messages
       end
     end
 
   private
 
     class NullStatsd
-      def increment(_key)
-      end
+      def increment(_key); end
 
-      def count(_key, _volume)
-      end
+      def count(_key, _volume); end
     end
 
     def message_consumer
@@ -79,9 +79,5 @@ module GovukMessageQueueConsumer
     def channel
       @channel ||= @rabbitmq_connection.create_channel
     end
-
-    def self.default_connection_from_env
-      Bunny.new(GovukMessageQueueConsumer::RabbitMQConfig.from_environment(ENV))
-    end
   end
 end

data/lib/govuk_message_queue_consumer/message_consumer.rb

@@ -17,7 +17,7 @@ module GovukMessageQueueConsumer
 
     def handles_batches?(processor)
       case processor
-      when HeartbeatProcessor,JSONProcessor
+      when HeartbeatProcessor, JSONProcessor
         false
       else
         @handle_batches

data/lib/govuk_message_queue_consumer/rabbitmq_config.rb

@@ -5,7 +5,7 @@ module GovukMessageQueueConsumer
 
     def self.from_environment(env)
       {
-        hosts: fetch(env, "RABBITMQ_HOSTS").split(','),
+        hosts: fetch(env, "RABBITMQ_HOSTS").split(","),
         vhost: fetch(env, "RABBITMQ_VHOST"),
         user: fetch(env, "RABBITMQ_USER"),
         pass: fetch(env, "RABBITMQ_PASSWORD"),
@@ -13,19 +13,17 @@ module GovukMessageQueueConsumer
       }
     end
 
-  private
-
     def self.fetch(env, variable_name)
       env[variable_name] || raise_error(variable_name)
     end
 
     def self.raise_error(variable_name)
-      raise ConfigurationError, <<-err
+      raise ConfigurationError, <<-ERR
         The environment variable #{variable_name} is not set. If you are in test
         mode, make sure you set the correct vars in your helpers. If you get this
         error in development, make sure you run rails or rake with `govuk_setenv`
         and puppet is up to date.
-      err
+      ERR
     end
   end
 end

data/lib/govuk_message_queue_consumer/test_helpers/mock_message.rb

@@ -2,14 +2,12 @@ module GovukMessageQueueConsumer
   class MockMessage < Message
     attr_reader :acked, :retried, :discarded, :payload, :header, :delivery_info
 
-    alias :acked? :acked
-    alias :discarded? :discarded
-    alias :retried? :retried
+    alias_method :acked?, :acked
+    alias_method :discarded?, :discarded
+    alias_method :retried?, :retried
 
     def initialize(payload = {}, headers = {}, delivery_info = {})
-      @payload = payload
-      @headers = OpenStruct.new(headers)
-      @delivery_info = OpenStruct.new(delivery_info)
+      super(payload, OpenStruct.new(headers), OpenStruct.new(delivery_info))
     end
 
     def ack

data/lib/govuk_message_queue_consumer/test_helpers.rb

@@ -1,2 +1,2 @@
-require 'govuk_message_queue_consumer/test_helpers/shared_examples'
-require 'govuk_message_queue_consumer/test_helpers/mock_message'
+require "govuk_message_queue_consumer/test_helpers/shared_examples"
+require "govuk_message_queue_consumer/test_helpers/mock_message"

data/lib/govuk_message_queue_consumer/version.rb

@@ -1,3 +1,3 @@
 module GovukMessageQueueConsumer
-  VERSION = '3.5.0'
+  VERSION = "4.0.0".freeze
 end

data/lib/govuk_message_queue_consumer.rb

@@ -1,11 +1,10 @@
-require 'bunny'
-require 'json'
+require "bunny"
+require "json"
 
-require 'govuk_message_queue_consumer/version'
-require 'govuk_message_queue_consumer/heartbeat_processor'
-require 'govuk_message_queue_consumer/json_processor'
-require 'govuk_message_queue_consumer/message'
-require 'govuk_message_queue_consumer/message_consumer'
-require 'govuk_message_queue_consumer/consumer'
-require 'govuk_message_queue_consumer/batch_consumer'
-require 'govuk_message_queue_consumer/rabbitmq_config'
+require "govuk_message_queue_consumer/version"
+require "govuk_message_queue_consumer/heartbeat_processor"
+require "govuk_message_queue_consumer/json_processor"
+require "govuk_message_queue_consumer/message"
+require "govuk_message_queue_consumer/message_consumer"
+require "govuk_message_queue_consumer/consumer"
+require "govuk_message_queue_consumer/rabbitmq_config"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: govuk_message_queue_consumer
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 4.0.0
 platform: ruby
 authors:
 - GOV.UK Dev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-17 00:00:00.000000000 Z
+date: 2022-09-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -25,35 +25,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.11'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: bunny-mock
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.8.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 3.8.0
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.2
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 12.3.2
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: yard
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -67,21 +67,35 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bunny-mock
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.11'
 - !ruby/object:Gem::Dependency
-  name: pry-byebug
+  name: rubocop-govuk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 4.7.0
+- !ruby/object:Gem::Dependency
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -106,7 +120,6 @@ files:
 - LICENCE
 - README.md
 - lib/govuk_message_queue_consumer.rb
-- lib/govuk_message_queue_consumer/batch_consumer.rb
 - lib/govuk_message_queue_consumer/consumer.rb
 - lib/govuk_message_queue_consumer/heartbeat_processor.rb
 - lib/govuk_message_queue_consumer/json_processor.rb
@@ -128,14 +141,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.3.22
 signing_key: 
 specification_version: 4
 summary: AMQP message queue consumption with GOV.UK conventions

data/lib/govuk_message_queue_consumer/batch_consumer.rb

@@ -1,72 +0,0 @@
-module GovukMessageQueueConsumer
-  class BatchConsumer < Consumer
-    HANDLE_BATCHES = true
-    DEFAULT_BATCH_SIZE = 100
-    DEFAULT_BATCH_TIMEOUT = 5
-    # we want to increase the prefetch size here to the batch size
-    # so that we don't need to do multiple fetches for a batch.
-    NUMBER_OF_MESSAGES_TO_PREFETCH = DEFAULT_BATCH_SIZE
-
-    def initialize(options={})
-      opts = options.dup
-      @batch_size = opts.delete(:batch_size) || DEFAULT_BATCH_SIZE
-      @batch_timeout = opts.delete(:batch_timeout) || DEFAULT_BATCH_TIMEOUT
-      super(opts)
-    end
-
-    def run
-      @rabbitmq_connection.start
-      @running = true
-      while @running do
-        process_batch
-      end
-    end
-
-    # used for testing to stop processing in a different thread
-    def stop
-      @running = false
-    end
-
-  private
-
-    def process_batch
-      messages = []
-      with_timeout do
-        while messages.count < @batch_size do
-          delivery_info, headers, payload = queue.pop(manual_ack: true)
-          messages << Message.new(payload, headers, delivery_info) if payload
-        end
-      end
-
-      if messages.any?
-        @statsd_client.count("#{@queue_name}.started", messages.count)
-        @statsd_client.increment("#{@queue_name}.batch_started")
-        message_consumer.process(messages)
-
-        status_counts = messages.map(&:status).each_with_object(Hash.new(0)) { |s, h| h[s] += 1 }
-        status_counts.each do |status, count|
-          @statsd_client.count("#{@queue_name}.#{status}", count)
-        end
-        @statsd_client.increment("#{@queue_name}.batch_complete")
-      end
-    rescue SignalException => e
-      @logger.error "SignalException in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
-      exit(1) # Ensure rabbitmq requeues outstanding messages
-    rescue Exception => e
-      @statsd_client.increment("#{@queue_name}.uncaught_exception")
-      GovukError.notify(e) if defined?(GovukError)
-      @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
-
-      exit(1) # Ensure rabbitmq requeues outstanding messages
-    end
-
-    def with_timeout
-      begin
-        Timeout.timeout(@batch_timeout) do
-          yield
-        end
-      rescue Timeout::Error
-      end
-    end
-  end
-end