govuk_message_queue_consumer 3.1.0 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 48237ae34d62ad2c19bfa687518f0f83b8953855
-  data.tar.gz: 3078b5856d7cbd3f1b3eeb2365f7a924337ba9bd
+SHA256:
+  metadata.gz: 92d5ee51b1ed29b2d86c75475cf767c3cb978d19bfe17cc92cab3cadb9de4bce
+  data.tar.gz: 9f2667dff5e909beccace001dd9f2cef64911253f019ec86009ba210e722a69b
 SHA512:
-  metadata.gz: a821d0c9b7c3d65de3d8aa88cafca57ac50ea3196f5add31a3108edfcfe90664f723ab4de9357f697e17512fead995fb0036a7be9f0270379908ee16769461e5
-  data.tar.gz: 7b1c8112e94b10db93a0038377e980a06fbfe4acfbe8953308a1a90302e44150fd2972f6880533e8cddb952a30104c23a4040f7af8cb83b3949577c631e29171
+  metadata.gz: eedbec227fd052d8427c9b8de008550467b3bc10d57e88f0a2605c1a5053b76488bec2b9d4964aceb5bcda0d67953ba4ec6045ad10bee52224ce4ac7d041efb8
+  data.tar.gz: c7f8c8d4a6b8273a30f8483c88daf3d7f712b13ac933ee9c64b844f72e6713ba763c6fe019e61316562b989a21c390d096a38cb73ed1ebf22267fe6daa6ed659

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 3.2.0
+
+- Add batch process capabilities
+- Refactor `process_chain` to `message_consumer` 
+
 # 3.1.0
 
 - Change Airbrake to GovukError

data/lib/govuk_message_queue_consumer.rb

@@ -5,5 +5,7 @@ require 'govuk_message_queue_consumer/version'
 require 'govuk_message_queue_consumer/heartbeat_processor'
 require 'govuk_message_queue_consumer/json_processor'
 require 'govuk_message_queue_consumer/message'
+require 'govuk_message_queue_consumer/message_consumer'
 require 'govuk_message_queue_consumer/consumer'
+require 'govuk_message_queue_consumer/batch_consumer'
 require 'govuk_message_queue_consumer/rabbitmq_config'

data/lib/govuk_message_queue_consumer/batch_consumer.rb

@@ -0,0 +1,69 @@
+module GovukMessageQueueConsumer
+  class BatchConsumer < Consumer
+    HANDLE_BATCHES = true
+    DEFAULT_BATCH_SIZE = 100
+    DEFAULT_BATCH_TIMEOUT = 5
+    # we want to increase the prefetch size here to the batch size
+    # so that we don't need to do multiple fetches for a batch.
+    NUMBER_OF_MESSAGES_TO_PREFETCH = DEFAULT_BATCH_SIZE
+
+    def initialize(options={})
+      opts = options.dup
+      @batch_size = opts.delete(:batch_size) || DEFAULT_BATCH_SIZE
+      @batch_timeout = opts.delete(:batch_timeout) || DEFAULT_BATCH_TIMEOUT
+      super(opts)
+    end
+
+    def run
+      @rabbitmq_connection.start
+      @running = true
+      while @running do
+        process_batch
+      end
+    end
+
+    # used for testing to stop processing in a different thread
+    def stop
+      @running = false
+    end
+
+  private
+
+    def process_batch
+      messages = []
+      with_timeout do
+        while messages.count < @batch_size do
+          delivery_info, headers, payload = queue.pop(manual_ack: true)
+          messages << Message.new(payload, headers, delivery_info) if payload
+        end
+      end
+
+      if messages.any?
+        @statsd_client.count("#{@queue_name}.started", messages.count)
+        @statsd_client.increment("#{@queue_name}.batch_started")
+        message_consumer.process(messages)
+
+        status_counts = messages.map(&:status).each_with_object(Hash.new(0)) { |s, h| h[s] += 1 }
+        status_counts.each do |status, count|
+          @statsd_client.count("#{@queue_name}.#{status}", count)
+        end
+        @statsd_client.increment("#{@queue_name}.batch_complete")
+      end
+    rescue Exception => e
+      @statsd_client.increment("#{@queue_name}.uncaught_exception")
+      GovukError.notify(e) if defined?(GovukError)
+      @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
+
+      exit(1) # Ensure rabbitmq requeues outstanding messages
+    end
+
+    def with_timeout
+      begin
+        Timeout.timeout(@batch_timeout) do
+          yield
+        end
+      rescue Timeout::Error
+      end
+    end
+  end
+end

data/lib/govuk_message_queue_consumer/consumer.rb

@@ -1,5 +1,6 @@
 module GovukMessageQueueConsumer
   class Consumer
+    HANDLE_BATCHES = false
     # Only fetch one message at a time on the channel.
     #
     # By default, queues will grab messages eagerly, which reduces latency.
@@ -32,7 +33,7 @@ module GovukMessageQueueConsumer
         begin
           message = Message.new(payload, headers, delivery_info)
           @statsd_client.increment("#{@queue_name}.started")
-          processor_chain.process(message)
+          message_consumer.process(message)
           @statsd_client.increment("#{@queue_name}.#{message.status}")
         rescue Exception => e
           @statsd_client.increment("#{@queue_name}.uncaught_exception")
@@ -48,15 +49,25 @@ module GovukMessageQueueConsumer
     class NullStatsd
       def increment(_key)
       end
+
+      def count(_key, _volume)
+      end
     end
 
-    def processor_chain
-      @processor_chain ||= HeartbeatProcessor.new(JSONProcessor.new(@processor))
+    def message_consumer
+      @message_consumer ||= MessageConsumer.new(
+        processors: [
+          HeartbeatProcessor.new,
+          JSONProcessor.new,
+          @processor,
+        ],
+        handle_batches: self.class::HANDLE_BATCHES,
+      )
     end
 
     def queue
       @queue ||= begin
-        channel.prefetch(NUMBER_OF_MESSAGES_TO_PREFETCH)
+        channel.prefetch(self.class::NUMBER_OF_MESSAGES_TO_PREFETCH)
         channel.queue(@queue_name, no_declare: true)
       end
     end

data/lib/govuk_message_queue_consumer/heartbeat_processor.rb

@@ -1,16 +1,13 @@
 module GovukMessageQueueConsumer
   class HeartbeatProcessor
-    def initialize(next_processor)
-      @next_processor = next_processor
-    end
-
     def process(message)
       # Ignore heartbeat messages
       if message.headers.content_type == "application/x-heartbeat"
         message.ack
-      else
-        @next_processor.process(message)
+        return false
       end
+
+      true
     end
   end
 end

data/lib/govuk_message_queue_consumer/json_processor.rb

@@ -2,19 +2,16 @@ module GovukMessageQueueConsumer
   class JSONProcessor
     JSON_FORMAT = "application/json".freeze
 
-    def initialize(next_processor)
-      @next_processor = next_processor
-    end
-
     def process(message)
       if message.headers.content_type == JSON_FORMAT
         message.payload = JSON.parse(message.payload)
       end
 
-      @next_processor.process(message)
+      true
     rescue JSON::ParserError => e
       GovukError.notify(e) if defined?(GovukError)
       message.discard
+      false
     end
   end
 end

data/lib/govuk_message_queue_consumer/message_consumer.rb

@@ -0,0 +1,27 @@
+module GovukMessageQueueConsumer
+  class MessageConsumer
+    def initialize(processors:, handle_batches:)
+      @processors = processors
+      @handle_batches = handle_batches
+    end
+
+    def process(records)
+      @processors.inject(Array(records)) do |remaining_records, processor|
+        if handles_batches?(processor)
+          processor.process(remaining_records)
+        else
+          remaining_records.select { |record| processor.process(record) }
+        end
+      end
+    end
+
+    def handles_batches?(processor)
+      case processor
+      when HeartbeatProcessor,JSONProcessor
+        false
+      else
+        @handle_batches
+      end
+    end
+  end
+end

data/lib/govuk_message_queue_consumer/version.rb

@@ -1,3 +1,3 @@
 module GovukMessageQueueConsumer
-  VERSION = '3.1.0'
+  VERSION = '3.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: govuk_message_queue_consumer
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.0
 platform: ruby
 authors:
 - GOV.UK Dev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-27 00:00:00.000000000 Z
+date: 2018-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -66,6 +66,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: bunny-mock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Avoid writing boilerplate code in order to consume messages from an AMQP
   message queue. Plug in queue configuration, and how to process each message.
 email:
@@ -78,10 +106,12 @@ files:
 - LICENCE
 - README.md
 - lib/govuk_message_queue_consumer.rb
+- lib/govuk_message_queue_consumer/batch_consumer.rb
 - lib/govuk_message_queue_consumer/consumer.rb
 - lib/govuk_message_queue_consumer/heartbeat_processor.rb
 - lib/govuk_message_queue_consumer/json_processor.rb
 - lib/govuk_message_queue_consumer/message.rb
+- lib/govuk_message_queue_consumer/message_consumer.rb
 - lib/govuk_message_queue_consumer/rabbitmq_config.rb
 - lib/govuk_message_queue_consumer/test_helpers.rb
 - lib/govuk_message_queue_consumer/test_helpers/mock_message.rb
@@ -106,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: AMQP message queue consumption with GOV.UK conventions