govuk_message_queue_consumer 3.5.0 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8a3ef3f89aa60ea51f6335665bf01b9ce01f5d313be4770fa72452f234fd626c
4
- data.tar.gz: 2c3ba684115314f3329705c52e670d1055c9109e32e7d56ac2e59d9893ba9510
3
+ metadata.gz: 2ff831021fdec8b26b02d2a1b9c36d5b35499a617e6c43995bb1dea97710d62e
4
+ data.tar.gz: 68fb8c56450889fd41def5ce02fd520a6e5d4ead28f738688ebb26a351769e05
5
5
  SHA512:
6
- metadata.gz: 42c2a962a238605044afaa5fb3f84c494324033bf2426317ad66d730595bfe26060811cc6c9c209cfe0017f60769446d41de1a234dd4042c9932a65531f60aab
7
- data.tar.gz: 4bb19be3b6309721256772f0570f1a63095067d6f5ce3c1ba612c12706d96ad7a4626eba6794563d17016935648ebfd8232be8a9cbbc009f7b5ef92dcda3f02a
6
+ metadata.gz: ccd4008cf70e0a1edb4ff3e1712233518bed5321c7ed5531853a8336d596ecf06ba645a1b65b4071a0fcf7b77885f0c6ed4200f26eb2c8284e7f4d14834d18cb
7
+ data.tar.gz: 0dc0f62ca439285e225f0cf3b0261158ffc3444339c765937077759e4565134618488826b8547b255bdbc92d41a04c392154ff50b9919ede2a8405f1f26c9862
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ # 4.0.0
2
+
3
+ - Breaking: remove batch consumer ([#73](https://github.com/alphagov/govuk_message_queue_consumer/pull/73))
4
+
1
5
  # 3.5.0
2
6
 
3
7
  - Stop reporting `SignalExceptions` to `GovukError`.
data/README.md CHANGED
@@ -17,26 +17,11 @@ This gem is used by:
17
17
  - [Cache clearing service](https://github.com/alphagov/cache-clearing-service).
18
18
  - [Content Data API](https://github.com/alphagov/content-data-api).
19
19
  - [Email Alert Service](https://github.com/alphagov/email-alert-service/).
20
- - [Rummager](https://github.com/alphagov/rummager).
21
-
22
- ## Nomenclature
23
-
24
- ![A graph showing the message flow](docs/graph.png)
25
-
26
- - **Producer**: an application that sends messages RabbitMQ. On GOV.UK this could
27
- be [publishing-api](https://github.com/alphagov/publishing-api).
28
- - **Message**: an object sent over RabbitMQ. It consists of a _payload_ and
29
- _headers_. In the case of the publishing-api the payload is a
30
- [content item](https://github.com/alphagov/govuk-content-schemas).
31
- - **Consumer**: the app that receives the messages and does something with them.
32
- On GOV.UK, this is [email-alert-service](https://github.com/alphagov/email-alert-service).
33
- - **Exchange**: in RabbitMQ's model, producers send messages to an _exchange_.
34
- Consumers can create a Queue that listens to the exchange, instead of
35
- subscribing to the exchange directly. This is done so that the queue can buffer
36
- any messages and we can make sure all messages get delivered to the consumer.
37
- - **Queue**: a queue listens to an exchange. In most cases the queue will listen
38
- to all messages, but it's also possible to listen to a specific pattern.
39
- - **Processor**: the specific class that processes a message.
20
+ - [Search API](https://github.com/alphagov/search-api).
21
+
22
+ ## Overview of RabbitMQ
23
+
24
+ To see an overview of RabbitMQ and how we use it, see [here](https://docs.publishing.service.gov.uk/manual/rabbitmq.html#overview).
40
25
 
41
26
  ## Technical documentation
42
27
 
@@ -10,6 +10,10 @@ module GovukMessageQueueConsumer
10
10
  # time to share the work evenly.
11
11
  NUMBER_OF_MESSAGES_TO_PREFETCH = 1
12
12
 
13
+ def self.default_connection_from_env
14
+ Bunny.new(GovukMessageQueueConsumer::RabbitMQConfig.from_environment(ENV))
15
+ end
16
+
13
17
  # Create a new consumer
14
18
  #
15
19
  # @param queue_name [String] Your queue name. This is specific to your application,
@@ -18,7 +22,7 @@ module GovukMessageQueueConsumer
18
22
  # @param rabbitmq_connection [Object] A Bunny connection object derived from `Bunny.new`
19
23
  # @param statsd_client [Statsd] An instance of the Statsd class
20
24
  # @param logger [Object] A Logger object for emitting errors (to stderr by default)
21
- def initialize(queue_name:, processor:, rabbitmq_connection: Consumer.default_connection_from_env, statsd_client: NullStatsd.new, logger: Logger.new(STDERR))
25
+ def initialize(queue_name:, processor:, rabbitmq_connection: Consumer.default_connection_from_env, statsd_client: NullStatsd.new, logger: Logger.new($stderr))
22
26
  @queue_name = queue_name
23
27
  @processor = processor
24
28
  @rabbitmq_connection = rabbitmq_connection
@@ -29,33 +33,29 @@ module GovukMessageQueueConsumer
29
33
  def run(subscribe_opts: {})
30
34
  @rabbitmq_connection.start
31
35
 
32
- subscribe_opts = { block: true, manual_ack: true}.merge(subscribe_opts)
36
+ subscribe_opts = { block: true, manual_ack: true }.merge(subscribe_opts)
33
37
  queue.subscribe(subscribe_opts) do |delivery_info, headers, payload|
34
- begin
35
- message = Message.new(payload, headers, delivery_info)
36
- @statsd_client.increment("#{@queue_name}.started")
37
- message_consumer.process(message)
38
- @statsd_client.increment("#{@queue_name}.#{message.status}")
39
- rescue SignalException => e
40
- @logger.error "SignalException in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
41
- exit(1) # Ensure rabbitmq requeues outstanding messages
42
- rescue Exception => e
43
- @statsd_client.increment("#{@queue_name}.uncaught_exception")
44
- GovukError.notify(e) if defined?(GovukError)
45
- @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
46
- exit(1) # Ensure rabbitmq requeues outstanding messages
47
- end
38
+ message = Message.new(payload, headers, delivery_info)
39
+ @statsd_client.increment("#{@queue_name}.started")
40
+ message_consumer.process(message)
41
+ @statsd_client.increment("#{@queue_name}.#{message.status}")
42
+ rescue SignalException => e
43
+ @logger.error "SignalException in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
44
+ exit(1) # Ensure rabbitmq requeues outstanding messages
45
+ rescue StandardError => e
46
+ @statsd_client.increment("#{@queue_name}.uncaught_exception")
47
+ GovukError.notify(e) if defined?(GovukError)
48
+ @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
49
+ exit(1) # Ensure rabbitmq requeues outstanding messages
48
50
  end
49
51
  end
50
52
 
51
53
  private
52
54
 
53
55
  class NullStatsd
54
- def increment(_key)
55
- end
56
+ def increment(_key); end
56
57
 
57
- def count(_key, _volume)
58
- end
58
+ def count(_key, _volume); end
59
59
  end
60
60
 
61
61
  def message_consumer
@@ -79,9 +79,5 @@ module GovukMessageQueueConsumer
79
79
  def channel
80
80
  @channel ||= @rabbitmq_connection.create_channel
81
81
  end
82
-
83
- def self.default_connection_from_env
84
- Bunny.new(GovukMessageQueueConsumer::RabbitMQConfig.from_environment(ENV))
85
- end
86
82
  end
87
83
  end
@@ -17,7 +17,7 @@ module GovukMessageQueueConsumer
17
17
 
18
18
  def handles_batches?(processor)
19
19
  case processor
20
- when HeartbeatProcessor,JSONProcessor
20
+ when HeartbeatProcessor, JSONProcessor
21
21
  false
22
22
  else
23
23
  @handle_batches
@@ -5,7 +5,7 @@ module GovukMessageQueueConsumer
5
5
 
6
6
  def self.from_environment(env)
7
7
  {
8
- hosts: fetch(env, "RABBITMQ_HOSTS").split(','),
8
+ hosts: fetch(env, "RABBITMQ_HOSTS").split(","),
9
9
  vhost: fetch(env, "RABBITMQ_VHOST"),
10
10
  user: fetch(env, "RABBITMQ_USER"),
11
11
  pass: fetch(env, "RABBITMQ_PASSWORD"),
@@ -13,19 +13,17 @@ module GovukMessageQueueConsumer
13
13
  }
14
14
  end
15
15
 
16
- private
17
-
18
16
  def self.fetch(env, variable_name)
19
17
  env[variable_name] || raise_error(variable_name)
20
18
  end
21
19
 
22
20
  def self.raise_error(variable_name)
23
- raise ConfigurationError, <<-err
21
+ raise ConfigurationError, <<-ERR
24
22
  The environment variable #{variable_name} is not set. If you are in test
25
23
  mode, make sure you set the correct vars in your helpers. If you get this
26
24
  error in development, make sure you run rails or rake with `govuk_setenv`
27
25
  and puppet is up to date.
28
- err
26
+ ERR
29
27
  end
30
28
  end
31
29
  end
@@ -2,14 +2,12 @@ module GovukMessageQueueConsumer
2
2
  class MockMessage < Message
3
3
  attr_reader :acked, :retried, :discarded, :payload, :header, :delivery_info
4
4
 
5
- alias :acked? :acked
6
- alias :discarded? :discarded
7
- alias :retried? :retried
5
+ alias_method :acked?, :acked
6
+ alias_method :discarded?, :discarded
7
+ alias_method :retried?, :retried
8
8
 
9
9
  def initialize(payload = {}, headers = {}, delivery_info = {})
10
- @payload = payload
11
- @headers = OpenStruct.new(headers)
12
- @delivery_info = OpenStruct.new(delivery_info)
10
+ super(payload, OpenStruct.new(headers), OpenStruct.new(delivery_info))
13
11
  end
14
12
 
15
13
  def ack
@@ -1,2 +1,2 @@
1
- require 'govuk_message_queue_consumer/test_helpers/shared_examples'
2
- require 'govuk_message_queue_consumer/test_helpers/mock_message'
1
+ require "govuk_message_queue_consumer/test_helpers/shared_examples"
2
+ require "govuk_message_queue_consumer/test_helpers/mock_message"
@@ -1,3 +1,3 @@
1
1
  module GovukMessageQueueConsumer
2
- VERSION = '3.5.0'
2
+ VERSION = "4.0.0".freeze
3
3
  end
@@ -1,11 +1,10 @@
1
- require 'bunny'
2
- require 'json'
1
+ require "bunny"
2
+ require "json"
3
3
 
4
- require 'govuk_message_queue_consumer/version'
5
- require 'govuk_message_queue_consumer/heartbeat_processor'
6
- require 'govuk_message_queue_consumer/json_processor'
7
- require 'govuk_message_queue_consumer/message'
8
- require 'govuk_message_queue_consumer/message_consumer'
9
- require 'govuk_message_queue_consumer/consumer'
10
- require 'govuk_message_queue_consumer/batch_consumer'
11
- require 'govuk_message_queue_consumer/rabbitmq_config'
4
+ require "govuk_message_queue_consumer/version"
5
+ require "govuk_message_queue_consumer/heartbeat_processor"
6
+ require "govuk_message_queue_consumer/json_processor"
7
+ require "govuk_message_queue_consumer/message"
8
+ require "govuk_message_queue_consumer/message_consumer"
9
+ require "govuk_message_queue_consumer/consumer"
10
+ require "govuk_message_queue_consumer/rabbitmq_config"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: govuk_message_queue_consumer
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.5.0
4
+ version: 4.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - GOV.UK Dev
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-07-17 00:00:00.000000000 Z
11
+ date: 2022-09-23 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bunny
@@ -25,35 +25,35 @@ dependencies:
25
25
  - !ruby/object:Gem::Version
26
26
  version: '2.11'
27
27
  - !ruby/object:Gem::Dependency
28
- name: rspec
28
+ name: bunny-mock
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - "~>"
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: 3.8.0
33
+ version: '0'
34
34
  type: :development
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - "~>"
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: 3.8.0
40
+ version: '0'
41
41
  - !ruby/object:Gem::Dependency
42
- name: rake
42
+ name: pry-byebug
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
- version: 12.3.2
47
+ version: '0'
48
48
  type: :development
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
- version: 12.3.2
54
+ version: '0'
55
55
  - !ruby/object:Gem::Dependency
56
- name: yard
56
+ name: rake
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
59
  - - ">="
@@ -67,21 +67,35 @@ dependencies:
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
69
  - !ruby/object:Gem::Dependency
70
- name: bunny-mock
70
+ name: rspec
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - ">="
73
+ - - "~>"
74
74
  - !ruby/object:Gem::Version
75
- version: '0'
75
+ version: '3.11'
76
76
  type: :development
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - ">="
80
+ - - "~>"
81
81
  - !ruby/object:Gem::Version
82
- version: '0'
82
+ version: '3.11'
83
83
  - !ruby/object:Gem::Dependency
84
- name: pry-byebug
84
+ name: rubocop-govuk
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '='
88
+ - !ruby/object:Gem::Version
89
+ version: 4.7.0
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '='
95
+ - !ruby/object:Gem::Version
96
+ version: 4.7.0
97
+ - !ruby/object:Gem::Dependency
98
+ name: yard
85
99
  requirement: !ruby/object:Gem::Requirement
86
100
  requirements:
87
101
  - - ">="
@@ -106,7 +120,6 @@ files:
106
120
  - LICENCE
107
121
  - README.md
108
122
  - lib/govuk_message_queue_consumer.rb
109
- - lib/govuk_message_queue_consumer/batch_consumer.rb
110
123
  - lib/govuk_message_queue_consumer/consumer.rb
111
124
  - lib/govuk_message_queue_consumer/heartbeat_processor.rb
112
125
  - lib/govuk_message_queue_consumer/json_processor.rb
@@ -128,14 +141,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
128
141
  requirements:
129
142
  - - ">="
130
143
  - !ruby/object:Gem::Version
131
- version: '0'
144
+ version: '2.7'
132
145
  required_rubygems_version: !ruby/object:Gem::Requirement
133
146
  requirements:
134
147
  - - ">="
135
148
  - !ruby/object:Gem::Version
136
149
  version: '0'
137
150
  requirements: []
138
- rubygems_version: 3.0.3
151
+ rubygems_version: 3.3.22
139
152
  signing_key:
140
153
  specification_version: 4
141
154
  summary: AMQP message queue consumption with GOV.UK conventions
@@ -1,72 +0,0 @@
1
- module GovukMessageQueueConsumer
2
- class BatchConsumer < Consumer
3
- HANDLE_BATCHES = true
4
- DEFAULT_BATCH_SIZE = 100
5
- DEFAULT_BATCH_TIMEOUT = 5
6
- # we want to increase the prefetch size here to the batch size
7
- # so that we don't need to do multiple fetches for a batch.
8
- NUMBER_OF_MESSAGES_TO_PREFETCH = DEFAULT_BATCH_SIZE
9
-
10
- def initialize(options={})
11
- opts = options.dup
12
- @batch_size = opts.delete(:batch_size) || DEFAULT_BATCH_SIZE
13
- @batch_timeout = opts.delete(:batch_timeout) || DEFAULT_BATCH_TIMEOUT
14
- super(opts)
15
- end
16
-
17
- def run
18
- @rabbitmq_connection.start
19
- @running = true
20
- while @running do
21
- process_batch
22
- end
23
- end
24
-
25
- # used for testing to stop processing in a different thread
26
- def stop
27
- @running = false
28
- end
29
-
30
- private
31
-
32
- def process_batch
33
- messages = []
34
- with_timeout do
35
- while messages.count < @batch_size do
36
- delivery_info, headers, payload = queue.pop(manual_ack: true)
37
- messages << Message.new(payload, headers, delivery_info) if payload
38
- end
39
- end
40
-
41
- if messages.any?
42
- @statsd_client.count("#{@queue_name}.started", messages.count)
43
- @statsd_client.increment("#{@queue_name}.batch_started")
44
- message_consumer.process(messages)
45
-
46
- status_counts = messages.map(&:status).each_with_object(Hash.new(0)) { |s, h| h[s] += 1 }
47
- status_counts.each do |status, count|
48
- @statsd_client.count("#{@queue_name}.#{status}", count)
49
- end
50
- @statsd_client.increment("#{@queue_name}.batch_complete")
51
- end
52
- rescue SignalException => e
53
- @logger.error "SignalException in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
54
- exit(1) # Ensure rabbitmq requeues outstanding messages
55
- rescue Exception => e
56
- @statsd_client.increment("#{@queue_name}.uncaught_exception")
57
- GovukError.notify(e) if defined?(GovukError)
58
- @logger.error "Uncaught exception in processor: \n\n #{e.class}: #{e.message}\n\n#{e.backtrace.join("\n")}"
59
-
60
- exit(1) # Ensure rabbitmq requeues outstanding messages
61
- end
62
-
63
- def with_timeout
64
- begin
65
- Timeout.timeout(@batch_timeout) do
66
- yield
67
- end
68
- rescue Timeout::Error
69
- end
70
- end
71
- end
72
- end