surfliner-metadata_consumer 0.1.0.pre.alpha.3 → 0.1.0.pre.alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fac43306c988b19a28f9cb92449724b7cb326a787f54c7da78c9b134a61220aa
4
- data.tar.gz: af91a86d088e4b863425a45faf97b2473cbca69cb17178d75d4bcce19d79384a
3
+ metadata.gz: 2fff0fe7a29854ecb3b75f91ed5aa1e5abdbf6fe7fc625b4b117a3e2c4e4805c
4
+ data.tar.gz: 534536637b909d309f90095aaba4e9cb51a75d4ca92a78531971d3550ea42bfc
5
5
  SHA512:
6
- metadata.gz: ea1dbf415bd4339f875562a7f629ea432915e3dc1b511ba1268765069db0227031dc5eb71e2acfcb7f4b5c434a3c82ba88b4cc266dec94b94479462ae3721781
7
- data.tar.gz: d0840fc308daec6d1ed9f22038f62e2ed4974ea450054b3bec580e1a28f8aff52c5081cfd600e46137546785cc726a420190b0bdf82a0eb7f0c2db5fa447736b
6
+ metadata.gz: 6e20f325aa14042b5ed7e950803d8a045834df543da1f5c4a8f30297645015d97add22a5de38e01d15a6ab74adb856777bba4fccfb609d50bd622c9fae1f51d5
7
+ data.tar.gz: a69109802f8c408f077b2fc2d1c748c02a900eda042d6e40a4423ee3257b39d2dde892eecf9c77c690df06aa53d367136a8246f3d8bc69114d0e1278326b4531
data/CHANGES.md CHANGED
@@ -1,3 +1,40 @@
1
+ # 0.1.0.pre.alpha.5 (2025-03-25)
2
+
3
+ In order to support using one connection (and one underlying `Bunny::Session`)
4
+ with multiple topics, queues, and/or routing keys, as well as publishing to
5
+ a topic without creating or subscribing to a queue, this release introduces
6
+ the following **breaking changes**:
7
+
8
+ - RabbitMQ-related configuration and connection code (i.e. not specific to
9
+ the consumer/handler setup) has been moved from `Surfliner::MetadataConsumer`
10
+ to `Surfliner::Mq`. This may move to a separate gem in a future release.
11
+ - `MetadataConsumer::MqConnection` is now `Mq::Connection`
12
+ - `MetadataConsumer::MqConfig` is now `Mq::ConnectionConfig`
13
+ - `ConnectionConfig` no longer includes topic or queue configuration;
14
+ topics are configured with `TopicConfig` and queues with
15
+ `QueueConfig`.
16
+ - The `MqConnection#open` method, which yielded a `Bunny::Queue`, has been
17
+ replaced by the `Connection#with_topic` method, which yields an
18
+ `Mq::Topic` object wrapping a single topic; call `Mq::Topic#bind_queue`
19
+ to bind to a queue. Like the removed`#open`, `#with_topic` opens and
20
+ closes the connection implicitly and allows connecting only to a single
21
+ topic.
22
+ - The `Mq::Connection#topic_from(config)` method allows you to connect to
23
+ any number of topics, but you must open and close the connection
24
+ explicitly.
25
+ - The `MqConnection#publish` method has been moved to `Mq::Topic` and can
26
+ now take an explicit routing key, although if one is not provided it will
27
+ still default to `ENV["RABBITMQ_PLATFORM_ROUTING_KEY"]`.
28
+ - `MetadataConsumer::Consumer` now requires an explicit `Mq:Connection`
29
+ instead of creating one by default.
30
+
31
+ In addition, the `daylight-index-listen` script has been renamed to the more
32
+ generic `index-on-publish`.
33
+
34
+ # 0.1.0.pre.alpha.4 (2025-03-17)
35
+
36
+ - lower required Ruby version to 3.2
37
+
1
38
  # 0.1.0.pre.alpha.3 (2025-03-07)
2
39
 
3
40
  - improve topic/queue option override API
@@ -15,4 +52,4 @@
15
52
 
16
53
  # 0.1.0.pre.alpha (2025-02-12)
17
54
 
18
- - Initial release
55
+ - Initial release
data/Dockerfile CHANGED
@@ -56,4 +56,4 @@ COPY --from=build $APP_PATH/Gemfile.lock $APP_PATH
56
56
  RUN bundle config set frozen 'true'
57
57
 
58
58
  # TODO: don't hard-code a specific handler / script
59
- CMD ["./bin/daylight-index-listen"]
59
+ CMD ["./bin/index-on-publish"]
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- surfliner-metadata_consumer (0.1.0.pre.alpha.3)
4
+ surfliner-metadata_consumer (0.1.0.pre.alpha.5)
5
5
  bunny (~> 2.23)
6
6
  opentelemetry-exporter-otlp (~> 0.26.3)
7
7
  opentelemetry-instrumentation-all (~> 0.60.0)
@@ -14,11 +14,11 @@ GEM
14
14
  addressable (2.8.7)
15
15
  public_suffix (>= 2.0.2, < 7.0)
16
16
  amq-protocol (2.3.3)
17
- ast (2.4.2)
17
+ ast (2.4.3)
18
18
  bigdecimal (3.1.9)
19
19
  builder (3.3.0)
20
- bunny (2.23.0)
21
- amq-protocol (~> 2.3, >= 2.3.1)
20
+ bunny (2.24.0)
21
+ amq-protocol (~> 2.3)
22
22
  sorted_set (~> 1, >= 1.0.2)
23
23
  ci_reporter (2.1.0)
24
24
  builder (>= 2.1.2)
@@ -34,7 +34,7 @@ GEM
34
34
  debug (1.9.2)
35
35
  irb (~> 1.10)
36
36
  reline (>= 0.3.8)
37
- diff-lcs (1.6.0)
37
+ diff-lcs (1.6.1)
38
38
  docile (1.4.1)
39
39
  dotenv (2.8.1)
40
40
  faraday (2.12.2)
@@ -43,8 +43,9 @@ GEM
43
43
  logger
44
44
  faraday-net_http (3.4.0)
45
45
  net-http (>= 0.5.0)
46
+ github-markup (5.0.1)
46
47
  google-protobuf (3.25.6)
47
- googleapis-common-protos-types (1.18.0)
48
+ googleapis-common-protos-types (1.19.0)
48
49
  google-protobuf (>= 3.18, < 5.a)
49
50
  hashdiff (1.1.2)
50
51
  io-console (0.8.0)
@@ -52,7 +53,7 @@ GEM
52
53
  pp (>= 0.6.0)
53
54
  rdoc (>= 4.0.0)
54
55
  reline (>= 0.4.2)
55
- json (2.10.1)
56
+ json (2.10.2)
56
57
  language_server-protocol (3.17.0.4)
57
58
  lint_roller (1.1.0)
58
59
  logger (1.6.6)
@@ -255,12 +256,13 @@ GEM
255
256
  opentelemetry-semantic_conventions (1.11.0)
256
257
  opentelemetry-api (~> 1.0)
257
258
  parallel (1.26.3)
258
- parser (3.3.7.1)
259
+ parser (3.3.7.2)
259
260
  ast (~> 2.4.1)
260
261
  racc
261
262
  pp (0.6.2)
262
263
  prettyprint
263
264
  prettyprint (0.2.0)
265
+ prism (1.4.0)
264
266
  psych (5.2.3)
265
267
  date
266
268
  stringio
@@ -269,8 +271,9 @@ GEM
269
271
  rainbow (3.1.1)
270
272
  rake (13.2.1)
271
273
  rbtree (0.4.6)
272
- rdoc (6.12.0)
274
+ rdoc (6.13.0)
273
275
  psych (>= 4.0.0)
276
+ redcarpet (3.6.1)
274
277
  regexp_parser (2.10.0)
275
278
  reline (0.6.0)
276
279
  io-console (~> 0.5)
@@ -291,9 +294,10 @@ GEM
291
294
  diff-lcs (>= 1.2.0, < 2.0)
292
295
  rspec-support (~> 3.13.0)
293
296
  rspec-support (3.13.2)
294
- rubocop (1.71.2)
297
+ rubocop (1.73.2)
295
298
  json (~> 2.3)
296
- language_server-protocol (>= 3.17.0)
299
+ language_server-protocol (~> 3.17.0.2)
300
+ lint_roller (~> 1.1.0)
297
301
  parallel (~> 1.10)
298
302
  parser (>= 3.3.0.2)
299
303
  rainbow (>= 2.2.2, < 4.0)
@@ -301,11 +305,13 @@ GEM
301
305
  rubocop-ast (>= 1.38.0, < 2.0)
302
306
  ruby-progressbar (~> 1.7)
303
307
  unicode-display_width (>= 2.4.0, < 4.0)
304
- rubocop-ast (1.38.1)
305
- parser (>= 3.3.1.0)
306
- rubocop-performance (1.23.1)
307
- rubocop (>= 1.48.1, < 2.0)
308
- rubocop-ast (>= 1.31.1, < 2.0)
308
+ rubocop-ast (1.43.0)
309
+ parser (>= 3.3.7.2)
310
+ prism (~> 1.4)
311
+ rubocop-performance (1.24.0)
312
+ lint_roller (~> 1.1)
313
+ rubocop (>= 1.72.1, < 2.0)
314
+ rubocop-ast (>= 1.38.0, < 2.0)
309
315
  ruby-progressbar (1.13.0)
310
316
  set (1.1.1)
311
317
  simplecov (0.22.0)
@@ -320,24 +326,24 @@ GEM
320
326
  sorted_set (1.0.3)
321
327
  rbtree
322
328
  set (~> 1.0)
323
- standard (1.45.0)
329
+ standard (1.47.0)
324
330
  language_server-protocol (~> 3.17.0.2)
325
331
  lint_roller (~> 1.0)
326
- rubocop (~> 1.71.0)
332
+ rubocop (~> 1.73.0)
327
333
  standard-custom (~> 1.0.0)
328
- standard-performance (~> 1.6)
334
+ standard-performance (~> 1.7)
329
335
  standard-custom (1.0.2)
330
336
  lint_roller (~> 1.0)
331
337
  rubocop (~> 1.50)
332
- standard-performance (1.6.0)
338
+ standard-performance (1.7.0)
333
339
  lint_roller (~> 1.1)
334
- rubocop-performance (~> 1.23.0)
335
- stringio (3.1.5)
340
+ rubocop-performance (~> 1.24.0)
341
+ stringio (3.1.6)
336
342
  unicode-display_width (3.1.4)
337
343
  unicode-emoji (~> 4.0, >= 4.0.4)
338
344
  unicode-emoji (4.0.4)
339
345
  uri (1.0.3)
340
- webmock (3.25.0)
346
+ webmock (3.25.1)
341
347
  addressable (>= 2.8.0)
342
348
  crack (>= 0.3.2)
343
349
  hashdiff (>= 0.4.0, < 2.0.0)
@@ -352,7 +358,9 @@ DEPENDENCIES
352
358
  colorize (~> 0.8)
353
359
  debug (~> 1.9.2)
354
360
  dotenv (~> 2.7)
361
+ github-markup (~> 5.0)
355
362
  rake (~> 13.0)
363
+ redcarpet (~> 3.6)
356
364
  rspec (~> 3.13)
357
365
  simplecov (~> 0.22)
358
366
  simplecov-cobertura (~> 2.1)
data/README.md CHANGED
@@ -36,23 +36,59 @@ if prior steps succeed.
36
36
 
37
37
  ## Configuration
38
38
 
39
- <!--
40
- TODO: Just provide a sample env file
41
- -->
39
+ > Note: Sample values taken from Surfliner's development
40
+ > [daylight-listener.sh](https://gitlab.com/surfliner/surfliner/-/blob/trunk/docker-compose/env/daylight-listener.sh).)
41
+
42
+ An `Mq::Connection` object, suitable for listening or publishing, can be configured
43
+ explicitly with a `ConnectionConfig` object, or implicitly, reading a default
44
+ configuration with `ConnectionConfig#from_env`. `ConnectionConfig#from_env` expects
45
+ the following environment variables:
46
+
47
+ | Variable | Sample value | Description |
48
+ |-----------------------------|--------------|------------------------------|
49
+ | `RABBITMQ_HOST` | `rabbitmq` | Hostname of RabbitMQ server |
50
+ | `RABBITMQ_NODE_PORT_NUMBER` | `5672` | Port name of RabbitMQ server |
51
+ | `RABBITMQ_USERNAME` | `user` | RabbitMQ username |
52
+ | `RABBITMQ_PASSWORD` | `bitnami` | RabbitMQ password |
53
+
54
+ The `Mq::Connection#with_topic` and `Mq::Connection#topic_from` can either take an
55
+ explicit `TopicConfig` object, or implicitly read a default configuration with
56
+ `TopicConfig#from_env`. `TopicConfig#from_env` expects the following environment variables:
57
+
58
+ | Variable | Sample value | Description |
59
+ |------------------|----------------------|---------------------|
60
+ | `RABBITMQ_TOPIC` | `surfliner.metadata` | RabbitMQ topic name |
61
+
62
+ Note that `TopicConfig#from_env` also accepts keyword options, which are passed through
63
+ to [`Bunny::Channel#topic`](https://api.rubybunny.info/Bunny/Channel.html#topic-instance_method).
64
+ E.g.
65
+
66
+ ```ruby
67
+ topic_config = TopicConfig.from_env(durable: true, auto_delete: true)
68
+ connection.with_topic(topic_config) do |topic|
69
+ # ...
70
+ end
71
+ ```
42
72
 
43
- The `MqConnection` class expects the following environment variables:
73
+ Similarly, `Mq::Topic#bind_queue` can either take an explicit `QueueConfig`, or implicitly
74
+ read a default with `QueConfig#from_env`. `QueueConfig#from_env` expects the following environment
75
+ variables:
44
76
 
45
- ```sh
46
- RABBITMQ_HOST=rabbitmq
47
- RABBITMQ_NODE_PORT_NUMBER=5672
48
- RABBITMQ_PASSWORD=bitnami
49
- RABBITMQ_QUEUE=surfliner.metadata
50
- RABBITMQ_PLATFORM_ROUTING_KEY=surfliner.metadata.daylight
51
- RABBITMQ_TOPIC=surfliner.metadata
52
- RABBITMQ_USERNAME=user
53
- ```
77
+ | Variable | Sample value | Description |
78
+ |---------------------------------|-------------------------------|----------------------|
79
+ | `RABBITMQ_QUEUE` | `surfliner.metadata` | RabbitMQ queue name |
80
+ | `RABBITMQ_PLATFORM_ROUTING_KEY` | `surfliner.metadata.daylight` | RabbitMQ routing key |
54
81
 
55
- (Sample values taken from Surfliner's development [daylight-listener.sh](https://gitlab.com/surfliner/surfliner/-/blob/trunk/docker-compose/env/daylight-listener.sh).)
82
+ And `QueueConfig#from_env` similarly accepts keyword options, which are forwarded to
83
+ [`Bunny::Channel#queue`](https://api.rubybunny.info/Bunny/Channel.html#queue-instance_method):
84
+
85
+ ```ruby
86
+ connection.with_topic(topic_config) do |topic|
87
+ queue_config = QueueConfig.from_env(exclusive: true, durable: true)
88
+ queue = topic.bind_queue(queue_config)
89
+ # ...
90
+ end
91
+ ```
56
92
 
57
93
  The Solr / Daylight handler implementation (see below) additionally expects
58
94
  a configured `SOLR_URL`, e.g.:
@@ -74,17 +110,19 @@ The `Surfliner::MetadataConsumer::Solr` package contains a handler that retrieve
74
110
  metadata for `:published`/`:updated` events, transforms results for indexing, and
75
111
  updates a target Solr index.
76
112
 
77
- <!-- TODO: better name for this script -->
113
+ The `bin/index-on-publish` script starts a consumer using this handler, and accepts
114
+ (but does not require) the following environment variables:
78
115
 
79
- The `bin/daylight-index-listen` script starts a consumer using this handler.
80
-
81
- In addition to the environment variables listed above, this handler expects
82
- a configured `SOLR_URL`, e.g.:
116
+ | Variable | Sample value | Default | Description |
117
+ |---------------------|-------------------------------|----------------|----------------------------------|
118
+ | `OTEL_SDK_DISABLED` | `true` | (not set) | Whether to disable OpenTelemetry |
119
+ | `OTEL_SERVICE_NAME` | `surfliner-daylight-consumer` | name of script | OpenTelemetry service name |
120
+ | `OTEL_TRACER_NAME` | `DaylightConsumerTracer` | name of script | OpenTelemetry tracer name |
83
121
 
84
122
  ### Running the service
85
123
 
86
124
  <!--
87
- TODO: Dockerfile shouldn't be daylight-index-listen specific
125
+ TODO: Dockerfile shouldn't be index-on-publish specific
88
126
  -->
89
127
 
90
128
  Build the listener Dockerfile and tag it with the name `solr-listener`:
@@ -101,5 +139,5 @@ docker run --env-file <env-file> solr-listener
101
139
 
102
140
  ## Utility scripts
103
141
 
104
- - `bin/daylight-index-listen` script starts a consumer using the Daylight Solr handler.
105
- - `bin/simulate-publish-event` posts a publish event to the queue configured with `MqConnection`
142
+ - `bin/index-on-publish` script starts a consumer using `MetadataConsumer::Solr::MessageHandler`.
143
+ - `bin/simulate-publish-event` posts a publish event to the queue configured with `Mq::Connection`
@@ -0,0 +1,33 @@
1
+ #!/usr/bin/env ruby
2
+ require "bundler/setup"
3
+ require "logger"
4
+
5
+ require "surfliner/metadata_consumer"
6
+ require "surfliner/mq"
7
+
8
+ $stdout.sync = true # don't buffer log output
9
+ logger = Logger.new($stdout).tap do |logger|
10
+ logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
11
+ end
12
+
13
+ unless ENV["OTEL_SDK_DISABLED"] == "true"
14
+ require "opentelemetry/sdk"
15
+ require "opentelemetry-exporter-otlp"
16
+
17
+ otel_service_name = ENV.fetch('OTEL_SERVICE_NAME', File.basename(__FILE__))
18
+ otel_tracer_name = ENV.fetch('OTEL_TRACER_NAME', File.basename(__FILE__))
19
+
20
+ OpenTelemetry::SDK.configure do |c|
21
+ c.service_name = otel_service_name
22
+ c.use_all # enables auto instrumentation for Bunny, Net::HTTP, etc...
23
+ end
24
+
25
+ tracer = OpenTelemetry.tracer_provider.tracer(otel_tracer_name)
26
+ end
27
+
28
+ consumer = Surfliner::MetadataConsumer::Consumer.new(
29
+ connection: Surfliner::Mq::Connection.new(logger: logger),
30
+ tracer: (tracer if defined?(tracer)),
31
+ handler: Surfliner::MetadataConsumer::Solr::MessageHandler
32
+ )
33
+ consumer.run(topic_config: Surfliner::Mq::TopicConfig.from_env(auto_delete: true))
@@ -4,24 +4,24 @@ require "json"
4
4
  require "logger"
5
5
 
6
6
  require "surfliner/metadata_consumer"
7
+ require "surfliner/mq"
7
8
 
8
9
  module Surfliner
9
10
  module MetadataConsumer
10
11
  class Publisher
12
+ attr_reader :topic
13
+ attr_reader :api_base
14
+ attr_reader :routing_key
11
15
 
12
- attr_reader :logger
13
- attr_reader :topic_opts
14
- attr_reader :queue_opts
15
-
16
- def initialize(logger:, topic_opts: {}, queue_opts: {})
17
- @logger = logger
18
- @topic_opts = topic_opts
19
- @queue_opts = queue_opts
16
+ def initialize(topic, api_base:, routing_key:)
17
+ @topic = topic
18
+ @api_base = api_base
19
+ @routing_key = routing_key
20
20
  end
21
21
 
22
22
  def publish(resource_id)
23
23
  payload = payload_for(resource_id)
24
- connection.publish(payload)
24
+ topic.publish(payload, routing_key:)
25
25
  end
26
26
 
27
27
  def payload_for(id)
@@ -30,27 +30,37 @@ module Surfliner
30
30
  time_stamp: DateTime.now }.to_json
31
31
  end
32
32
 
33
- def connection
34
- @connection ||= MqConnection.new(logger:).connect(topic_opts:, queue_opts:)
35
- end
36
-
37
- def api_base
38
- @api_base ||= ENV.fetch("METADATA_API_URL_BASE", "http://metadata.test/resources")
33
+ class << self
34
+ # Connects with default (environment-configured) settings, yields a publisher
35
+ # for the specified topic, and closes the connection after the provided block
36
+ # completes.
37
+ #
38
+ # @param logger [Logger] the logger
39
+ # @param topic_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
40
+ #
41
+ # @yieldparam publisher [Publisher] the publisher
42
+ def connect(logger:, **topic_opts)
43
+ topic_config = Mq::TopicConfig.from_env(**topic_opts)
44
+ Mq::Connection.new(logger:).with_topic(topic_config) do |topic|
45
+ yield Publisher.new(
46
+ topic,
47
+ api_base: ENV.fetch("METADATA_API_URL_BASE", "http://metadata.test/resources"),
48
+ routing_key: ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
49
+ )
50
+ end
51
+ end
39
52
  end
40
53
  end
41
54
  end
42
55
  end
43
56
 
44
57
  $stdout.sync = true # don't buffer log output
45
- logger = Logger.new($stdout).tap do |logger|
58
+ logger = Logger.new($stdout).tap do |logger|
46
59
  logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
47
60
  end
48
61
 
49
- publisher = Surfliner::MetadataConsumer::Publisher.new(logger:)
50
- begin
62
+ Surfliner::MetadataConsumer::Publisher.connect(logger:, auto_delete: true) do |publisher|
51
63
  ARGV.each do |resource_id|
52
64
  publisher.publish(resource_id)
53
65
  end
54
- ensure
55
- publisher.connection.close
56
66
  end
@@ -1,5 +1,8 @@
1
1
  require "rsolr"
2
- require "surfliner/metadata_consumer/mq_connection"
2
+ require "surfliner/util/assert"
3
+ require "surfliner/mq/connection"
4
+ require "surfliner/mq/topic_config"
5
+ require "surfliner/mq/queue_config"
3
6
  require "surfliner/metadata_consumer/solr/message_handler"
4
7
 
5
8
  module Surfliner
@@ -7,29 +10,40 @@ module Surfliner
7
10
  # A metadata consumer that subscribes to a RabbitMQ queue and passes
8
11
  # messages to the specified handler.
9
12
  class Consumer
10
- attr_reader :connection, :logger, :tracer, :handler
13
+ include Util::Assert
14
+
15
+ # @return [Mq::Connection] the connection
16
+ attr_reader :connection
17
+
18
+ # @return [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
19
+ attr_reader :tracer
20
+
21
+ # @return [#handle] an object accepting a JSON string
22
+ attr_reader :handler
11
23
 
12
24
  # Initializes a new `Consumer`
25
+ #
26
+ # @param connection [Mq::Connection] the connection
27
+ # @param handler [#handle] an object accepting a JSON string
13
28
  # @param tracer [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
14
- # @param logger [Logger] log message destination
15
- # @param handler #handle an object accepting a JSON string
16
- def initialize(tracer:, logger:, handler:)
17
- @connection = MqConnection.new(logger:)
18
- @logger = logger
29
+ def initialize(connection:, handler:, tracer: nil)
30
+ @connection = not_nil!(connection, "connection")
31
+ @handler = not_nil!(handler, "handler")
19
32
  @tracer = tracer
20
- @handler = handler
33
+ end
34
+
35
+ # @return [Logger] log message destination
36
+ def logger
37
+ connection.logger
21
38
  end
22
39
 
23
40
  # Starts listening to the message queue and passing messages to the handler.
24
- # @param queue_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
25
- # @param topic_opts [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
26
- def run(queue_opts: {}, topic_opts: {})
27
- connection.open(queue_opts:, topic_opts:) do |queue|
28
- queue.subscribe(block: true) do |_delivery_info, _properties, payload_json|
29
- tracer.in_span("surfliner metadata consumer message") do |_span|
30
- handle(payload_json)
31
- end
32
- end
41
+ # @param topic_config [TopicConfig] topic configuration
42
+ # @param queue_config [QueueConfig] queue configuration
43
+ def run(topic_config: Mq::TopicConfig.from_env, queue_config: Mq::QueueConfig.from_env)
44
+ connection.with_topic(topic_config) do |topic|
45
+ queue = topic.bind_queue(queue_config)
46
+ queue.subscribe(block: true, &method(:on_delivery))
33
47
  end
34
48
  end
35
49
 
@@ -43,6 +57,16 @@ module Surfliner
43
57
  rescue => err
44
58
  logger.error(" [!] failed to handle message: #{err.full_message}")
45
59
  end
60
+
61
+ private
62
+
63
+ def on_delivery(_delivery_info, _properties, payload_json)
64
+ return handle(payload_json) unless tracer
65
+
66
+ tracer.in_span("surfliner metadata consumer message") do |_span, _context|
67
+ handle(payload_json)
68
+ end
69
+ end
46
70
  end
47
71
  end
48
72
  end
@@ -1,8 +1,7 @@
1
1
  # Umbrella module for general Surfliner code
2
2
  module Surfliner
3
- # Parent module for this gem
4
3
  module MetadataConsumer
5
4
  # The gem version
6
- VERSION = "0.1.0.pre.alpha.3"
5
+ VERSION = "0.1.0.pre.alpha.5"
7
6
  end
8
7
  end
@@ -1 +1,4 @@
1
- Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).sort.each(&method(:require))
1
+ Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).each(&method(:require))
2
+
3
+ # Functionality specific to consuming RabbitMQ feeds
4
+ module Surfliner::MetadataConsumer; end
@@ -0,0 +1,127 @@
1
+ require "bunny"
2
+ require "surfliner/util/assert"
3
+
4
+ module Surfliner
5
+ module Mq
6
+ # An object encapsulating a RabbitMQ connection.
7
+ class Connection
8
+ include Util::Assert
9
+
10
+ # @return [Logger] The logger
11
+ attr_reader :logger
12
+
13
+ # @return [Bunny::Session] The current RabbitMQ session
14
+ attr_reader :session
15
+
16
+ # @return [Bunny::Channel] The channel being listened to
17
+ attr_reader :channel
18
+
19
+ # @return [ConnectionConfig] The configuration
20
+ attr_reader :config
21
+
22
+ # Initializes a new `MqConnection`.
23
+ #
24
+ # @param logger [Logger] the logger
25
+ # @param config [ConnectionConfig] the configuration
26
+ def initialize(logger:, config: ConnectionConfig.from_env)
27
+ @logger = not_nil!(logger)
28
+ @config = not_nil!(config)
29
+ end
30
+
31
+ # Opens a connection.
32
+ # @return [self]
33
+ # @raise [IOError] if already connected
34
+ def connect
35
+ raise IOError, "RabbitMQ session #{session} already open." if open?
36
+
37
+ logger.info("Rabbitmq message broker session url: #{config.redacted_url}")
38
+ @session = Bunny.new(config.session_url, logger: logger)
39
+ connect_on(session)
40
+ @channel = session.create_channel
41
+
42
+ self
43
+ rescue Bunny::TCPConnectionFailed => err
44
+ # TODO: realistically, this only happens in session.start, where we're eating it
45
+ logger.error("Connection to #{config.redacted_url} failed")
46
+ raise err
47
+ rescue Bunny::PossibleAuthenticationFailureError => err
48
+ # TODO: realistically, this only happens in session.start, where we're eating it
49
+ logger.error("Failed to authenticate to #{config.redacted_url}")
50
+ raise err
51
+ end
52
+
53
+ # Opens a session, yields a client for the specified topic, and closes the
54
+ # session after the provided block completes.
55
+ # @param config [TopicConfig] topic configuration
56
+ # @yield [MqTopic] A client for the topic
57
+ def with_topic(config = TopicConfig.from_env)
58
+ connect
59
+
60
+ yield topic_from(config)
61
+ ensure
62
+ close
63
+ end
64
+
65
+ # Returns a client for the specified topic. Note that this does _not_ open
66
+ # or close the connection.
67
+ #
68
+ # @param config [TopicConfig] topic configuration
69
+ # @return [Topic] A client for the topic
70
+ # @raise [IOError] if the connection is not open
71
+ def topic_from(config)
72
+ raise IOError, "RabbitMQ session not open" unless open?
73
+
74
+ Topic.from_config(config, channel:, logger:)
75
+ end
76
+
77
+ # Closes the session.
78
+ def close
79
+ return unless channel
80
+ return if channel.closed?
81
+ logger.info("closing channel")
82
+ channel.close
83
+ ensure
84
+ logger.info("closing session")
85
+ session&.close
86
+ end
87
+
88
+ # @return [true, false] True if the session is open, false otherwise
89
+ def open?
90
+ session&.status == :open
91
+ end
92
+
93
+ # @return [Symbol, nil] The session status, or nil if there is no session
94
+ def status
95
+ session&.status
96
+ end
97
+
98
+ # @return [String] The RabbitMQ hostname
99
+ def host
100
+ config.host
101
+ end
102
+
103
+ # @return [String] The RabbitMQ port
104
+ def port
105
+ config.port
106
+ end
107
+
108
+ private
109
+
110
+ def connect_on(session, timeout = 120)
111
+ timer = 0
112
+ logger.info "Trying to open queue session with timeout=#{timeout}"
113
+ while timer < timeout
114
+ begin
115
+ session.start
116
+ rescue
117
+ # TODO: do we actually want to rescue from everything?
118
+ end
119
+ return session if session.status == :open
120
+ sleep 1
121
+ timer += 1
122
+ end
123
+ raise "Failed to connect to queue."
124
+ end
125
+ end
126
+ end
127
+ end