surfliner-metadata_consumer 0.1.0.pre.alpha.4 → 0.1.0.pre.alpha.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 832b89f48813e1a03fa41326e48e32030ecdd20f1f1719705b9fb87b6883b17e
-  data.tar.gz: 406fc690ac6d2a771737953d53babf83f52406b9870b5837653e02357083bb1b
+  metadata.gz: 25566009d8ccce03904d9020a0340511bbeb2fac08b5151c65705cd2d6117096
+  data.tar.gz: 24136a3ca25672bf7fcc8670ea61f11a34c547b515022f92e46b440fe1d315cc
 SHA512:
-  metadata.gz: 5f6fda0ab5aa1cc54053bd688ef353d3d303adb27a3820aef8506c90357860592fae98b201622b82513ba501b111dc35c52379b434ed6082a1b73edb4acef88b
-  data.tar.gz: 8106c7d167c2bcd1046c403d9a171f9b8c1c2d72651d002f08a828326f993cce7f45863e3e7db32928eeb295bdf58ee947303a502eb69366522a30ed14262dad
+  metadata.gz: fef5c2196922e8f5476422ffd83400bc17edc957b0822a323628dd9cd525440e231e1d7402b761c71894806282c2ad11c8c450eb6dc37a6a6e99dac7d3adc9c5
+  data.tar.gz: 5014bdadb04910ca9888d8fe445342641462e33f6ec445333c4502dd322e428df5c5ecd01710018d7f905e80a889ce84815f7ff59b1d3908c00b405d5f02c048

data/CHANGES.md

@@ -1,3 +1,48 @@
+# 0.1.0.pre.alpha.6 (2025-04-25)
+
+- Update to Bunny 2.24
+- Allow `Mq::Connection#connect` to take a block, in which case the method
+  yields the open connection, closing the connection and channel afterwards.
+- Add an option `await_response_on_close` to `Mq::ConnectionConfig`, defaulting
+  to true, to indicate whether the RabbitMQ client should wait for a response
+  when closing a session, along with a corresponding environment variable
+  `RABBITMQ_AWAIT_ON_CLOSE`
+- Allows passing additional options to `Mq:ConnectionConfig#initialize` and
+  `#from_env`, which will then be passed to `Bunny::Session#initialize`
+
+# 0.1.0.pre.alpha.5 (2025-03-25)
+
+In order to support using one connection (and one underlying `Bunny::Session`) 
+with multiple topics, queues, and/or routing keys, as well as publishing to
+a topic without creating or subscribing to a queue, this release introduces
+the following **breaking changes**:
+
+- RabbitMQ-related configuration and connection code (i.e. not specific to
+  the consumer/handler setup) has been moved from `Surfliner::MetadataConsumer`
+  to `Surfliner::Mq`. This may move to a separate gem in a future release.
+  - `MetadataConsumer::MqConnection` is now `Mq::Connection`
+  - `MetadataConsumer::MqConfig` is now `Mq::ConnectionConfig`
+  - `ConnectionConfig` no longer includes topic or queue configuration;
+     topics are configured with `TopicConfig` and queues with
+     `QueueConfig`.
+- The `MqConnection#open` method, which yielded a `Bunny::Queue`, has been
+  replaced by the `Connection#with_topic` method, which yields an
+  `Mq::Topic` object wrapping a single topic; call `Mq::Topic#bind_queue`
+  to bind to a queue. Like the removed`#open`, `#with_topic` opens and
+  closes the connection implicitly and allows connecting only to a single
+  topic.
+- The `Mq::Connection#topic_from(config)` method allows you to connect to
+  any number of topics, but you must open and close the connection
+  explicitly.
+- The `MqConnection#publish` method has been moved to `Mq::Topic` and can
+  now take an explicit routing key, although if one is not provided it will
+  still default to `ENV["RABBITMQ_PLATFORM_ROUTING_KEY"]`.
+- `MetadataConsumer::Consumer` now requires an explicit `Mq:Connection`
+  instead of creating one by default. 
+
+In addition, the `daylight-index-listen` script has been renamed to the more
+generic `index-on-publish`.
+
 # 0.1.0.pre.alpha.4 (2025-03-17)
 
 - lower required Ruby version to 3.2
@@ -19,4 +64,4 @@
 
 # 0.1.0.pre.alpha (2025-02-12)
 
-- Initial release
+- Initial release

data/Dockerfile

@@ -56,4 +56,4 @@ COPY --from=build $APP_PATH/Gemfile.lock $APP_PATH
 RUN bundle config set frozen 'true'
 
 # TODO: don't hard-code a specific handler / script
-CMD ["./bin/daylight-index-listen"]
+CMD ["./bin/index-on-publish"]

data/Gemfile.lock

@@ -1,8 +1,8 @@
 PATH
   remote: .
   specs:
-    surfliner-metadata_consumer (0.1.0.pre.alpha.4)
-      bunny (~> 2.23)
+    surfliner-metadata_consumer (0.1.0.pre.alpha.6)
+      bunny (~> 2.24)
       opentelemetry-exporter-otlp (~> 0.26.3)
       opentelemetry-instrumentation-all (~> 0.60.0)
       opentelemetry-sdk (~> 1.4.1)
@@ -14,11 +14,11 @@ GEM
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     amq-protocol (2.3.3)
-    ast (2.4.2)
+    ast (2.4.3)
     bigdecimal (3.1.9)
     builder (3.3.0)
-    bunny (2.23.0)
-      amq-protocol (~> 2.3, >= 2.3.1)
+    bunny (2.24.0)
+      amq-protocol (~> 2.3)
       sorted_set (~> 1, >= 1.0.2)
     ci_reporter (2.1.0)
       builder (>= 2.1.2)
@@ -34,28 +34,29 @@ GEM
     debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
-    diff-lcs (1.6.0)
+    diff-lcs (1.6.1)
     docile (1.4.1)
     dotenv (2.8.1)
-    faraday (2.12.2)
+    faraday (2.13.1)
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
     faraday-net_http (3.4.0)
       net-http (>= 0.5.0)
-    google-protobuf (3.25.6)
-    googleapis-common-protos-types (1.18.0)
+    github-markup (5.0.1)
+    google-protobuf (3.25.7)
+    googleapis-common-protos-types (1.19.0)
       google-protobuf (>= 3.18, < 5.a)
     hashdiff (1.1.2)
     io-console (0.8.0)
-    irb (1.15.1)
+    irb (1.15.2)
       pp (>= 0.6.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
-    json (2.10.2)
+    json (2.11.3)
     language_server-protocol (3.17.0.4)
     lint_roller (1.1.0)
-    logger (1.6.6)
+    logger (1.7.0)
     net-http (0.6.0)
       uri
     opentelemetry-api (1.5.0)
@@ -254,13 +255,14 @@ GEM
       opentelemetry-semantic_conventions
     opentelemetry-semantic_conventions (1.11.0)
       opentelemetry-api (~> 1.0)
-    parallel (1.26.3)
-    parser (3.3.7.1)
+    parallel (1.27.0)
+    parser (3.3.8.0)
       ast (~> 2.4.1)
       racc
     pp (0.6.2)
       prettyprint
     prettyprint (0.2.0)
+    prism (1.4.0)
     psych (5.2.3)
       date
       stringio
@@ -269,10 +271,11 @@ GEM
     rainbow (3.1.1)
     rake (13.2.1)
     rbtree (0.4.6)
-    rdoc (6.12.0)
+    rdoc (6.13.1)
       psych (>= 4.0.0)
+    redcarpet (3.6.1)
     regexp_parser (2.10.0)
-    reline (0.6.0)
+    reline (0.6.1)
       io-console (~> 0.5)
     rexml (3.4.1)
     rsolr (2.6.0)
@@ -291,7 +294,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
     rspec-support (3.13.2)
-    rubocop (1.73.2)
+    rubocop (1.75.3)
       json (~> 2.3)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.1.0)
@@ -299,14 +302,15 @@ GEM
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 2.9.3, < 3.0)
-      rubocop-ast (>= 1.38.0, < 2.0)
+      rubocop-ast (>= 1.44.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.39.0)
-      parser (>= 3.3.1.0)
-    rubocop-performance (1.24.0)
+    rubocop-ast (1.44.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-performance (1.25.0)
       lint_roller (~> 1.1)
-      rubocop (>= 1.72.1, < 2.0)
+      rubocop (>= 1.75.0, < 2.0)
       rubocop-ast (>= 1.38.0, < 2.0)
     ruby-progressbar (1.13.0)
     set (1.1.1)
@@ -322,19 +326,19 @@ GEM
     sorted_set (1.0.3)
       rbtree
       set (~> 1.0)
-    standard (1.47.0)
+    standard (1.49.0)
       language_server-protocol (~> 3.17.0.2)
       lint_roller (~> 1.0)
-      rubocop (~> 1.73.0)
+      rubocop (~> 1.75.2)
       standard-custom (~> 1.0.0)
-      standard-performance (~> 1.7)
+      standard-performance (~> 1.8)
     standard-custom (1.0.2)
       lint_roller (~> 1.0)
       rubocop (~> 1.50)
-    standard-performance (1.7.0)
+    standard-performance (1.8.0)
       lint_roller (~> 1.1)
-      rubocop-performance (~> 1.24.0)
-    stringio (3.1.5)
+      rubocop-performance (~> 1.25.0)
+    stringio (3.1.7)
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
@@ -354,7 +358,9 @@ DEPENDENCIES
   colorize (~> 0.8)
   debug (~> 1.9.2)
   dotenv (~> 2.7)
+  github-markup (~> 5.0)
   rake (~> 13.0)
+  redcarpet (~> 3.6)
   rspec (~> 3.13)
   simplecov (~> 0.22)
   simplecov-cobertura (~> 2.1)
@@ -364,4 +370,4 @@ DEPENDENCIES
   yard (~> 0.9.37)
 
 BUNDLED WITH
-   2.6.2
+   2.6.7

data/README.md

@@ -36,23 +36,60 @@ if prior steps succeed.
 
 ## Configuration
 
-<!--
-TODO: Just provide a sample env file
--->
+> Note: Sample values taken from Surfliner's development 
+> [daylight-listener.sh](https://gitlab.com/surfliner/surfliner/-/blob/trunk/docker-compose/env/daylight-listener.sh).)
+
+An `Mq::Connection` object, suitable for listening or publishing, can be configured
+explicitly with a `ConnectionConfig` object, or implicitly, reading a default 
+configuration with `ConnectionConfig#from_env`. `ConnectionConfig#from_env` expects
+the following environment variables:
+
+| Variable                    | Sample value | Description                                               |
+|-----------------------------|--------------|-----------------------------------------------------------|
+| `RABBITMQ_HOST`             | `rabbitmq`   | Hostname of RabbitMQ server                               |
+| `RABBITMQ_NODE_PORT_NUMBER` | `5672`       | Port name of RabbitMQ server                              |
+| `RABBITMQ_USERNAME`         | `user`       | RabbitMQ username                                         |
+| `RABBITMQ_PASSWORD`         | `bitnami`    | RabbitMQ password                                         |
+| `RABBITMQ_AWAIT_ON_CLOSE`   | `false`      | Whether to wait on response when closing (default = true) |
+
+The `Mq::Connection#with_topic` and `Mq::Connection#topic_from` can either take an
+explicit `TopicConfig` object, or implicitly read a default configuration with
+`TopicConfig#from_env`. `TopicConfig#from_env` expects the following environment variables:
+
+| Variable         | Sample value         | Description         |
+|------------------|----------------------|---------------------|
+| `RABBITMQ_TOPIC` | `surfliner.metadata` | RabbitMQ topic name |
+
+Note that `TopicConfig#from_env` also accepts keyword options, which are passed through
+to [`Bunny::Channel#topic`](https://api.rubybunny.info/Bunny/Channel.html#topic-instance_method).
+E.g.
+
+```ruby
+topic_config = TopicConfig.from_env(durable: true, auto_delete: true)
+connection.with_topic(topic_config) do |topic|
+  # ...
+end
+```
 
-The `MqConnection` class expects the following environment variables:
+Similarly, `Mq::Topic#bind_queue` can either take an explicit `QueueConfig`, or implicitly
+read a default with `QueConfig#from_env`. `QueueConfig#from_env` expects the following environment
+variables:
 
-```sh
-RABBITMQ_HOST=rabbitmq
-RABBITMQ_NODE_PORT_NUMBER=5672
-RABBITMQ_PASSWORD=bitnami
-RABBITMQ_QUEUE=surfliner.metadata
-RABBITMQ_PLATFORM_ROUTING_KEY=surfliner.metadata.daylight
-RABBITMQ_TOPIC=surfliner.metadata
-RABBITMQ_USERNAME=user
-```
+| Variable                        | Sample value                  | Description          | 
+|---------------------------------|-------------------------------|----------------------|
+| `RABBITMQ_QUEUE`                | `surfliner.metadata`          | RabbitMQ queue name  |
+| `RABBITMQ_PLATFORM_ROUTING_KEY` | `surfliner.metadata.daylight` | RabbitMQ routing key |
 
-(Sample values taken from Surfliner's development [daylight-listener.sh](https://gitlab.com/surfliner/surfliner/-/blob/trunk/docker-compose/env/daylight-listener.sh).)
+And `QueueConfig#from_env` similarly accepts keyword options, which are forwarded to
+[`Bunny::Channel#queue`](https://api.rubybunny.info/Bunny/Channel.html#queue-instance_method):
+
+```ruby
+connection.with_topic(topic_config) do |topic|
+  queue_config = QueueConfig.from_env(exclusive: true, durable: true)
+  queue = topic.bind_queue(queue_config)
+  # ...
+end
+```
 
 The Solr / Daylight handler implementation (see below) additionally expects
 a configured `SOLR_URL`, e.g.:
@@ -74,17 +111,19 @@ The `Surfliner::MetadataConsumer::Solr` package contains a handler that retrieve
 metadata for `:published`/`:updated` events, transforms results for indexing, and 
 updates a target Solr index.
 
-<!-- TODO: better name for this script -->
+The `bin/index-on-publish` script starts a consumer using this handler, and accepts
+(but does not require) the following environment variables:
 
-The `bin/daylight-index-listen` script starts a consumer using this handler.
-
-In addition to the environment variables listed above, this handler expects
-a configured `SOLR_URL`, e.g.:
+| Variable            | Sample value                  | Default        | Description                      |
+|---------------------|-------------------------------|----------------|----------------------------------|
+| `OTEL_SDK_DISABLED` | `true`                        | (not set)      | Whether to disable OpenTelemetry |
+| `OTEL_SERVICE_NAME` | `surfliner-daylight-consumer` | name of script | OpenTelemetry service name       |
+| `OTEL_TRACER_NAME`  | `DaylightConsumerTracer`      | name of script | OpenTelemetry tracer name        |
 
 ### Running the service
 
 <!--
-TODO: Dockerfile shouldn't be daylight-index-listen specific
+TODO: Dockerfile shouldn't be index-on-publish specific
 -->
 
 Build the listener Dockerfile and tag it with the name `solr-listener`:
@@ -101,5 +140,5 @@ docker run --env-file <env-file> solr-listener
 
 ## Utility scripts
 
-- `bin/daylight-index-listen` script starts a consumer using the Daylight Solr handler.
-- `bin/simulate-publish-event` posts a publish event to the queue configured with `MqConnection`
+- `bin/index-on-publish` script starts a consumer using `MetadataConsumer::Solr::MessageHandler`.
+- `bin/simulate-publish-event` posts a publish event to the queue configured with `Mq::Connection`

data/bin/index-on-publish

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require "bundler/setup"
+require "logger"
+
+require "surfliner/metadata_consumer"
+require "surfliner/mq"
+
+$stdout.sync = true # don't buffer log output
+logger       = Logger.new($stdout).tap do |logger|
+  logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
+end
+
+unless ENV["OTEL_SDK_DISABLED"] == "true"
+  require "opentelemetry/sdk"
+  require "opentelemetry-exporter-otlp"
+
+  otel_service_name = ENV.fetch('OTEL_SERVICE_NAME', File.basename(__FILE__))
+  otel_tracer_name  = ENV.fetch('OTEL_TRACER_NAME', File.basename(__FILE__))
+
+  OpenTelemetry::SDK.configure do |c|
+    c.service_name = otel_service_name
+    c.use_all # enables auto instrumentation for Bunny, Net::HTTP, etc...
+  end
+
+  tracer = OpenTelemetry.tracer_provider.tracer(otel_tracer_name)
+end
+
+consumer = Surfliner::MetadataConsumer::Consumer.new(
+  connection: Surfliner::Mq::Connection.new(logger: logger),
+  tracer:     (tracer if defined?(tracer)),
+  handler:    Surfliner::MetadataConsumer::Solr::MessageHandler
+)
+consumer.run(topic_config: Surfliner::Mq::TopicConfig.from_env(auto_delete: true))

data/bin/simulate-publish-event

@@ -4,24 +4,24 @@ require "json"
 require "logger"
 
 require "surfliner/metadata_consumer"
+require "surfliner/mq"
 
 module Surfliner
   module MetadataConsumer
     class Publisher
+      attr_reader :topic
+      attr_reader :api_base
+      attr_reader :routing_key
 
-      attr_reader :logger
-      attr_reader :topic_opts
-      attr_reader :queue_opts
-
-      def initialize(logger:, topic_opts: {}, queue_opts: {})
-        @logger = logger
-        @topic_opts = topic_opts
-        @queue_opts = queue_opts
+      def initialize(topic, api_base:, routing_key:)
+        @topic       = topic
+        @api_base    = api_base
+        @routing_key = routing_key
       end
 
       def publish(resource_id)
         payload = payload_for(resource_id)
-        connection.publish(payload)
+        topic.publish(payload, routing_key:)
       end
 
       def payload_for(id)
@@ -30,27 +30,37 @@ module Surfliner
           time_stamp:  DateTime.now }.to_json
       end
 
-      def connection
-        @connection ||= MqConnection.new(logger:).connect(topic_opts:, queue_opts:)
-      end
-
-      def api_base
-        @api_base ||= ENV.fetch("METADATA_API_URL_BASE", "http://metadata.test/resources")
+      class << self
+        # Connects with default (environment-configured) settings, yields a publisher
+        # for the specified topic, and closes the connection after the provided block
+        # completes.
+        #
+        # @param logger [Logger] the logger
+        # @param topic_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+        #
+        # @yieldparam publisher [Publisher] the publisher
+        def connect(logger:, **topic_opts)
+          topic_config = Mq::TopicConfig.from_env(**topic_opts)
+          Mq::Connection.new(logger:).with_topic(topic_config) do |topic|
+            yield Publisher.new(
+              topic,
+              api_base:    ENV.fetch("METADATA_API_URL_BASE", "http://metadata.test/resources"),
+              routing_key: ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
+            )
+          end
+        end
       end
     end
   end
 end
 
 $stdout.sync = true # don't buffer log output
-logger = Logger.new($stdout).tap do |logger|
+logger       = Logger.new($stdout).tap do |logger|
   logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
 end
 
-publisher = Surfliner::MetadataConsumer::Publisher.new(logger:)
-begin
+Surfliner::MetadataConsumer::Publisher.connect(logger:, auto_delete: true) do |publisher|
   ARGV.each do |resource_id|
     publisher.publish(resource_id)
   end
-ensure
-  publisher.connection.close
 end

data/lib/surfliner/metadata_consumer/consumer.rb

@@ -1,5 +1,8 @@
 require "rsolr"
-require "surfliner/metadata_consumer/mq_connection"
+require "surfliner/util/assert"
+require "surfliner/mq/connection"
+require "surfliner/mq/topic_config"
+require "surfliner/mq/queue_config"
 require "surfliner/metadata_consumer/solr/message_handler"
 
 module Surfliner
@@ -7,29 +10,40 @@ module Surfliner
     # A metadata consumer that subscribes to a RabbitMQ queue and passes
     # messages to the specified handler.
     class Consumer
-      attr_reader :connection, :logger, :tracer, :handler
+      include Util::Assert
+
+      # @return [Mq::Connection] the connection
+      attr_reader :connection
+
+      # @return [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
+      attr_reader :tracer
+
+      # @return [#handle] an object accepting a JSON string
+      attr_reader :handler
 
       # Initializes a new `Consumer`
+      #
+      # @param connection [Mq::Connection] the connection
+      # @param handler [#handle] an object accepting a JSON string
       # @param tracer [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
-      # @param logger [Logger] log message destination
-      # @param handler #handle an object accepting a JSON string
-      def initialize(tracer:, logger:, handler:)
-        @connection = MqConnection.new(logger:)
-        @logger = logger
+      def initialize(connection:, handler:, tracer: nil)
+        @connection = not_nil!(connection, "connection")
+        @handler = not_nil!(handler, "handler")
         @tracer = tracer
-        @handler = handler
+      end
+
+      # @return [Logger] log message destination
+      def logger
+        connection.logger
       end
 
       # Starts listening to the message queue and passing messages to the handler.
-      # @param queue_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
-      # @param topic_opts [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
-      def run(queue_opts: {}, topic_opts: {})
-        connection.open(queue_opts:, topic_opts:) do |queue|
-          queue.subscribe(block: true) do |_delivery_info, _properties, payload_json|
-            tracer.in_span("surfliner metadata consumer message") do |_span|
-              handle(payload_json)
-            end
-          end
+      # @param topic_config [TopicConfig] topic configuration
+      # @param queue_config [QueueConfig] queue configuration
+      def run(topic_config: Mq::TopicConfig.from_env, queue_config: Mq::QueueConfig.from_env)
+        connection.with_topic(topic_config) do |topic|
+          queue = topic.bind_queue(queue_config)
+          queue.subscribe(block: true, &method(:on_delivery))
         end
       end
 
@@ -43,6 +57,16 @@ module Surfliner
       rescue => err
         logger.error(" [!] failed to handle message: #{err.full_message}")
       end
+
+      private
+
+      def on_delivery(_delivery_info, _properties, payload_json)
+        return handle(payload_json) unless tracer
+
+        tracer.in_span("surfliner metadata consumer message") do |_span, _context|
+          handle(payload_json)
+        end
+      end
     end
   end
 end

data/lib/surfliner/metadata_consumer/version.rb

@@ -1,8 +1,7 @@
 # Umbrella module for general Surfliner code
 module Surfliner
-  # Parent module for this gem
   module MetadataConsumer
     # The gem version
-    VERSION = "0.1.0.pre.alpha.4"
+    VERSION = "0.1.0.pre.alpha.6"
   end
 end

data/lib/surfliner/metadata_consumer.rb

@@ -1 +1,4 @@
-Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).sort.each(&method(:require))
+Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).each(&method(:require))
+
+# Functionality specific to consuming RabbitMQ feeds
+module Surfliner::MetadataConsumer; end