surfliner-metadata_consumer 0.1.0.pre.alpha.4 → 0.1.0.pre.alpha.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 832b89f48813e1a03fa41326e48e32030ecdd20f1f1719705b9fb87b6883b17e
-  data.tar.gz: 406fc690ac6d2a771737953d53babf83f52406b9870b5837653e02357083bb1b
+  metadata.gz: 2fff0fe7a29854ecb3b75f91ed5aa1e5abdbf6fe7fc625b4b117a3e2c4e4805c
+  data.tar.gz: 534536637b909d309f90095aaba4e9cb51a75d4ca92a78531971d3550ea42bfc
 SHA512:
-  metadata.gz: 5f6fda0ab5aa1cc54053bd688ef353d3d303adb27a3820aef8506c90357860592fae98b201622b82513ba501b111dc35c52379b434ed6082a1b73edb4acef88b
-  data.tar.gz: 8106c7d167c2bcd1046c403d9a171f9b8c1c2d72651d002f08a828326f993cce7f45863e3e7db32928eeb295bdf58ee947303a502eb69366522a30ed14262dad
+  metadata.gz: 6e20f325aa14042b5ed7e950803d8a045834df543da1f5c4a8f30297645015d97add22a5de38e01d15a6ab74adb856777bba4fccfb609d50bd622c9fae1f51d5
+  data.tar.gz: a69109802f8c408f077b2fc2d1c748c02a900eda042d6e40a4423ee3257b39d2dde892eecf9c77c690df06aa53d367136a8246f3d8bc69114d0e1278326b4531

data/CHANGES.md

@@ -1,3 +1,36 @@
+# 0.1.0.pre.alpha.5 (2025-03-25)
+
+In order to support using one connection (and one underlying `Bunny::Session`) 
+with multiple topics, queues, and/or routing keys, as well as publishing to
+a topic without creating or subscribing to a queue, this release introduces
+the following **breaking changes**:
+
+- RabbitMQ-related configuration and connection code (i.e. not specific to
+  the consumer/handler setup) has been moved from `Surfliner::MetadataConsumer`
+  to `Surfliner::Mq`. This may move to a separate gem in a future release.
+  - `MetadataConsumer::MqConnection` is now `Mq::Connection`
+  - `MetadataConsumer::MqConfig` is now `Mq::ConnectionConfig`
+  - `ConnectionConfig` no longer includes topic or queue configuration;
+     topics are configured with `TopicConfig` and queues with
+     `QueueConfig`.
+- The `MqConnection#open` method, which yielded a `Bunny::Queue`, has been
+  replaced by the `Connection#with_topic` method, which yields an
+  `Mq::Topic` object wrapping a single topic; call `Mq::Topic#bind_queue`
+  to bind to a queue. Like the removed`#open`, `#with_topic` opens and
+  closes the connection implicitly and allows connecting only to a single
+  topic.
+- The `Mq::Connection#topic_from(config)` method allows you to connect to
+  any number of topics, but you must open and close the connection
+  explicitly.
+- The `MqConnection#publish` method has been moved to `Mq::Topic` and can
+  now take an explicit routing key, although if one is not provided it will
+  still default to `ENV["RABBITMQ_PLATFORM_ROUTING_KEY"]`.
+- `MetadataConsumer::Consumer` now requires an explicit `Mq:Connection`
+  instead of creating one by default. 
+
+In addition, the `daylight-index-listen` script has been renamed to the more
+generic `index-on-publish`.
+
 # 0.1.0.pre.alpha.4 (2025-03-17)
 
 - lower required Ruby version to 3.2
@@ -19,4 +52,4 @@
 
 # 0.1.0.pre.alpha (2025-02-12)
 
-- Initial release
+- Initial release

data/Dockerfile

@@ -56,4 +56,4 @@ COPY --from=build $APP_PATH/Gemfile.lock $APP_PATH
 RUN bundle config set frozen 'true'
 
 # TODO: don't hard-code a specific handler / script
-CMD ["./bin/daylight-index-listen"]
+CMD ["./bin/index-on-publish"]

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    surfliner-metadata_consumer (0.1.0.pre.alpha.4)
+    surfliner-metadata_consumer (0.1.0.pre.alpha.5)
       bunny (~> 2.23)
       opentelemetry-exporter-otlp (~> 0.26.3)
       opentelemetry-instrumentation-all (~> 0.60.0)
@@ -14,11 +14,11 @@ GEM
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
     amq-protocol (2.3.3)
-    ast (2.4.2)
+    ast (2.4.3)
     bigdecimal (3.1.9)
     builder (3.3.0)
-    bunny (2.23.0)
-      amq-protocol (~> 2.3, >= 2.3.1)
+    bunny (2.24.0)
+      amq-protocol (~> 2.3)
       sorted_set (~> 1, >= 1.0.2)
     ci_reporter (2.1.0)
       builder (>= 2.1.2)
@@ -34,7 +34,7 @@ GEM
     debug (1.9.2)
       irb (~> 1.10)
       reline (>= 0.3.8)
-    diff-lcs (1.6.0)
+    diff-lcs (1.6.1)
     docile (1.4.1)
     dotenv (2.8.1)
     faraday (2.12.2)
@@ -43,8 +43,9 @@ GEM
       logger
     faraday-net_http (3.4.0)
       net-http (>= 0.5.0)
+    github-markup (5.0.1)
     google-protobuf (3.25.6)
-    googleapis-common-protos-types (1.18.0)
+    googleapis-common-protos-types (1.19.0)
       google-protobuf (>= 3.18, < 5.a)
     hashdiff (1.1.2)
     io-console (0.8.0)
@@ -255,12 +256,13 @@ GEM
     opentelemetry-semantic_conventions (1.11.0)
       opentelemetry-api (~> 1.0)
     parallel (1.26.3)
-    parser (3.3.7.1)
+    parser (3.3.7.2)
       ast (~> 2.4.1)
       racc
     pp (0.6.2)
       prettyprint
     prettyprint (0.2.0)
+    prism (1.4.0)
     psych (5.2.3)
       date
       stringio
@@ -269,8 +271,9 @@ GEM
     rainbow (3.1.1)
     rake (13.2.1)
     rbtree (0.4.6)
-    rdoc (6.12.0)
+    rdoc (6.13.0)
       psych (>= 4.0.0)
+    redcarpet (3.6.1)
     regexp_parser (2.10.0)
     reline (0.6.0)
       io-console (~> 0.5)
@@ -302,8 +305,9 @@ GEM
       rubocop-ast (>= 1.38.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 2.4.0, < 4.0)
-    rubocop-ast (1.39.0)
-      parser (>= 3.3.1.0)
+    rubocop-ast (1.43.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
     rubocop-performance (1.24.0)
       lint_roller (~> 1.1)
       rubocop (>= 1.72.1, < 2.0)
@@ -334,7 +338,7 @@ GEM
     standard-performance (1.7.0)
       lint_roller (~> 1.1)
       rubocop-performance (~> 1.24.0)
-    stringio (3.1.5)
+    stringio (3.1.6)
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
@@ -354,7 +358,9 @@ DEPENDENCIES
   colorize (~> 0.8)
   debug (~> 1.9.2)
   dotenv (~> 2.7)
+  github-markup (~> 5.0)
   rake (~> 13.0)
+  redcarpet (~> 3.6)
   rspec (~> 3.13)
   simplecov (~> 0.22)
   simplecov-cobertura (~> 2.1)

data/README.md

@@ -36,23 +36,59 @@ if prior steps succeed.
 
 ## Configuration
 
-<!--
-TODO: Just provide a sample env file
--->
+> Note: Sample values taken from Surfliner's development 
+> [daylight-listener.sh](https://gitlab.com/surfliner/surfliner/-/blob/trunk/docker-compose/env/daylight-listener.sh).)
+
+An `Mq::Connection` object, suitable for listening or publishing, can be configured
+explicitly with a `ConnectionConfig` object, or implicitly, reading a default 
+configuration with `ConnectionConfig#from_env`. `ConnectionConfig#from_env` expects
+the following environment variables:
+
+| Variable                    | Sample value | Description                  |
+|-----------------------------|--------------|------------------------------|
+| `RABBITMQ_HOST`             | `rabbitmq`   | Hostname of RabbitMQ server  |
+| `RABBITMQ_NODE_PORT_NUMBER` | `5672`       | Port name of RabbitMQ server |
+| `RABBITMQ_USERNAME`         | `user`       | RabbitMQ username            |
+| `RABBITMQ_PASSWORD`         | `bitnami`    | RabbitMQ password            |
+
+The `Mq::Connection#with_topic` and `Mq::Connection#topic_from` can either take an
+explicit `TopicConfig` object, or implicitly read a default configuration with
+`TopicConfig#from_env`. `TopicConfig#from_env` expects the following environment variables:
+
+| Variable         | Sample value         | Description         |
+|------------------|----------------------|---------------------|
+| `RABBITMQ_TOPIC` | `surfliner.metadata` | RabbitMQ topic name |
+
+Note that `TopicConfig#from_env` also accepts keyword options, which are passed through
+to [`Bunny::Channel#topic`](https://api.rubybunny.info/Bunny/Channel.html#topic-instance_method).
+E.g.
+
+```ruby
+topic_config = TopicConfig.from_env(durable: true, auto_delete: true)
+connection.with_topic(topic_config) do |topic|
+  # ...
+end
+```
 
-The `MqConnection` class expects the following environment variables:
+Similarly, `Mq::Topic#bind_queue` can either take an explicit `QueueConfig`, or implicitly
+read a default with `QueConfig#from_env`. `QueueConfig#from_env` expects the following environment
+variables:
 
-```sh
-RABBITMQ_HOST=rabbitmq
-RABBITMQ_NODE_PORT_NUMBER=5672
-RABBITMQ_PASSWORD=bitnami
-RABBITMQ_QUEUE=surfliner.metadata
-RABBITMQ_PLATFORM_ROUTING_KEY=surfliner.metadata.daylight
-RABBITMQ_TOPIC=surfliner.metadata
-RABBITMQ_USERNAME=user
-```
+| Variable                        | Sample value                  | Description          | 
+|---------------------------------|-------------------------------|----------------------|
+| `RABBITMQ_QUEUE`                | `surfliner.metadata`          | RabbitMQ queue name  |
+| `RABBITMQ_PLATFORM_ROUTING_KEY` | `surfliner.metadata.daylight` | RabbitMQ routing key |
 
-(Sample values taken from Surfliner's development [daylight-listener.sh](https://gitlab.com/surfliner/surfliner/-/blob/trunk/docker-compose/env/daylight-listener.sh).)
+And `QueueConfig#from_env` similarly accepts keyword options, which are forwarded to
+[`Bunny::Channel#queue`](https://api.rubybunny.info/Bunny/Channel.html#queue-instance_method):
+
+```ruby
+connection.with_topic(topic_config) do |topic|
+  queue_config = QueueConfig.from_env(exclusive: true, durable: true)
+  queue = topic.bind_queue(queue_config)
+  # ...
+end
+```
 
 The Solr / Daylight handler implementation (see below) additionally expects
 a configured `SOLR_URL`, e.g.:
@@ -74,17 +110,19 @@ The `Surfliner::MetadataConsumer::Solr` package contains a handler that retrieve
 metadata for `:published`/`:updated` events, transforms results for indexing, and 
 updates a target Solr index.
 
-<!-- TODO: better name for this script -->
+The `bin/index-on-publish` script starts a consumer using this handler, and accepts
+(but does not require) the following environment variables:
 
-The `bin/daylight-index-listen` script starts a consumer using this handler.
-
-In addition to the environment variables listed above, this handler expects
-a configured `SOLR_URL`, e.g.:
+| Variable            | Sample value                  | Default        | Description                      |
+|---------------------|-------------------------------|----------------|----------------------------------|
+| `OTEL_SDK_DISABLED` | `true`                        | (not set)      | Whether to disable OpenTelemetry |
+| `OTEL_SERVICE_NAME` | `surfliner-daylight-consumer` | name of script | OpenTelemetry service name       |
+| `OTEL_TRACER_NAME`  | `DaylightConsumerTracer`      | name of script | OpenTelemetry tracer name        |
 
 ### Running the service
 
 <!--
-TODO: Dockerfile shouldn't be daylight-index-listen specific
+TODO: Dockerfile shouldn't be index-on-publish specific
 -->
 
 Build the listener Dockerfile and tag it with the name `solr-listener`:
@@ -101,5 +139,5 @@ docker run --env-file <env-file> solr-listener
 
 ## Utility scripts
 
-- `bin/daylight-index-listen` script starts a consumer using the Daylight Solr handler.
-- `bin/simulate-publish-event` posts a publish event to the queue configured with `MqConnection`
+- `bin/index-on-publish` script starts a consumer using `MetadataConsumer::Solr::MessageHandler`.
+- `bin/simulate-publish-event` posts a publish event to the queue configured with `Mq::Connection`

data/bin/index-on-publish

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require "bundler/setup"
+require "logger"
+
+require "surfliner/metadata_consumer"
+require "surfliner/mq"
+
+$stdout.sync = true # don't buffer log output
+logger       = Logger.new($stdout).tap do |logger|
+  logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
+end
+
+unless ENV["OTEL_SDK_DISABLED"] == "true"
+  require "opentelemetry/sdk"
+  require "opentelemetry-exporter-otlp"
+
+  otel_service_name = ENV.fetch('OTEL_SERVICE_NAME', File.basename(__FILE__))
+  otel_tracer_name  = ENV.fetch('OTEL_TRACER_NAME', File.basename(__FILE__))
+
+  OpenTelemetry::SDK.configure do |c|
+    c.service_name = otel_service_name
+    c.use_all # enables auto instrumentation for Bunny, Net::HTTP, etc...
+  end
+
+  tracer = OpenTelemetry.tracer_provider.tracer(otel_tracer_name)
+end
+
+consumer = Surfliner::MetadataConsumer::Consumer.new(
+  connection: Surfliner::Mq::Connection.new(logger: logger),
+  tracer:     (tracer if defined?(tracer)),
+  handler:    Surfliner::MetadataConsumer::Solr::MessageHandler
+)
+consumer.run(topic_config: Surfliner::Mq::TopicConfig.from_env(auto_delete: true))

data/bin/simulate-publish-event

@@ -4,24 +4,24 @@ require "json"
 require "logger"
 
 require "surfliner/metadata_consumer"
+require "surfliner/mq"
 
 module Surfliner
   module MetadataConsumer
     class Publisher
+      attr_reader :topic
+      attr_reader :api_base
+      attr_reader :routing_key
 
-      attr_reader :logger
-      attr_reader :topic_opts
-      attr_reader :queue_opts
-
-      def initialize(logger:, topic_opts: {}, queue_opts: {})
-        @logger = logger
-        @topic_opts = topic_opts
-        @queue_opts = queue_opts
+      def initialize(topic, api_base:, routing_key:)
+        @topic       = topic
+        @api_base    = api_base
+        @routing_key = routing_key
       end
 
       def publish(resource_id)
         payload = payload_for(resource_id)
-        connection.publish(payload)
+        topic.publish(payload, routing_key:)
       end
 
       def payload_for(id)
@@ -30,27 +30,37 @@ module Surfliner
           time_stamp:  DateTime.now }.to_json
       end
 
-      def connection
-        @connection ||= MqConnection.new(logger:).connect(topic_opts:, queue_opts:)
-      end
-
-      def api_base
-        @api_base ||= ENV.fetch("METADATA_API_URL_BASE", "http://metadata.test/resources")
+      class << self
+        # Connects with default (environment-configured) settings, yields a publisher
+        # for the specified topic, and closes the connection after the provided block
+        # completes.
+        #
+        # @param logger [Logger] the logger
+        # @param topic_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+        #
+        # @yieldparam publisher [Publisher] the publisher
+        def connect(logger:, **topic_opts)
+          topic_config = Mq::TopicConfig.from_env(**topic_opts)
+          Mq::Connection.new(logger:).with_topic(topic_config) do |topic|
+            yield Publisher.new(
+              topic,
+              api_base:    ENV.fetch("METADATA_API_URL_BASE", "http://metadata.test/resources"),
+              routing_key: ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
+            )
+          end
+        end
       end
     end
   end
 end
 
 $stdout.sync = true # don't buffer log output
-logger = Logger.new($stdout).tap do |logger|
+logger       = Logger.new($stdout).tap do |logger|
   logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
 end
 
-publisher = Surfliner::MetadataConsumer::Publisher.new(logger:)
-begin
+Surfliner::MetadataConsumer::Publisher.connect(logger:, auto_delete: true) do |publisher|
   ARGV.each do |resource_id|
     publisher.publish(resource_id)
   end
-ensure
-  publisher.connection.close
 end

data/lib/surfliner/metadata_consumer/consumer.rb

@@ -1,5 +1,8 @@
 require "rsolr"
-require "surfliner/metadata_consumer/mq_connection"
+require "surfliner/util/assert"
+require "surfliner/mq/connection"
+require "surfliner/mq/topic_config"
+require "surfliner/mq/queue_config"
 require "surfliner/metadata_consumer/solr/message_handler"
 
 module Surfliner
@@ -7,29 +10,40 @@ module Surfliner
     # A metadata consumer that subscribes to a RabbitMQ queue and passes
     # messages to the specified handler.
     class Consumer
-      attr_reader :connection, :logger, :tracer, :handler
+      include Util::Assert
+
+      # @return [Mq::Connection] the connection
+      attr_reader :connection
+
+      # @return [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
+      attr_reader :tracer
+
+      # @return [#handle] an object accepting a JSON string
+      attr_reader :handler
 
       # Initializes a new `Consumer`
+      #
+      # @param connection [Mq::Connection] the connection
+      # @param handler [#handle] an object accepting a JSON string
       # @param tracer [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
-      # @param logger [Logger] log message destination
-      # @param handler #handle an object accepting a JSON string
-      def initialize(tracer:, logger:, handler:)
-        @connection = MqConnection.new(logger:)
-        @logger = logger
+      def initialize(connection:, handler:, tracer: nil)
+        @connection = not_nil!(connection, "connection")
+        @handler = not_nil!(handler, "handler")
         @tracer = tracer
-        @handler = handler
+      end
+
+      # @return [Logger] log message destination
+      def logger
+        connection.logger
       end
 
       # Starts listening to the message queue and passing messages to the handler.
-      # @param queue_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
-      # @param topic_opts [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
-      def run(queue_opts: {}, topic_opts: {})
-        connection.open(queue_opts:, topic_opts:) do |queue|
-          queue.subscribe(block: true) do |_delivery_info, _properties, payload_json|
-            tracer.in_span("surfliner metadata consumer message") do |_span|
-              handle(payload_json)
-            end
-          end
+      # @param topic_config [TopicConfig] topic configuration
+      # @param queue_config [QueueConfig] queue configuration
+      def run(topic_config: Mq::TopicConfig.from_env, queue_config: Mq::QueueConfig.from_env)
+        connection.with_topic(topic_config) do |topic|
+          queue = topic.bind_queue(queue_config)
+          queue.subscribe(block: true, &method(:on_delivery))
         end
       end
 
@@ -43,6 +57,16 @@ module Surfliner
       rescue => err
         logger.error(" [!] failed to handle message: #{err.full_message}")
       end
+
+      private
+
+      def on_delivery(_delivery_info, _properties, payload_json)
+        return handle(payload_json) unless tracer
+
+        tracer.in_span("surfliner metadata consumer message") do |_span, _context|
+          handle(payload_json)
+        end
+      end
     end
   end
 end

data/lib/surfliner/metadata_consumer/version.rb

@@ -1,8 +1,7 @@
 # Umbrella module for general Surfliner code
 module Surfliner
-  # Parent module for this gem
   module MetadataConsumer
     # The gem version
-    VERSION = "0.1.0.pre.alpha.4"
+    VERSION = "0.1.0.pre.alpha.5"
   end
 end

data/lib/surfliner/metadata_consumer.rb

@@ -1 +1,4 @@
-Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).sort.each(&method(:require))
+Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).each(&method(:require))
+
+# Functionality specific to consuming RabbitMQ feeds
+module Surfliner::MetadataConsumer; end

data/lib/surfliner/mq/connection.rb

@@ -0,0 +1,127 @@
+require "bunny"
+require "surfliner/util/assert"
+
+module Surfliner
+  module Mq
+    # An object encapsulating a RabbitMQ connection.
+    class Connection
+      include Util::Assert
+
+      # @return [Logger]  The logger
+      attr_reader :logger
+
+      # @return [Bunny::Session]  The current RabbitMQ session
+      attr_reader :session
+
+      # @return [Bunny::Channel]  The channel being listened to
+      attr_reader :channel
+
+      # @return [ConnectionConfig]  The configuration
+      attr_reader :config
+
+      # Initializes a new `MqConnection`.
+      #
+      # @param logger [Logger] the logger
+      # @param config [ConnectionConfig] the configuration
+      def initialize(logger:, config: ConnectionConfig.from_env)
+        @logger = not_nil!(logger)
+        @config = not_nil!(config)
+      end
+
+      # Opens a connection.
+      # @return [self]
+      # @raise [IOError] if already connected
+      def connect
+        raise IOError, "RabbitMQ session #{session} already open." if open?
+
+        logger.info("Rabbitmq message broker session url: #{config.redacted_url}")
+        @session = Bunny.new(config.session_url, logger: logger)
+        connect_on(session)
+        @channel = session.create_channel
+
+        self
+      rescue Bunny::TCPConnectionFailed => err
+        # TODO: realistically, this only happens in session.start, where we're eating it
+        logger.error("Connection to #{config.redacted_url} failed")
+        raise err
+      rescue Bunny::PossibleAuthenticationFailureError => err
+        # TODO: realistically, this only happens in session.start, where we're eating it
+        logger.error("Failed to authenticate to #{config.redacted_url}")
+        raise err
+      end
+
+      # Opens a session, yields a client for the specified topic, and closes the
+      # session after the provided block completes.
+      # @param config [TopicConfig] topic configuration
+      # @yield [MqTopic] A client for the topic
+      def with_topic(config = TopicConfig.from_env)
+        connect
+
+        yield topic_from(config)
+      ensure
+        close
+      end
+
+      # Returns a client for the specified topic. Note that this does _not_ open
+      # or close the connection.
+      #
+      # @param config [TopicConfig] topic configuration
+      # @return [Topic] A client for the topic
+      # @raise [IOError] if the connection is not open
+      def topic_from(config)
+        raise IOError, "RabbitMQ session not open" unless open?
+
+        Topic.from_config(config, channel:, logger:)
+      end
+
+      # Closes the session.
+      def close
+        return unless channel
+        return if channel.closed?
+        logger.info("closing channel")
+        channel.close
+      ensure
+        logger.info("closing session")
+        session&.close
+      end
+
+      # @return [true, false] True if the session is open, false otherwise
+      def open?
+        session&.status == :open
+      end
+
+      # @return [Symbol, nil] The session status, or nil if there is no session
+      def status
+        session&.status
+      end
+
+      # @return [String]  The RabbitMQ hostname
+      def host
+        config.host
+      end
+
+      # @return [String]  The RabbitMQ port
+      def port
+        config.port
+      end
+
+      private
+
+      def connect_on(session, timeout = 120)
+        timer = 0
+        logger.info "Trying to open queue session with timeout=#{timeout}"
+        while timer < timeout
+          begin
+            session.start
+          rescue
+            # TODO: do we actually want to rescue from everything?
+          end
+          return session if session.status == :open
+          sleep 1
+          timer += 1
+        end
+        raise "Failed to connect to queue."
+      end
+    end
+  end
+end

data/lib/surfliner/mq/connection_config.rb

@@ -0,0 +1,62 @@
+module Surfliner
+  module Mq
+    # An object encapsulating RabbitMQ configuration.
+    class ConnectionConfig
+      # @return [String]  The RabbitMQ hostname
+      attr_reader :host
+
+      # @return [String]  The RabbitMQ AMQP port
+      attr_reader :port
+
+      # @return [String]  The RabbitMQ username
+      attr_reader :username
+
+      # @return [String]  The RabbitMQ passsword
+      attr_reader :password
+
+      # Initializes a new `MqConfig` object.
+      # @param host [String] RabbitMQ hostname
+      # @param port [String] RabbitMQ AMQP port
+      # @param username [String] RabbitMQ username
+      # @param password [String] RabbitMQ passsword
+      def initialize(host:, port:, username:, password:)
+        @host = host
+        @port = port
+        @username = username
+        @password = password
+      end
+
+      class << self
+        # Reads RabbitMQ configuration from environment variables and
+        # returns it as a new `ConnectionConfig` object.
+        #
+        # | Variable                    | Sample value | Description                  |
+        # |-----------------------------|--------------|------------------------------|
+        # | `RABBITMQ_HOST`             | `rabbitmq`   | Hostname of RabbitMQ server  |
+        # | `RABBITMQ_NODE_PORT_NUMBER` | `5672`       | Port name of RabbitMQ server |
+        # | `RABBITMQ_USERNAME`         | `user`       | RabbitMQ username            |
+        # | `RABBITMQ_PASSWORD`         | `bitnami`    | RabbitMQ password            |
+        #
+        # @return [ConnectionConfig] The configuration.
+        def from_env
+          ConnectionConfig.new(
+            host: ENV.fetch("RABBITMQ_HOST"),
+            port: ENV.fetch("RABBITMQ_NODE_PORT_NUMBER"),
+            username: ENV.fetch("RABBITMQ_USERNAME"),
+            password: ENV.fetch("RABBITMQ_PASSWORD")
+          )
+        end
+      end
+
+      # @return [String]  the connection URL as a string
+      def session_url
+        @session_url ||= "amqp://#{username}:#{password}@#{host}:#{port}"
+      end
+
+      # @return [String]  the connection URL as a string, without the password
+      def redacted_url
+        @redacted_url ||= session_url.sub(password, "REDACTED")
+      end
+    end
+  end
+end

data/lib/surfliner/mq/queue_config.rb

@@ -0,0 +1,49 @@
+module Surfliner
+  module Mq
+    # Encapsulates queue configuration
+    class QueueConfig
+      # @return [String]  The queue to listen to
+      attr_reader :name
+
+      # @return [String] platform routing key to listen to
+      attr_reader :routing_key
+
+      # @return [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+      attr_reader :options
+
+      # @param name [String] queue exchange to listen to
+      # @param routing_key [String] platform routing key to listen to
+      # @param options [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+      def initialize(name:, routing_key:, options: {})
+        @name = name
+        @routing_key = routing_key
+        @options = options
+      end
+
+      class << self
+        # Returns a default (environment-variable-based) configuration with the
+        # specified options.
+        #
+        # | Variable                        | Sample value                  | Description          |
+        # |---------------------------------|-------------------------------|----------------------|
+        # | `RABBITMQ_QUEUE`                | `surfliner.metadata`          | RabbitMQ queue name  |
+        # | `RABBITMQ_PLATFORM_ROUTING_KEY` | `surfliner.metadata.daylight` | RabbitMQ routing key |
+        #
+        # @param options [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+        # @return [QueueConfig] The configuration.
+        def from_env(**options)
+          QueueConfig.new(
+            name: ENV.fetch("RABBITMQ_QUEUE"),
+            routing_key: default_routing_key,
+            options:
+          )
+        end
+
+        # @return [String] The default routing key from `ENV["RABBITMQ_PLATFORM_ROUTING_KEY"]`
+        def default_routing_key
+          ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/mq/topic.rb

@@ -0,0 +1,65 @@
+require "surfliner/util/assert"
+
+module Surfliner
+  module Mq
+    # Encapsulates a RabbitMQ topic
+    class Topic
+      include Surfliner::Util::Assert
+
+      # @return [String] The name of the topic
+      attr_reader :name
+
+      # @return [Bunny::Channel]  The channel being used
+      attr_reader :channel
+
+      # @return [Logger]  The logger
+      attr_reader :logger
+
+      # @return [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      attr_reader :options
+
+      # @param name [String] The name of the topic
+      # @param channel [Bunny::Channel]  The channel to use
+      # @param logger [Logger] the logger
+      # @param options [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      def initialize(name, channel:, logger:, options: {})
+        @name = not_nil!(name, "topic name")
+        @channel = not_nil!(channel, "channel")
+        @logger = not_nil!(logger, "logger")
+        @options = not_nil!(options, "options")
+      end
+
+      # @return [Bunny::Exchange]  The exchange for the topic
+      def exchange
+        @exchange ||= channel.topic(name, options)
+      end
+
+      # Publishes the specified payload
+      # @param payload [String] the payload to publish
+      # @param routing_key [String] platform routing key to publish to
+      # @return [Bunny::Exchange] see #exchange
+      def publish(payload, routing_key: QueueConfig.default_routing_key)
+        logger.info "Publishing to #{routing_key} with payload: #{payload}"
+        exchange.publish(payload, routing_key:)
+      end
+
+      # @param config [QueueConfig] queue configuration
+      # @return  [Bunny::Queue]  the queue
+      def bind_queue(config = QueueConfig.from_env)
+        channel.queue(config.name, config.options).tap do |q|
+          q.bind(exchange, routing_key: config.routing_key)
+        end
+      end
+
+      class << self
+        # Creates a new `MqTopic` from the specified configuration
+        # @param config [TopicConfig] the configuration
+        # @param channel [Bunny::Channel]  The channel to use
+        # @param logger [Logger] the logger
+        def from_config(config, channel:, logger:)
+          new(config.name, channel:, logger:, options: config.options)
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/mq/topic_config.rb

@@ -0,0 +1,37 @@
+module Surfliner
+  module Mq
+    # Encapsulates topic configuration
+    class TopicConfig
+      # @return [String]  The topic exchange to listen to
+      attr_reader :name
+
+      # @return [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      attr_reader :options
+
+      # @param name [String] topic exchange to listen to
+      # @param options [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      def initialize(name:, options: {})
+        @name = name
+        @options = options
+      end
+
+      class << self
+        # Returns a default (environment-variable-based) configuration with the
+        # specified options.
+        #
+        # | Variable         | Sample value         | Description         |
+        # |------------------|----------------------|---------------------|
+        # | `RABBITMQ_TOPIC` | `surfliner.metadata` | RabbitMQ topic name |
+        #
+        # @param options [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+        # @return [TopicConfig] The configuration.
+        def from_env(**options)
+          TopicConfig.new(
+            name: ENV.fetch("RABBITMQ_TOPIC"),
+            options:
+          )
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/mq.rb

@@ -0,0 +1,4 @@
+Dir.glob(File.expand_path("mq/*.rb", __dir__)).each(&method(:require))
+
+# General RabbitMQ functionality
+module Surfliner::Mq; end

data/lib/surfliner/util/assert.rb

@@ -0,0 +1,21 @@
+module Surfliner
+  # Miscellaneous utility module
+  module Util
+    # Helper methods for argument validation
+    module Assert
+      class << self
+        include Assert
+      end
+
+      # @param arg_value [Object, nil] the argument value
+      # @param arg_name [String] the name of the argument, for error messages
+      # @return arg_value
+      # @raise [ArgumentError] if `arg_value` is `nil`
+      def not_nil!(arg_value, arg_name = "argument")
+        return arg_value unless arg_value.nil?
+
+        raise ArgumentError, "#{arg_name} must not be nil"
+      end
+    end
+  end
+end

data/lib/surfliner/util.rb

@@ -0,0 +1,4 @@
+Dir.glob(File.expand_path("util/*.rb", __dir__)).each(&method(:require))
+
+# Miscellaneous utility module
+module Surfliner::Util; end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: surfliner-metadata_consumer
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.alpha.4
+  version: 0.1.0.pre.alpha.5
 platform: ruby
 authors:
 - Project Surfliner
 bindir: bin
 cert_chain: []
-date: 2025-03-17 00:00:00.000000000 Z
+date: 2025-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -86,103 +86,117 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: debug
+  name: ci_reporter_rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.2
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.2
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
-  name: standard
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.31'
+        version: 1.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.31'
+        version: 1.9.2
 - !ruby/object:Gem::Dependency
-  name: ci_reporter_rspec
+  name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: github-markup
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: dotenv
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '3.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -211,6 +225,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.31'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.31'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -240,7 +268,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.9.37
 executables:
-- daylight-index-listen
+- index-on-publish
 - simulate-publish-event
 extensions: []
 extra_rdoc_files: []
@@ -252,12 +280,10 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/daylight-index-listen
+- bin/index-on-publish
 - bin/simulate-publish-event
 - lib/surfliner/metadata_consumer.rb
 - lib/surfliner/metadata_consumer/consumer.rb
-- lib/surfliner/metadata_consumer/mq_config.rb
-- lib/surfliner/metadata_consumer/mq_connection.rb
 - lib/surfliner/metadata_consumer/payload.rb
 - lib/surfliner/metadata_consumer/solr.rb
 - lib/surfliner/metadata_consumer/solr/delete_handler.rb
@@ -265,6 +291,14 @@ files:
 - lib/surfliner/metadata_consumer/solr/message_handler.rb
 - lib/surfliner/metadata_consumer/superskunk_client.rb
 - lib/surfliner/metadata_consumer/version.rb
+- lib/surfliner/mq.rb
+- lib/surfliner/mq/connection.rb
+- lib/surfliner/mq/connection_config.rb
+- lib/surfliner/mq/queue_config.rb
+- lib/surfliner/mq/topic.rb
+- lib/surfliner/mq/topic_config.rb
+- lib/surfliner/util.rb
+- lib/surfliner/util/assert.rb
 homepage: https://gitlab.com/surfliner/metadata_consumer
 licenses:
 - MIT

data/bin/daylight-index-listen

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-require "bundler/setup"
-require "logger"
-require "opentelemetry/sdk"
-
-require "surfliner/metadata_consumer"
-
-unless ENV["OTEL_SDK_DISABLED"] == "true"
-  OpenTelemetry::SDK.configure do |c|
-    c.service_name = "surfliner-daylight-consumer"
-    c.use_all # enables auto instrumentation for Bunny, Net::HTTP, etc...
-  end
-end
-tracer   = OpenTelemetry.tracer_provider.tracer("DaylightConsumerTracer")
-
-$stdout.sync = true # don't buffer log output
-logger       = Logger.new($stdout).tap do |logger|
-  logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
-end
-
-handler  = Surfliner::MetadataConsumer::Solr::MessageHandler
-consumer = Surfliner::MetadataConsumer::Consumer.new(tracer:, logger:, handler:)
-consumer.run

data/lib/surfliner/metadata_consumer/mq_config.rb

@@ -1,79 +0,0 @@
-module Surfliner
-  module MetadataConsumer
-    # An object encapsulating RabbitMQ configuration.
-    class MqConfig
-      # @return [String]  The RabbitMQ hostname
-      attr_reader :host
-
-      # @return [String]  The RabbitMQ AMQP port
-      attr_reader :port
-
-      # @return [String]  The RabbitMQ username
-      attr_reader :username
-
-      # @return [String]  The RabbitMQ passsword
-      attr_reader :password
-
-      # @return [String]  The topic exchange to listen to
-      attr_reader :topic
-
-      # @return [String]  The name of the queue to listen to
-      attr_reader :queue_name
-
-      # @return [String]  The platform routing key to listen to
-      attr_reader :routing_key
-
-      # Initializes a new `MqConfig` object.
-      # @param host [The] RabbitMQ hostname
-      # @param port [The] RabbitMQ AMQP port
-      # @param username [The] RabbitMQ username
-      # @param password [The] RabbitMQ passsword
-      # @param topic [The] topic exchange to listen to
-      # @param queue_name [The] name of the queue to listen to
-      # @param routing_key [The] platform routing key to listen to
-      def initialize(host:, port:, username:, password:, topic:, queue_name:, routing_key:)
-        @host = host
-        @port = port
-        @username = username
-        @password = password
-        @topic = topic
-        @queue_name = queue_name
-        @routing_key = routing_key
-      end
-
-      class << self
-        # Reads RabbitMQ configuration from environment variables and
-        # returns it as a new `MqConfig` object.
-        #
-        # - `RABBITMQ_HOST` → `host`
-        # - `RABBITMQ_NODE_PORT_NUMBER` → `port`
-        # - `RABBITMQ_USERNAME` → `username`
-        # - `RABBITMQ_PASSWORD` → `password`
-        # - `RABBITMQ_TOPIC` → `topic`
-        # - `RABBITMQ_QUEUE` → `queue_name`
-        # - `RABBITMQ_PLATFORM_ROUTING_KEY` → `routing_key`
-        def from_env
-          MqConfig.new(
-            host: ENV.fetch("RABBITMQ_HOST"),
-            port: ENV.fetch("RABBITMQ_NODE_PORT_NUMBER"),
-            username: ENV.fetch("RABBITMQ_USERNAME"),
-            password: ENV.fetch("RABBITMQ_PASSWORD"),
-            topic: ENV.fetch("RABBITMQ_TOPIC"),
-            queue_name: ENV.fetch("RABBITMQ_QUEUE"),
-            routing_key: ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
-          )
-        end
-      end
-
-      # @return [String]  the connection URL as a string
-      def connection_url
-        @connection_url ||= "amqp://#{username}:#{password}@#{host}:#{port}"
-      end
-
-      # @return [String]  the connection URL as a string, without the password
-      def redacted_url
-        @redacted_url ||= connection_url.sub(password, "REDACTED")
-      end
-    end
-  end
-end

data/lib/surfliner/metadata_consumer/mq_connection.rb

@@ -1,136 +0,0 @@
-require "bunny"
-
-module Surfliner
-  module MetadataConsumer
-    # An object encapsulating a RabbitMQ connection.
-    class MqConnection
-      # @return [Logger]  The logger
-      attr_reader :logger
-
-      # @return [Bunny::Session]  The current RabbitMQ session
-      attr_reader :connection
-
-      # @return [Bunny::Channel]  The channel being listened to
-      attr_reader :channel
-
-      # @return [Bunny::Exchange]  The exchange being listened to
-      attr_reader :exchange
-
-      # @return [Bunny::Queue]  The queue being listened to
-      attr_reader :queue
-
-      # @return [MqConfig]  The configuration
-      attr_reader :config
-
-      # Initializes a new `MqConnection`.
-      #
-      # @param logger [Logger] the logger
-      # @param config [MqConfig] the configuration
-      def initialize(logger:, config: MqConfig.from_env)
-        @logger = logger
-        @config = config
-      end
-
-      # Opens a connection.
-      # @param topic_opts [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
-      # @param queue_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
-      # @return [self]
-      # @raise RuntimeError if already connected
-      def connect(topic_opts: {}, queue_opts: {})
-        raise "RabbitMQ connection #{connection} already open." if open?
-
-        logger.info("Rabbitmq message broker connection url: #{config.redacted_url}")
-        @connection = Bunny.new(config.connection_url, logger: logger)
-        connect_on(connection)
-        @channel = connection.create_channel
-        @exchange = channel.topic(config.topic, topic_opts)
-        @queue = channel.queue(config.queue_name, queue_opts)
-        queue.bind(exchange, routing_key: config.routing_key)
-
-        self
-      rescue Bunny::TCPConnectionFailed => err
-        # TODO: realistically, this only happens in connection.start, where we're eating it
-        logger.error("Connection to #{config.redacted_url} failed")
-        raise err
-      rescue Bunny::PossibleAuthenticationFailureError => err
-        # TODO: realistically, this only happens in connection.start, where we're eating it
-        logger.error("Failed to authenticate to #{config.redacted_url}")
-        raise err
-      end
-
-      # Opens a connection, yields the queue, and closes the connection after
-      # the provided block completes.
-      # @param topic_opts [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
-      # @param queue_opts [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
-      # @yield [Bunny::Queue]  the queue
-      def open(topic_opts: {}, queue_opts: {})
-        connect(topic_opts:, queue_opts:)
-        yield queue
-      ensure
-        close
-      end
-
-      # Closes the connection.
-      def close
-        return unless channel
-        return if channel.closed?
-        logger.info("closing channel")
-        channel.close
-      ensure
-        logger.info("closing connection")
-        connection&.close
-      end
-
-      # @return [true, false] True if the connection is open, false otherwise
-      def open?
-        connection&.status == :open
-      end
-
-      # @return [Symbol, nil] The connection status, or nil if there is no connection
-      def status
-        connection&.status
-      end
-
-      # @return [String]  The RabbitMQ hostname
-      def host
-        config.host
-      end
-
-      # @return [String]  The RabbitMQ port
-      def port
-        config.port
-      end
-
-      # @return [String] The routing key
-      def routing_key
-        config.routing_key
-      end
-
-      # Publishes the specified payload
-      # @param payload [String] the payload to publish
-      # @return [Bunny::Exchange] see #exchange
-      def publish(payload)
-        logger.info "Publishing to #{routing_key} with payload: #{payload}"
-        exchange.publish(payload, routing_key:)
-      end
-
-      private
-
-      def connect_on(connection, timeout = 120)
-        timer = 0
-        logger.info "Trying to open queue connection with timeout=#{timeout}"
-        while timer < timeout
-          begin
-            connection.start
-          rescue
-            # TODO: do we actually want to rescue from everything?
-          end
-          return connection if connection.status == :open
-          sleep 1
-          timer += 1
-        end
-        raise "Failed to connect to queue."
-      end
-    end
-  end
-end