surfliner-metadata_consumer 0.1.0.pre.alpha.4 → 0.1.0.pre.alpha.6

data/lib/surfliner/mq/connection.rb

@@ -0,0 +1,135 @@
+require "bunny"
+require "surfliner/util/assert"
+
+module Surfliner
+  module Mq
+    # An object encapsulating a RabbitMQ connection.
+    class Connection
+      include Util::Assert
+
+      # @return [Logger]  The logger
+      attr_reader :logger
+
+      # @return [Bunny::Session]  The current RabbitMQ session
+      attr_reader :session
+
+      # @return [Bunny::Channel]  The channel being listened to
+      attr_reader :channel
+
+      # @return [ConnectionConfig]  The configuration
+      attr_reader :config
+
+      # Initializes a new `MqConnection`.
+      #
+      # @param logger [Logger] the logger
+      # @param config [ConnectionConfig] the configuration
+      def initialize(logger:, config: ConnectionConfig.from_env)
+        @logger = not_nil!(logger)
+        @config = not_nil!(config)
+      end
+
+      # Opens this connection and creates a channel. If a block is
+      # given, yields the open connection, closing the connection and
+      # channel afterwards; otherwise, returns the open connection.
+      # @return [self] an open connection
+      # @yieldparam connection [self] an open connection
+      # @raise [IOError] if already connected
+      def connect
+        init_connection
+
+        return self unless block_given?
+
+        begin
+          yield self
+        ensure
+          close
+        end
+      end
+
+      # Opens a session, yields a client for the specified topic, and closes the
+      # session after the provided block completes.
+      # @param config [TopicConfig] topic configuration
+      # @yield [MqTopic] A client for the topic
+      def with_topic(config = TopicConfig.from_env)
+        connect do |cxn|
+          yield cxn.topic_from(config)
+        end
+      end
+
+      # Returns a client for the specified topic. Note that this does _not_ open
+      # or close the connection.
+      #
+      # @param config [TopicConfig] topic configuration
+      # @return [Topic] A client for the topic
+      # @raise [IOError] if the connection is not open
+      def topic_from(config)
+        raise IOError, "RabbitMQ session not open" unless open?
+
+        Topic.from_config(config, channel:, logger:)
+      end
+
+      # Closes the session.
+      def close
+        logger.info("closing session")
+        # Note: This will also close any open channels
+        session&.close(config.await_response_on_close)
+      end
+
+      # @return [true, false] True if the session is open, false otherwise
+      def open?
+        session&.status == :open
+      end
+
+      # @return [Symbol, nil] The session status, or nil if there is no session
+      def status
+        session&.status
+      end
+
+      # @return [String]  The RabbitMQ hostname
+      def host
+        config.host
+      end
+
+      # @return [String]  The RabbitMQ port
+      def port
+        config.port
+      end
+
+      private
+
+      def init_connection
+        raise IOError, "RabbitMQ session #{session} already open." if open?
+
+        logger.info("Rabbitmq message broker session url: #{config.redacted_url}")
+        opts = {logger:}.merge(config.opts)
+        @session = Bunny.new(config.session_url, **opts)
+        connect_on(session)
+        @channel = session.create_channel
+      rescue Bunny::TCPConnectionFailed => err
+        # TODO: realistically, this only happens in session.start, where we're eating it
+        logger.error("Connection to #{config.redacted_url} failed")
+        raise err
+      rescue Bunny::PossibleAuthenticationFailureError => err
+        # TODO: realistically, this only happens in session.start, where we're eating it
+        logger.error("Failed to authenticate to #{config.redacted_url}")
+        raise err
+      end
+
+      def connect_on(session, timeout = 120)
+        timer = 0
+        logger.info "Trying to open queue session with timeout=#{timeout}"
+        while timer < timeout
+          begin
+            session.start
+          rescue
+            # TODO: do we actually want to rescue from everything?
+          end
+          return session if session.status == :open
+          sleep 1
+          timer += 1
+        end
+        raise "Failed to connect to queue."
+      end
+    end
+  end
+end

data/lib/surfliner/mq/connection_config.rb

@@ -0,0 +1,83 @@
+module Surfliner
+  module Mq
+    # An object encapsulating RabbitMQ configuration.
+    class ConnectionConfig
+      FALSE_VALUES = ([""] + %w[0 off Off OFF f false False FALSE F n no No NO N]).freeze
+
+      # @return [String]  The RabbitMQ hostname
+      attr_reader :host
+
+      # @return [String]  The RabbitMQ AMQP port
+      attr_reader :port
+
+      # @return [String]  The RabbitMQ username
+      attr_reader :username
+
+      # @return [String]  The RabbitMQ passsword
+      attr_reader :password
+
+      # @return [Boolean] whether the RabbitMQ client should wait for a response when closing the connection
+      attr_reader :await_response_on_close
+
+      # @return [Hash] Additional RabbitMQ connection options (see Bunny::Session#initialize)
+      attr_reader :opts
+
+      # Initializes a new `MqConfig` object.
+      # @param host [String] RabbitMQ hostname
+      # @param port [String] RabbitMQ AMQP port
+      # @param username [String] RabbitMQ username
+      # @param password [String] RabbitMQ passsword
+      # @param await_response_on_close [Boolean] whether the RabbitMQ client should wait for a response when closing the connection
+      # @param opts [Hash] additional RabbitMQ conection options (see Bunny::Session#initialize)
+      def initialize(host:, port:, username:, password:, await_response_on_close: true, **opts)
+        @host = host
+        @port = port
+        @username = username
+        @password = password
+        @await_response_on_close = parse_boolean(await_response_on_close)
+        @opts = opts
+      end
+
+      class << self
+        # Reads RabbitMQ configuration from environment variables and
+        # returns it as a new `ConnectionConfig` object.
+        #
+        # | Variable                    | Sample value | Description                                               |
+        # |-----------------------------|--------------|-----------------------------------------------------------|
+        # | `RABBITMQ_HOST`             | `rabbitmq`   | Hostname of RabbitMQ server                               |
+        # | `RABBITMQ_NODE_PORT_NUMBER` | `5672`       | Port name of RabbitMQ server                              |
+        # | `RABBITMQ_USERNAME`         | `user`       | RabbitMQ username                                         |
+        # | `RABBITMQ_PASSWORD`         | `bitnami`    | RabbitMQ password                                         |
+        # | `RABBITMQ_AWAIT_ON_CLOSE`   | `false`      | Whether to wait on response when closing (default = true) |
+        #
+        # @return [ConnectionConfig] The configuration.
+        def from_env(**opts)
+          ConnectionConfig.new(
+            host: ENV.fetch("RABBITMQ_HOST"),
+            port: ENV.fetch("RABBITMQ_NODE_PORT_NUMBER"),
+            username: ENV.fetch("RABBITMQ_USERNAME"),
+            password: ENV.fetch("RABBITMQ_PASSWORD"),
+            await_response_on_close: ENV.fetch("RABBITMQ_AWAIT_ON_CLOSE") || true,
+            **opts
+          )
+        end
+      end
+
+      # @return [String]  the connection URL as a string
+      def session_url
+        @session_url ||= "amqp://#{username}:#{password}@#{host}:#{port}"
+      end
+
+      # @return [String]  the connection URL as a string, without the password
+      def redacted_url
+        @redacted_url ||= session_url.sub(password, "REDACTED")
+      end
+
+      private
+
+      def parse_boolean(v)
+        !FALSE_VALUES.include?(v.to_s)
+      end
+    end
+  end
+end

data/lib/surfliner/mq/queue_config.rb

@@ -0,0 +1,49 @@
+module Surfliner
+  module Mq
+    # Encapsulates queue configuration
+    class QueueConfig
+      # @return [String]  The queue to listen to
+      attr_reader :name
+
+      # @return [String] platform routing key to listen to
+      attr_reader :routing_key
+
+      # @return [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+      attr_reader :options
+
+      # @param name [String] queue exchange to listen to
+      # @param routing_key [String] platform routing key to listen to
+      # @param options [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+      def initialize(name:, routing_key:, options: {})
+        @name = name
+        @routing_key = routing_key
+        @options = options
+      end
+
+      class << self
+        # Returns a default (environment-variable-based) configuration with the
+        # specified options.
+        #
+        # | Variable                        | Sample value                  | Description          |
+        # |---------------------------------|-------------------------------|----------------------|
+        # | `RABBITMQ_QUEUE`                | `surfliner.metadata`          | RabbitMQ queue name  |
+        # | `RABBITMQ_PLATFORM_ROUTING_KEY` | `surfliner.metadata.daylight` | RabbitMQ routing key |
+        #
+        # @param options [Hash] RabbitMQ queue options. (See Bunny::Channel#queue)
+        # @return [QueueConfig] The configuration.
+        def from_env(**options)
+          QueueConfig.new(
+            name: ENV.fetch("RABBITMQ_QUEUE"),
+            routing_key: default_routing_key,
+            options:
+          )
+        end
+
+        # @return [String] The default routing key from `ENV["RABBITMQ_PLATFORM_ROUTING_KEY"]`
+        def default_routing_key
+          ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/mq/topic.rb

@@ -0,0 +1,65 @@
+require "surfliner/util/assert"
+
+module Surfliner
+  module Mq
+    # Encapsulates a RabbitMQ topic
+    class Topic
+      include Surfliner::Util::Assert
+
+      # @return [String] The name of the topic
+      attr_reader :name
+
+      # @return [Bunny::Channel]  The channel being used
+      attr_reader :channel
+
+      # @return [Logger]  The logger
+      attr_reader :logger
+
+      # @return [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      attr_reader :options
+
+      # @param name [String] The name of the topic
+      # @param channel [Bunny::Channel]  The channel to use
+      # @param logger [Logger] the logger
+      # @param options [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      def initialize(name, channel:, logger:, options: {})
+        @name = not_nil!(name, "topic name")
+        @channel = not_nil!(channel, "channel")
+        @logger = not_nil!(logger, "logger")
+        @options = not_nil!(options, "options")
+      end
+
+      # @return [Bunny::Exchange]  The exchange for the topic
+      def exchange
+        @exchange ||= channel.topic(name, options)
+      end
+
+      # Publishes the specified payload
+      # @param payload [String] the payload to publish
+      # @param routing_key [String] platform routing key to publish to
+      # @return [Bunny::Exchange] see #exchange
+      def publish(payload, routing_key: QueueConfig.default_routing_key)
+        logger.info "Publishing to #{routing_key} with payload: #{payload}"
+        exchange.publish(payload, routing_key:)
+      end
+
+      # @param config [QueueConfig] queue configuration
+      # @return  [Bunny::Queue]  the queue
+      def bind_queue(config = QueueConfig.from_env)
+        channel.queue(config.name, config.options).tap do |q|
+          q.bind(exchange, routing_key: config.routing_key)
+        end
+      end
+
+      class << self
+        # Creates a new `MqTopic` from the specified configuration
+        # @param config [TopicConfig] the configuration
+        # @param channel [Bunny::Channel]  The channel to use
+        # @param logger [Logger] the logger
+        def from_config(config, channel:, logger:)
+          new(config.name, channel:, logger:, options: config.options)
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/mq/topic_config.rb

@@ -0,0 +1,37 @@
+module Surfliner
+  module Mq
+    # Encapsulates topic configuration
+    class TopicConfig
+      # @return [String]  The topic exchange to listen to
+      attr_reader :name
+
+      # @return [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      attr_reader :options
+
+      # @param name [String] topic exchange to listen to
+      # @param options [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+      def initialize(name:, options: {})
+        @name = name
+        @options = options
+      end
+
+      class << self
+        # Returns a default (environment-variable-based) configuration with the
+        # specified options.
+        #
+        # | Variable         | Sample value         | Description         |
+        # |------------------|----------------------|---------------------|
+        # | `RABBITMQ_TOPIC` | `surfliner.metadata` | RabbitMQ topic name |
+        #
+        # @param options [Hash] RabbitMQ topic options. (See Bunny::Channel#topic)
+        # @return [TopicConfig] The configuration.
+        def from_env(**options)
+          TopicConfig.new(
+            name: ENV.fetch("RABBITMQ_TOPIC"),
+            options:
+          )
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/mq.rb

@@ -0,0 +1,4 @@
+Dir.glob(File.expand_path("mq/*.rb", __dir__)).each(&method(:require))
+
+# General RabbitMQ functionality
+module Surfliner::Mq; end

data/lib/surfliner/util/assert.rb

@@ -0,0 +1,21 @@
+module Surfliner
+  # Miscellaneous utility module
+  module Util
+    # Helper methods for argument validation
+    module Assert
+      class << self
+        include Assert
+      end
+
+      # @param arg_value [Object, nil] the argument value
+      # @param arg_name [String] the name of the argument, for error messages
+      # @return arg_value
+      # @raise [ArgumentError] if `arg_value` is `nil`
+      def not_nil!(arg_value, arg_name = "argument")
+        return arg_value unless arg_value.nil?
+
+        raise ArgumentError, "#{arg_name} must not be nil"
+      end
+    end
+  end
+end

data/lib/surfliner/util.rb

@@ -0,0 +1,4 @@
+Dir.glob(File.expand_path("util/*.rb", __dir__)).each(&method(:require))
+
+# Miscellaneous utility module
+module Surfliner::Util; end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: surfliner-metadata_consumer
 version: !ruby/object:Gem::Version
-  version: 0.1.0.pre.alpha.4
+  version: 0.1.0.pre.alpha.6
 platform: ruby
 authors:
 - Project Surfliner
 bindir: bin
 cert_chain: []
-date: 2025-03-17 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bunny
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.23'
+        version: '2.24'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.23'
+        version: '2.24'
 - !ruby/object:Gem::Dependency
   name: opentelemetry-exporter-otlp
   requirement: !ruby/object:Gem::Requirement
@@ -86,103 +86,117 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
-  name: debug
+  name: ci_reporter_rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.2
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.9.2
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '0.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
-  name: standard
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.31'
+        version: 1.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.31'
+        version: 1.9.2
 - !ruby/object:Gem::Dependency
-  name: ci_reporter_rspec
+  name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.7'
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: github-markup
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
-  name: dotenv
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '3.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '13.0'
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.13'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -211,6 +225,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.31'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.31'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -240,7 +268,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.9.37
 executables:
-- daylight-index-listen
+- index-on-publish
 - simulate-publish-event
 extensions: []
 extra_rdoc_files: []
@@ -252,12 +280,10 @@ files:
 - LICENSE
 - README.md
 - Rakefile
-- bin/daylight-index-listen
+- bin/index-on-publish
 - bin/simulate-publish-event
 - lib/surfliner/metadata_consumer.rb
 - lib/surfliner/metadata_consumer/consumer.rb
-- lib/surfliner/metadata_consumer/mq_config.rb
-- lib/surfliner/metadata_consumer/mq_connection.rb
 - lib/surfliner/metadata_consumer/payload.rb
 - lib/surfliner/metadata_consumer/solr.rb
 - lib/surfliner/metadata_consumer/solr/delete_handler.rb
@@ -265,6 +291,14 @@ files:
 - lib/surfliner/metadata_consumer/solr/message_handler.rb
 - lib/surfliner/metadata_consumer/superskunk_client.rb
 - lib/surfliner/metadata_consumer/version.rb
+- lib/surfliner/mq.rb
+- lib/surfliner/mq/connection.rb
+- lib/surfliner/mq/connection_config.rb
+- lib/surfliner/mq/queue_config.rb
+- lib/surfliner/mq/topic.rb
+- lib/surfliner/mq/topic_config.rb
+- lib/surfliner/util.rb
+- lib/surfliner/util/assert.rb
 homepage: https://gitlab.com/surfliner/metadata_consumer
 licenses:
 - MIT
@@ -286,7 +320,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Surfliner metadata consumer
 test_files: []

data/bin/daylight-index-listen

@@ -1,23 +0,0 @@
-#!/usr/bin/env ruby
-require "bundler/setup"
-require "logger"
-require "opentelemetry/sdk"
-
-require "surfliner/metadata_consumer"
-
-unless ENV["OTEL_SDK_DISABLED"] == "true"
-  OpenTelemetry::SDK.configure do |c|
-    c.service_name = "surfliner-daylight-consumer"
-    c.use_all # enables auto instrumentation for Bunny, Net::HTTP, etc...
-  end
-end
-tracer   = OpenTelemetry.tracer_provider.tracer("DaylightConsumerTracer")
-
-$stdout.sync = true # don't buffer log output
-logger       = Logger.new($stdout).tap do |logger|
-  logger.level = ENV.fetch("LOG_LEVEL", Logger::INFO)
-end
-
-handler  = Surfliner::MetadataConsumer::Solr::MessageHandler
-consumer = Surfliner::MetadataConsumer::Consumer.new(tracer:, logger:, handler:)
-consumer.run