surfliner-metadata_consumer 0.1.0.pre.alpha

data/lib/surfliner/metadata_consumer/consumer.rb

@@ -0,0 +1,47 @@
+require "rsolr"
+require "surfliner/metadata_consumer/mq_connection"
+require "surfliner/metadata_consumer/solr/message_handler"
+
+module Surfliner
+  module MetadataConsumer
+    # A metadata consumer that subscribes to a RabbitMQ queue and passes
+    # messages to the specified handler.
+    class Consumer
+      attr_reader :connection, :logger, :tracer, :handler
+
+      # Initializes a new `Consumer`
+      # @param tracer [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
+      # @param logger [Logger] log message destination
+      # @param handler #handle an object accepting a JSON string
+      def initialize(tracer:, logger:, handler:)
+        @connection = MqConnection.new(logger:)
+        @logger = logger
+        @tracer = tracer
+        @handler = handler
+      end
+
+      # Initializes and starts a new `Consumer`
+      # @param tracer [OpenTelemetry::Trace::Tracer] OpenTelemetry tracer
+      # @param logger [Logger] log message destination
+      # @param handler #handle an object accepting a JSON string payload
+      def self.run(tracer:, logger:, handler:)
+        new(tracer:, logger:, handler:).run
+      end
+
+      # Starts listening to the message queue and passing messages to the handler.
+      def run
+        connection.open do |queue|
+          queue.subscribe(block: true) do |_delivery_info, _properties, payload_json|
+            tracer.in_span("surfliner metadata consumer message") do |_span|
+              logger.info(" [  ] message received with payload: #{payload_json}")
+
+              handler.handle(payload_json)
+            end
+          rescue => err
+            logger.error(" [❌] failed to handle message: #{err}\n#{err.backtrace}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/mq_config.rb

@@ -0,0 +1,79 @@
+module Surfliner
+  module MetadataConsumer
+    # An object encapsulating RabbitMQ configuration.
+    class MqConfig
+      # @return [String]  The RabbitMQ hostname
+      attr_reader :host
+
+      # @return [String]  The RabbitMQ AMQP port
+      attr_reader :port
+
+      # @return [String]  The RabbitMQ username
+      attr_reader :username
+
+      # @return [String]  The RabbitMQ passsword
+      attr_reader :password
+
+      # @return [String]  The topic exchange to listen to
+      attr_reader :topic
+
+      # @return [String]  The name of the queue to listen to
+      attr_reader :queue_name
+
+      # @return [String]  The platform routing key to listen to
+      attr_reader :routing_key
+
+      # Initializes a new `MqConfig` object.
+      # @param host [The] RabbitMQ hostname
+      # @param port [The] RabbitMQ AMQP port
+      # @param username [The] RabbitMQ username
+      # @param password [The] RabbitMQ passsword
+      # @param topic [The] topic exchange to listen to
+      # @param queue_name [The] name of the queue to listen to
+      # @param routing_key [The] platform routing key to listen to
+      def initialize(host:, port:, username:, password:, topic:, queue_name:, routing_key:)
+        @host = host
+        @port = port
+        @username = username
+        @password = password
+        @topic = topic
+        @queue_name = queue_name
+        @routing_key = routing_key
+      end
+
+      class << self
+        # Reads RabbitMQ configuration from environment variables and
+        # returns it as a new `MqConfig` object.
+        #
+        # - `RABBITMQ_HOST` → `host`
+        # - `RABBITMQ_NODE_PORT_NUMBER` → `port`
+        # - `RABBITMQ_USERNAME` → `username`
+        # - `RABBITMQ_PASSWORD` → `password`
+        # - `RABBITMQ_TOPIC` → `topic`
+        # - `RABBITMQ_QUEUE` → `queue_name`
+        # - `RABBITMQ_PLATFORM_ROUTING_KEY` → `routing_key`
+        def from_env
+          MqConfig.new(
+            host: ENV.fetch("RABBITMQ_HOST"),
+            port: ENV.fetch("RABBITMQ_NODE_PORT_NUMBER"),
+            username: ENV.fetch("RABBITMQ_USERNAME"),
+            password: ENV.fetch("RABBITMQ_PASSWORD"),
+            topic: ENV.fetch("RABBITMQ_TOPIC"),
+            queue_name: ENV.fetch("RABBITMQ_QUEUE"),
+            routing_key: ENV.fetch("RABBITMQ_PLATFORM_ROUTING_KEY")
+          )
+        end
+      end
+
+      # @return [String]  the connection URL as a string
+      def connection_url
+        @connection_url ||= "amqp://#{username}:#{password}@#{host}:#{port}"
+      end
+
+      # @return [String]  the connection URL as a string, without the password
+      def redacted_url
+        @redacted_url ||= connection_url.sub(password, "REDACTED")
+      end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/mq_connection.rb

@@ -0,0 +1,115 @@
+require "bunny"
+
+module Surfliner
+  module MetadataConsumer
+    # An object encapsulating a RabbitMQ connection.
+    class MqConnection
+      # @return [Logger]  The logger
+      attr_reader :logger
+
+      # @return [Bunny::Session]  The current RabbitMQ session
+      attr_reader :connection
+
+      # @return [Bunny::Channel]  The channel being listened to
+      attr_reader :channel
+
+      # @return [Bunny::Exchange]  The exchange being listened to
+      attr_reader :exchange
+
+      # @return [Bunny::Queue]  The queue being listened to
+      attr_reader :queue
+
+      # @return [MqConfig]  The configuration
+      attr_reader :config
+
+      # Initializes a new `MqConnection`.
+      #
+      # @param logger [Logger] the logger
+      # @param config [MqConfig] the configuration
+      def initialize(logger:, config: MqConfig.from_env)
+        @logger = logger
+        @config = config
+      end
+
+      # Opens a connection.
+      # @return [self]
+      # @raise RuntimeError if already connected
+      def connect
+        raise "RabbitMQ connection #{connection} already open." if open?
+
+        logger.info("Rabbitmq message broker connection url: #{config.redacted_url}")
+        @connection = Bunny.new(config.connection_url, logger: logger)
+        connect_on(connection)
+        @channel = connection.create_channel
+        @exchange = channel.topic(config.topic, auto_delete: true)
+        @queue = channel.queue(config.queue_name, durable: true)
+        queue.bind(exchange, routing_key: config.routing_key)
+
+        self
+      rescue Bunny::TCPConnectionFailed => err
+        # TODO: realistically, this only happens in connection.start, where we're eating it
+        logger.error("Connection to #{config.redacted_url} failed")
+        raise err
+      rescue Bunny::PossibleAuthenticationFailureError => err
+        # TODO: realistically, this only happens in connection.start, where we're eating it
+        logger.error("Failed to authenticate to #{config.redacted_url}")
+        raise err
+      end
+
+      # Opens a connection, yields the queue, and closes the connection after
+      # the provided block completes.
+      # @yield [Bunny::Queue]  the queue
+      def open
+        connect
+        yield queue
+      ensure
+        close
+      end
+
+      # Closes the connection.
+      def close
+        channel&.close
+      ensure
+        connection&.close
+      end
+
+      # @return [true, false] True if the connection is open, false otherwise
+      def open?
+        connection&.status == :open
+      end
+
+      # @return [Symbol, nil] The connection status, or nil if there is no connection
+      def status
+        connection&.status
+      end
+
+      # @return [String]  The RabbitMQ hostname
+      def host
+        config.host
+      end
+
+      # @return [String]  The RabbitMQ port
+      def port
+        config.port
+      end
+
+      private
+
+      def connect_on(connection, timeout = 120)
+        timer = 0
+        logger.info "Trying to open queue connection with timeout=#{timeout}"
+        while timer < timeout
+          begin
+            connection.start
+          rescue
+            # TODO: do we actually want to rescue from everything?
+          end
+          return connection if connection.status == :open
+          sleep 1
+          timer += 1
+        end
+        raise "Failed to connect to queue."
+      end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/payload.rb

@@ -0,0 +1,39 @@
+module Surfliner
+  module MetadataConsumer
+    # Encapsulates a Surfliner resource message payload.
+    class Payload
+      # Expected resource status values.
+      KNOWN_STATUSES = [:published, :updated, :unpublished, :deleted]
+
+      # Initializes a new `Payload` from the specified JSON data.
+      # @param payload_json [String] The payload data as received from RabbitMQ.
+      def initialize(payload_json)
+        @data = JSON.parse(payload_json)
+      end
+
+      # @return [String]  The payload data as a JSON string.
+      def to_s
+        @data.to_s
+      end
+
+      # @return [String]  the URL for the resource.
+      def resource_url
+        @resource_url ||= @data.fetch("resourceUrl")
+      end
+
+      # @return [Symbol]  the resource status as a symbol.
+      # @raise UnknownStatus if the message does not provide the resource status.
+      def status
+        @status ||= begin
+          status_str = @data.fetch("status") do
+            raise(UnknownStatus, "Payload status is not defined in payload: #{@data}")
+          end
+          status_str.to_sym
+        end
+      end
+
+      # Exception raised if the resource status has not been provided in the message.
+      class UnknownStatus < RuntimeError; end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/solr/delete_handler.rb

@@ -0,0 +1,15 @@
+require "surfliner/metadata_consumer/solr/message_handler"
+
+module Surfliner
+  module MetadataConsumer
+    module Solr
+      # Message handler for `:deleted` status
+      class DeleteHandler < MessageHandler
+        # @raise NotImplementedError because not implemented
+        def handle
+          raise NotImplementedError, "IMPLEMENT ME: This consumer can't delete yet!"
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/solr/index_handler.rb

@@ -0,0 +1,45 @@
+require "json"
+require "net/http"
+require "rsolr"
+
+require "surfliner/metadata_consumer/solr/message_handler"
+require "surfliner/metadata_consumer/superskunk_client"
+
+module Surfliner
+  module MetadataConsumer
+    module Solr
+      # Message handler that indexes resources into Solr.
+      class IndexHandler < MessageHandler
+        # Attributes for the Solr "add" command; see
+        # https://cwiki.apache.org/confluence/display/solr/UpdateXmlMessages#UpdateXmlMessages-Optionalattributesfor%22add%22
+        SOLR_ATTRIBUTES = {commitWithin: 10}.freeze
+
+        # Retrieves the resource specified in the message from Superskunk, converts
+        # it to a Solr document, and adds it to Solr.
+        def handle
+          index(build_document(SuperskunkClient.get(payload.resource_url)))
+        end
+
+        private
+
+        ##
+        # @return [Hash]
+        def build_document(data)
+          index_document = {id: data["@id"].split("/").last}
+          index_document[:title_tesim] = data["title"]
+          index_document[:creator_ssim] = data["creator"]
+          index_document[:ark_si] = data["ark"]
+          index_document[:superskunk_uri_si] = data["@id"]
+          index_document
+        end
+
+        ##
+        # @param doc [Hash]
+        # @return [void]
+        def index(doc)
+          solr_connection.add([doc], add_attributes: SOLR_ATTRIBUTES)
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/solr/message_handler.rb

@@ -0,0 +1,73 @@
+require "json"
+require "surfliner/metadata_consumer/payload"
+
+module Surfliner
+  module MetadataConsumer
+    module Solr
+      # A message handler that delegates to sub-handlers based on the resource
+      # status provided in the message.
+      class MessageHandler
+        attr_reader :payload
+
+        # Connects to the configured Solr instance.
+        # @return [RSolr::Client]  The Solr connection.
+        def solr_connection
+          @solr_connection ||= begin
+            solr_host = ENV.fetch("SOLR_HOST")
+            solr_port = ENV.fetch("SOLR_PORT")
+            solr_collection_name = ENV.fetch("SOLR_COLLECTION_NAME")
+
+            solr_url = "http://#{solr_auth}#{solr_host}:#{solr_port}/solr/#{solr_collection_name}"
+            RSolr.connect(url: solr_url)
+          end
+        end
+
+        class << self
+          # Returns the appropriate handler based on the resource status provided in the message.
+          # @param payload_json [String] JSON message payload
+          # @return [#handle]
+          def handler_for(payload_json)
+            payload = Payload.new(payload_json)
+
+            case payload.status
+            when :published, :updated
+              IndexHandler.new(payload)
+            when :unpublished, :deleted
+              DeleteHandler.new(payload)
+            else
+              raise ArgumentError, "Couldn't handle message with payload status: #{payload.status}"
+            end
+          end
+
+          # Delegates payload handling to the appropriate handler.
+          # @param payload_json [String] JSON message payload
+          def handle(payload_json)
+            handler_for(payload_json).handle
+          end
+        end
+
+        # Initializes a new `MessageHandler`.
+        # @param payload [Payload]
+        def initialize(payload)
+          @payload = payload
+        end
+
+        # Implementations should handle the `Payload` provided in the initializer.
+        # @abstract
+        def handle
+          raise NotImplementedError
+        end
+
+        private
+
+        def solr_auth
+          solr_admin_user = ENV.fetch("SOLR_ADMIN_USER", nil)
+          solr_admin_password = ENV.fetch("SOLR_ADMIN_PASSWORD", nil)
+          return "" unless solr_admin_user && solr_admin_password
+
+          "#{solr_admin_user}:#{solr_admin_password}@"
+        end
+      end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/solr.rb

@@ -0,0 +1,9 @@
+Dir.glob(File.expand_path("solr/*.rb", __dir__)).sort.each(&method(:require))
+
+module Surfliner
+  module MetadataConsumer
+    # Message handlers for Solr indexing.
+    module Solr
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/superskunk_client.rb

@@ -0,0 +1,45 @@
+module Surfliner
+  module MetadataConsumer
+    # Retrieves resources from Superskunk.
+    class SuperskunkClient
+      # The JSON-LD profile to request when retrieving the resource
+      DEFAULT_JSONLD_PROFILE = "tag:surfliner.gitlab.io,2022:api/oai_dc"
+
+      class << self
+        # Retrieves the specified resource.
+        # @param url [String] The resource URL as a string
+        # @return [Hash] parsed JSON response data
+        #
+        # @raise [UnexpectedResponse] in the event of an unexpected HTTP status code.
+        def get(url)
+          uri = URI(url)
+          req = Net::HTTP::Get.new(uri)
+          req["Accept"] = "application/ld+json;profile=\"#{jsonld_profile}\""
+          req["User-Agent"] = ENV.fetch("USER_AGENT_PRODUCT_NAME") { "surfliner.daylight" } # TODO: make this more obviously configurable?
+
+          response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
+            http.request(req)
+          end
+
+          case response
+          when Net::HTTPSuccess
+            JSON.parse(response.body)
+          when Net::HTTPRedirection
+            get(response["location"])
+          else
+            raise UnexpectedResponse, "Failed to fetch data; status #{response.code}"
+          end
+        end
+
+        private
+
+        def jsonld_profile
+          DEFAULT_JSONLD_PROFILE
+        end
+      end
+
+      # Exception raised in the event of an unexpected HTTP status code.
+      class UnexpectedResponse < RuntimeError; end
+    end
+  end
+end

data/lib/surfliner/metadata_consumer/version.rb

@@ -0,0 +1,8 @@
+# Umbrella module for general Surfliner code
+module Surfliner
+  # Parent module for this gem
+  module MetadataConsumer
+    # The gem version
+    VERSION = "0.1.0.pre.alpha"
+  end
+end

data/lib/surfliner/metadata_consumer.rb

@@ -0,0 +1 @@
+Dir.glob(File.expand_path("metadata_consumer/*.rb", __dir__)).sort.each(&method(:require))

data/surfliner-metadata_consumer.gemspec

@@ -0,0 +1,42 @@
+require_relative "lib/surfliner/metadata_consumer/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "surfliner-metadata_consumer"
+  spec.version = Surfliner::MetadataConsumer::VERSION
+  spec.authors = ["Project Surfliner"]
+
+  spec.homepage = "https://gitlab.com/surfliner/metadata_consumer"
+  spec.license = "MIT"
+  spec.summary = "Surfliner metadata consumer"
+
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3.1")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://gitlab.com/surfliner/metadata_consumer.git"
+  spec.metadata["rubygems_mfa_required"] = "false"
+
+  spec.files = Dir["lib/**/*.rb"] + Dir["bin/*"] + Dir["[A-Z]*"]
+
+  ["daylight-index-listen", "simulate-publish-event"].each do |script|
+    spec.executables << script
+  end
+
+  spec.add_dependency "bunny", "~> 2.23"
+  # TODO: Figure out why we get "The otlp exporter cannot be configured - please add opentelemetry-exporter-otlp to your Gemfile"
+  spec.add_dependency "opentelemetry-exporter-otlp", "~> 0.26.3"
+  spec.add_dependency "opentelemetry-instrumentation-all", "~> 0.60.0"
+  spec.add_dependency "opentelemetry-sdk", "~> 1.4.1"
+  spec.add_dependency "rsolr", ">= 1.0", "< 3"
+
+  spec.add_development_dependency "debug", "~> 1.9.2"
+  spec.add_development_dependency "rspec", "~> 3.13"
+  spec.add_development_dependency "standard", "~> 1.31"
+  spec.add_development_dependency "ci_reporter_rspec", "~> 1.0"
+  spec.add_development_dependency "colorize", "~> 0.8"
+  spec.add_development_dependency "dotenv", "~> 2.7"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "simplecov", "~> 0.22"
+  spec.add_development_dependency "simplecov-cobertura", "~> 2.1"
+  spec.add_development_dependency "webmock", "~> 3.12"
+  spec.add_development_dependency "yard", "~> 0.9.37"
+end