cloudenvoy 0.1.0.dev → 0.4.0

data/lib/cloudenvoy/pub_sub_client.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Cloudenvoy
+  # Interface to publishing backend (GCP, emulator or memory backend)
+  class PubSubClient
+    #
+    # Return the backend to use for sending messages.
+    #
+    # @return [Module<Cloudenvoy::Backend::MemoryPubSub, Cloudenvoy::Backend::GoogleCloudTask>] The backend class.
+    #
+    def self.backend
+      # Re-evaluate backend every time if testing mode enabled
+      @backend = nil if defined?(Cloudenvoy::Testing)
+
+      @backend ||= begin
+        if defined?(Cloudenvoy::Testing) && Cloudenvoy::Testing.in_memory?
+          require 'cloudenvoy/backend/memory_pub_sub'
+          Backend::MemoryPubSub
+        else
+          require 'cloudenvoy/backend/google_pub_sub'
+          Backend::GooglePubSub
+        end
+      end
+    end
+
+    #
+    # Publish a message to a topic.
+    #
+    # @param [String] topic The name of the topic
+    # @param [Hash, String] payload The message content.
+    # @param [Hash] attrs The message attributes.
+    #
+    # @return [Cloudenvoy::Message] The created message.
+    #
+    def self.publish(topic, payload, attrs = {})
+      backend.publish(topic, payload, attrs)
+    end
+
+    #
+    # Create or update a subscription for a specific topic.
+    #
+    # @param [String] topic The name of the topic
+    # @param [String] name The name of the subscription
+    # @param [Hash] opts The subscription configuration options
+    # @option opts [Integer] :deadline The maximum number of seconds after a subscriber receives a message
+    #   before the subscriber should acknowledge the message.
+    # @option opts [Boolean] :retain_acked Indicates whether to retain acknowledged messages. If true,
+    #   then messages are not expunged from the subscription's backlog, even if they are acknowledged,
+    #   until they fall out of the retention window. Default is false.
+    # @option opts [<Type>] :retention How long to retain unacknowledged messages in the subscription's
+    #   backlog, from the moment a message is published. If retain_acked is true, then this also configures
+    #   the retention of acknowledged messages, and thus configures how far back in time a Subscription#seek
+    #   can be done. Cannot be more than 604,800 seconds (7 days) or less than 600 seconds (10 minutes).
+    #   Default is 604,800 seconds (7 days).
+    # @option opts [String] :filter An expression written in the Cloud Pub/Sub filter language.
+    #   If non-empty, then only Message instances whose attributes field matches the filter are delivered
+    #   on this subscription. If empty, then no messages are filtered out. Optional.
+    #
+    # @return [Cloudenvoy::Subscription] The upserted subscription.
+    #
+    def self.upsert_subscription(topic, name, opts = {})
+      backend.upsert_subscription(topic, name, opts)
+    end
+
+    #
+    # Create or update a topic.
+    #
+    # @param [String] topic The topic name.
+    #
+    # @return [Cloudenvoy::Topic] The upserted/topic.
+    #
+    def self.upsert_topic(topic)
+      backend.upsert_topic(topic)
+    end
+  end
+end

data/lib/cloudenvoy/publisher.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+module Cloudenvoy
+  # Use this module to define publishers. The module provides
+  # a simple DSL for transforming and publishing data objects to
+  # Pub/Sub.
+  #
+  # Publishers must at least implement the `payload` method, which defines
+  # how arguments are mapped to a Hash or String message.
+  #
+  # E.g.
+  #
+  # class UserPublisher
+  #   include Cloudenvoy::Publisher
+  #
+  #   # Specify publishing options
+  #   cloudenvoy_options topic: 'my-topic'
+  #
+  #   # Format message objects
+  #   def payload(user)
+  #     {
+  #       id: user.id,
+  #       name: user.name
+  #     }
+  #   end
+  # end
+  #
+  module Publisher
+    # Add class method to including class
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.attr_accessor :msg_args, :message, :publishing_started_at, :publishing_ended_at
+
+      # Register subscriber
+      Cloudenvoy.publishers.add(base)
+    end
+
+    # Module class methods
+    module ClassMethods
+      #
+      # Set the publisher runtime options.
+      #
+      # @param [Hash] opts The publisher options.
+      #
+      # @return [Hash] The options set.
+      #
+      def cloudenvoy_options(opts = {})
+        opt_list = opts&.map { |k, v| [k.to_sym, v] } || [] # symbolize
+        @cloudenvoy_options_hash = Hash[opt_list]
+      end
+
+      #
+      # Return the publisher runtime options.
+      #
+      # @return [Hash] The publisher runtime options.
+      #
+      def cloudenvoy_options_hash
+        @cloudenvoy_options_hash || {}
+      end
+
+      #
+      # Return the default topic this publisher publishes to.
+      # Raises an error if no default topic has been defined.
+      #
+      # @return [String] The default topic.
+      #
+      def default_topic
+        cloudenvoy_options_hash[:topic]
+      end
+
+      #
+      # Format and publish objects to Pub/Sub.
+      #
+      # @param [Any] *args The publisher arguments
+      #
+      # @return [Cloudenvoy::Message] The created message.
+      #
+      def publish(*args)
+        new(msg_args: args).publish
+      end
+
+      #
+      # Setup the default topic for this publisher.
+      #
+      # @return [Cloudenvoy::Topic] The upserted/topic.
+      #
+      def setup
+        return nil unless default_topic
+
+        PubSubClient.upsert_topic(default_topic)
+      end
+    end
+
+    #
+    # Build a new publisher instance.
+    #
+    # @param [Array<any>] msg_args The list of payload args.
+    #
+    def initialize(msg_args: nil)
+      @msg_args = msg_args || []
+    end
+
+    #
+    # Return the topic to publish to. The topic name
+    # can be dynamically evaluated at runtime based on
+    # publishing arguments.
+    #
+    # Defaults to the topic specified via cloudenvoy_options.
+    #
+    # @param [Any] *_args The publisher arguments.
+    #
+    # @return [String] The topic name.
+    #
+    def topic(*_args)
+      self.class.default_topic
+    end
+
+    #
+    # Publisher can optionally define message attributes (metadata).
+    # Message attributes are sent to Pub/Sub and can be used
+    # for filtering.
+    #
+    # @param [Any] *_args The publisher arguments.
+    #
+    # @return [Hash] The message attributes.
+    #
+    def metadata(*_args)
+      {}
+    end
+
+    #
+    # Return the Cloudenvoy logger instance.
+    #
+    # @return [Logger, any] The cloudenvoy logger.
+    #
+    def logger
+      @logger ||= PublisherLogger.new(self)
+    end
+
+    #
+    # Return the time taken (in seconds) to format and publish the message. This duration
+    # includes the middlewares and the actual publish method.
+    #
+    # @return [Float] The time taken in seconds as a floating point number.
+    #
+    def publishing_duration
+      return 0.0 unless publishing_ended_at && publishing_started_at
+
+      (publishing_ended_at - publishing_started_at).ceil(3)
+    end
+
+    #
+    # Send the instantiated Publisher (message) to
+    # Pub/Sub.
+    #
+    # @return [Cloudenvoy::Message] The created message.
+    #
+    def publish
+      # Format and publish message
+      resp = execute_middleware_chain
+
+      # Log job completion and return result
+      logger.info("Published message in #{publishing_duration}s") { { duration: publishing_duration } }
+      resp
+    rescue StandardError => e
+      logger.info("Publishing failed after #{publishing_duration}s") { { duration: publishing_duration } }
+      raise(e)
+    end
+
+    #=============================
+    # Private
+    #=============================
+    private
+
+    #
+    # Internal logic used to build, publish and capture message on the
+    # publisher.
+    #
+    # @return [Cloudenvoy::Message] The published message
+    #
+    def publish_message
+      # Publish message to pub/sub and attach created
+      # message to publisher
+      self.message = PubSubClient.publish(
+        topic(*msg_args),
+        payload(*msg_args),
+        metadata(*msg_args)
+      )
+    end
+
+    #
+    # Execute the subscriber process method through the middleware chain.
+    #
+    # @return [Any] The result of the perform method.
+    #
+    def execute_middleware_chain
+      self.publishing_started_at = Time.now
+
+      Cloudenvoy.config.publisher_middleware.invoke(self) do
+        begin
+          publish_message
+        rescue StandardError => e
+          try(:on_error, e)
+          return raise(e)
+        end
+      end
+    ensure
+      self.publishing_ended_at = Time.now
+    end
+  end
+end

data/lib/cloudenvoy/publisher_logger.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Cloudenvoy
+  # Logger configuration for publishers
+  class PublisherLogger < LoggerWrapper
+    #
+    # The subscriber default context processor.
+    #
+    # @return [Proc] The context processor proc.
+    #
+    def self.default_context_processor
+      @default_context_processor ||= ->(loggable) { loggable.message&.to_h&.slice(:id, :metadata, :topic) || {} }
+    end
+
+    #
+    # Format main log message.
+    #
+    # @param [String] msg The message to log.
+    #
+    # @return [String] The formatted log message
+    #
+    def formatted_message(msg)
+      [
+        '[Cloudenvoy]',
+        "[#{loggable.class}]",
+        loggable.message&.id ? "[#{loggable.message.id}]" : nil,
+        ' ',
+        msg
+      ].compact.join
+    end
+  end
+end

data/lib/cloudenvoy/subscriber.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module Cloudenvoy
+  # Use this module to define subscribers. Subscribers must implement
+  # the message processsing logic in the `process` method.
+  #
+  # E.g.
+  #
+  # class UserSubscriber
+  #   include Cloudenvoy::Subscriber
+  #
+  #   # Specify subscription options
+  #   cloudenvoy_options topics: ['my-topic']
+  #
+  #   # Process message objects
+  #   def process(message)
+  #     ...do something...
+  #   end
+  # end
+  #
+  module Subscriber
+    # Add class method to including class
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.attr_accessor :message, :process_started_at, :process_ended_at
+
+      # Register subscriber
+      Cloudenvoy.subscribers.add(base)
+    end
+
+    #
+    # Return the subscriber class for the provided
+    # class name.
+    #
+    # @param [String] sub_uri The subscription uri.
+    #
+    # @return [Class] The subscriber class
+    #
+    def self.from_sub_uri(sub_uri)
+      klass_name = Subscriber.parse_sub_uri(sub_uri)[0]
+
+      # Check that subscriber class is a valid subscriber
+      sub_klass = Object.const_get(klass_name.camelize)
+
+      sub_klass.include?(self) ? sub_klass : nil
+    end
+
+    #
+    # Parse the subscription name and return the subscriber name and topic.
+    #
+    # @param [String] sub_uri The subscription URI
+    #
+    # @return [Array<String,String>] A tuple [subscriber_name, topic ]
+    #
+    def self.parse_sub_uri(sub_uri)
+      sub_uri.split('/').last.split('.').last(2)
+    end
+
+    #
+    # Execute a subscriber from a payload object received from
+    # Pub/Sub.
+    #
+    # @param [Hash] input_payload The Pub/Sub webhook hash describing
+    # the message to process.
+    #
+    # @return [Any] The subscriber processing return value.
+    #
+    def self.execute_from_descriptor(input_payload)
+      message = Message.from_descriptor(input_payload)
+      subscriber = message.subscriber || raise(InvalidSubscriberError)
+      subscriber.execute
+    end
+
+    # Module class methods
+    module ClassMethods
+      #
+      # Set the subscriber runtime options.
+      #
+      # @param [Hash] opts The subscriber options.
+      #
+      # @return [Hash] The options set.
+      #
+      def cloudenvoy_options(opts = {})
+        opt_list = opts&.map { |k, v| [k.to_sym, v] } || [] # symbolize
+        @cloudenvoy_options_hash = Hash[opt_list]
+      end
+
+      #
+      # Return the subscriber runtime options.
+      #
+      # @return [Hash] The subscriber runtime options.
+      #
+      def cloudenvoy_options_hash
+        @cloudenvoy_options_hash || {}
+      end
+
+      #
+      # Return the list of topics this subscriber listens
+      # to.
+      #
+      # @return [Array<Hash>] The list of subscribed topics.
+      #
+      def topics
+        @topics ||= [cloudenvoy_options_hash[:topic], cloudenvoy_options_hash[:topics]].flatten.compact.map do |t|
+          t.is_a?(String) ? { name: t } : t
+        end
+      end
+
+      #
+      # Return the subscription name used by this subscriber
+      # to subscribe to a specific topic.
+      #
+      # @return [String] The subscription name.
+      #
+      def subscription_name(topic)
+        [
+          Cloudenvoy.config.gcp_sub_prefix.tr('.', '-'),
+          to_s.underscore,
+          topic
+        ].join('.')
+      end
+
+      #
+      # Create the Subscriber subscription in Pub/Sub.
+      #
+      # @return [Array<Cloudenvoy::Subscription>] The upserted subscription.
+      #
+      def setup
+        topics.map do |t|
+          topic_name = t[:name] || t['name']
+          sub_opts = t.reject { |k, _| k.to_sym == :name }
+          PubSubClient.upsert_subscription(topic_name, subscription_name(topic_name), sub_opts)
+        end
+      end
+    end
+
+    #
+    # Build a new subscriber instance.
+    #
+    # @param [Cloudenvoy::Message] message The message to process.
+    #
+    def initialize(message:)
+      @message = message
+    end
+
+    #
+    # Return the Cloudenvoy logger instance.
+    #
+    # @return [Logger, any] The cloudenvoy logger.
+    #
+    def logger
+      @logger ||= SubscriberLogger.new(self)
+    end
+
+    #
+    # Return the time taken (in seconds) to process the message. This duration
+    # includes the middlewares and the actual process method.
+    #
+    # @return [Float] The time taken in seconds as a floating point number.
+    #
+    def process_duration
+      return 0.0 unless process_ended_at && process_started_at
+
+      (process_ended_at - process_started_at).ceil(3)
+    end
+
+    #
+    # Execute the subscriber's logic.
+    #
+    # @return [Any] The logic return value
+    #
+    def execute
+      logger.info('Processing message...')
+
+      # Process message
+      resp = execute_middleware_chain
+
+      # Log processing completion and return result
+      logger.info("Processing done after #{process_duration}s") { { duration: process_duration } }
+      resp
+    rescue StandardError => e
+      logger.info("Processing failed after #{process_duration}s") { { duration: process_duration } }
+      raise(e)
+    end
+
+    #
+    # Equality operator.
+    #
+    # @param [Any] other The object to compare.
+    #
+    # @return [Boolean] True if the object is equal.
+    #
+    def ==(other)
+      other.is_a?(self.class) && other.message == message
+    end
+
+    #=============================
+    # Private
+    #=============================
+    private
+
+    #
+    # Execute the subscriber process method through the middleware chain.
+    #
+    # @return [Any] The result of the perform method.
+    #
+    def execute_middleware_chain
+      self.process_started_at = Time.now
+
+      Cloudenvoy.config.subscriber_middleware.invoke(self) do
+        begin
+          process(message)
+        rescue StandardError => e
+          try(:on_error, e)
+          return raise(e)
+        end
+      end
+    ensure
+      self.process_ended_at = Time.now
+    end
+  end
+end