pgmq-ruby 0.7.2 → 0.8.0

data/lib/pgmq/consumer.rb

@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+module PGMQ
+  # Minimal single-threaded consumer for one or more queues
+  #
+  # Wraps an existing {PGMQ::Client} with a blocking `each` loop, similar to the simple consumer of rdkafka-ruby. It
+  # long-polls the queue with `read_with_poll` (or several queues with `read_multi_with_poll`), yields one message at a
+  # time and acknowledges it (delete or archive) on its own queue when the block returns. The consumer does not own the
+  # client: closing the client stays the caller's job.
+  #
+  # Delivery is at-least-once. When the block raises, the message is not acknowledged and becomes visible again once
+  # its visibility timeout expires. When processing takes longer than `vt`, another reader may receive the same message.
+  #
+  # @example Basic usage
+  #   client = PGMQ::Client.new(ENV["DATABASE_URL"])
+  #   consumer = PGMQ::Consumer.new(client, "orders", vt: 30)
+  #
+  #   trap("TERM") { consumer.stop }
+  #   trap("INT") { consumer.stop }
+  #
+  #   consumer.each do |message|
+  #     process(JSON.parse(message.message))
+  #   end
+  #
+  #   client.close
+  #
+  # @example Several queues
+  #   consumer = PGMQ::Consumer.new(client, %w[orders emails], vt: 30)
+  #
+  #   consumer.each do |message|
+  #     puts "#{message.queue_name}: #{message.message}"
+  #   end
+  class Consumer
+    # Validates the options before the consumer is built
+    CONTRACT = Contracts::Consumer.new
+
+    private_constant :CONTRACT
+
+    # @return [PGMQ::Client] client used for all queue operations
+    attr_reader :client
+
+    # @return [Array<String>] names of the consumed queues
+    attr_reader :queue_names
+
+    # Creates a new consumer
+    #
+    # @param client [PGMQ::Client] client used for all queue operations
+    # @param queue_names [String, Array<String>] name of the queue to consume, or names of up to 50 queues
+    # @param vt [Integer] visibility timeout in seconds for read messages
+    # @param batch_size [Integer] number of messages to read per queue per poll (still yielded one at a time)
+    # @param max_poll_seconds [Integer] how long a single poll waits for messages
+    # @param poll_interval_ms [Integer] interval between server-side checks within a poll
+    # @param on_success [Symbol] `:delete` or `:archive`, applied after the block returns
+    # @param max_attempts [Integer, nil] archive without yielding once `read_ct` exceeds this value (nil disables).
+    #   Requires `batch_size: 1`; with several queues, one message is then read per poll
+    # @raise [PGMQ::Errors::ConfigurationError] when an option or the list of queues is invalid
+    # @raise [PGMQ::Errors::InvalidQueueNameError] when a queue name is invalid
+    #
+    # @example
+    #   consumer = PGMQ::Consumer.new(client, "orders", on_success: :archive, max_attempts: 5)
+    def initialize(
+      client,
+      queue_names,
+      vt: Client::DEFAULT_VT,
+      batch_size: 1,
+      max_poll_seconds: 5,
+      poll_interval_ms: 100,
+      on_success: :delete,
+      max_attempts: nil
+    )
+      queue_names = Array(queue_names)
+
+      CONTRACT.validate!(
+        {
+          queue_names: queue_names,
+          vt: vt,
+          batch_size: batch_size,
+          max_poll_seconds: max_poll_seconds,
+          poll_interval_ms: poll_interval_ms,
+          on_success: on_success,
+          max_attempts: max_attempts
+        },
+        Errors::ConfigurationError
+      )
+
+      @client = client
+      @queue_names = queue_names.map { |name| QueueName.validate!(name) }.freeze
+      @vt = vt
+      @batch_size = batch_size
+      @max_poll_seconds = max_poll_seconds
+      @poll_interval_ms = poll_interval_ms
+      @on_success = on_success
+      @max_attempts = max_attempts
+      @poll_offset = 0
+      @stopped = false
+    end
+
+    # Consumes messages until {#stop} is called
+    #
+    # Each message is acknowledged with the `on_success` action on its own queue after the block returns. An exception
+    # raised by the block stops the loop and is re-raised: that message is left to reappear after its visibility
+    # timeout, and the not yet yielded messages of the same batch are released at once. Messages read more than
+    # `max_attempts` times are archived without being yielded.
+    #
+    # {#stop} takes effect once the messages of the current poll are processed, or after the current poll (up to
+    # `max_poll_seconds`) when the queue is empty. Messages that were read are always yielded, so a stop never counts
+    # a read that was not processed.
+    #
+    # @yieldparam message [PGMQ::Message] message to process, with `queue_name` set
+    # @return [nil]
+    # @raise [ArgumentError] when no block is given
+    #
+    # @example
+    #   consumer.each { |message| puts message.message }
+    def each
+      raise ArgumentError, "a block is required" unless block_given?
+
+      until @stopped
+        pending = poll
+
+        begin
+          until pending.empty?
+            message = pending.shift
+
+            if exhausted?(message)
+              @client.archive(message.queue_name, message.msg_id)
+              next
+            end
+
+            yield message
+
+            acknowledge(message)
+          end
+        ensure
+          release(pending)
+        end
+      end
+
+      nil
+    end
+
+    # Requests the loop in {#each} to finish
+    #
+    # Only sets a flag, so it is safe to call from a signal trap or from another thread. A stopped consumer stays
+    # stopped: further calls to {#each} return at once.
+    #
+    # @return [void]
+    def stop
+      @stopped = true
+    end
+
+    # @return [Boolean] true once {#stop} was called
+    def stopped?
+      @stopped
+    end
+
+    private
+
+    # Reads the next messages
+    #
+    # With several queues the read order rotates each poll, so a busy queue listed first does not starve the others.
+    #
+    # @return [Array<PGMQ::Message>] read messages, each with its `queue_name` set
+    def poll
+      if @queue_names.size == 1
+        queue_name = @queue_names.first
+
+        messages = @client.read_with_poll(
+          queue_name,
+          vt: @vt,
+          qty: @batch_size,
+          max_poll_seconds: @max_poll_seconds,
+          poll_interval_ms: @poll_interval_ms
+        )
+
+        return messages.map { |message| message.with(queue_name: queue_name) }
+      end
+
+      queue_names = @queue_names.rotate(@poll_offset)
+      @poll_offset = (@poll_offset + 1) % @queue_names.size
+
+      @client.read_multi_with_poll(
+        queue_names,
+        vt: @vt,
+        qty: @batch_size,
+        # A message released after a failure keeps its raised read_ct, so max_attempts allows one read per poll
+        limit: @max_attempts.nil? ? nil : 1,
+        max_poll_seconds: @max_poll_seconds,
+        poll_interval_ms: @poll_interval_ms
+      )
+    end
+
+    # @param message [PGMQ::Message] read message
+    # @return [Boolean] true when the message exceeded the allowed number of reads
+    def exhausted?(message)
+      !@max_attempts.nil? && message.read_ct.to_i > @max_attempts
+    end
+
+    # @param message [PGMQ::Message] processed message
+    def acknowledge(message)
+      if @on_success == :archive
+        @client.archive(message.queue_name, message.msg_id)
+      else
+        @client.delete(message.queue_name, message.msg_id)
+      end
+    end
+
+    # Makes read but not yielded messages visible again right away
+    #
+    # Failures are ignored so they never replace an error raised by the block; the messages then reappear after
+    # their visibility timeout.
+    #
+    # @param messages [Array<PGMQ::Message>] messages to release
+    def release(messages)
+      messages.group_by(&:queue_name).each do |queue_name, queue_messages|
+        @client.set_vt_batch(queue_name, queue_messages.map(&:msg_id), vt: 0)
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/pgmq/contractable/contract.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contractable
+    # Base contract for all the contracts that check data format
+    #
+    # @example
+    #   class MyContract < PGMQ::Contractable::Contract
+    #     self.error_messages = { "id_format" => "must be a String" }
+    #
+    #     required(:id) { |id| id.is_a?(String) }
+    #   end
+    #
+    #   MyContract.new.validate!({ id: 1 }, PGMQ::Errors::ConfigurationError)
+    #
+    # @note This contract does NOT support rules inheritance.
+    class Contract
+      # Constant representing a miss during dig
+      # We use it as a result value not to return an array with found object and a state to prevent additional array
+      # allocation
+      DIG_MISS = Object.new
+
+      # Empty array for scope default to avoid allocating a new Array on each `#call` / `#validate!` invocation. Safe
+      # because scope is never mutated - it is only used in `scope + rule.path` which creates a new Array.
+      EMPTY_ARRAY = [].freeze
+
+      private_constant :DIG_MISS, :EMPTY_ARRAY
+
+      # Class level API definitions
+      class << self
+        # @return [Array<Rule>] all the validation rules defined for a given contract
+        attr_reader :rules
+
+        # @return [Hash{String => String}] error messages keyed by "<path>_<error key>" or "<error key>"
+        attr_accessor :error_messages
+
+        # Allows for definition of a scope/namespace for nested validations
+        #
+        # @param path [Symbol] path in the hash for nesting
+        #
+        # @example
+        #   nested(:key) do
+        #     required(:inside) { |inside| inside.is_a?(String) }
+        #   end
+        def nested(path, &)
+          init_accu
+          @nested << path
+          begin
+            instance_eval(&)
+          ensure
+            # Always pop, even if the block raises, so a rescued exception during contract definition does not
+            # prefix `path` onto every rule defined afterwards
+            @nested.pop
+          end
+        end
+
+        # Defines a rule for a required field (required means, that will automatically create an error if missing)
+        #
+        # @param keys [Array<Symbol>] single or full path
+        # @param block [Proc] validation rule
+        def required(*keys, &block)
+          init_accu
+          @rules << Rule.new(@nested + keys, :required, block).freeze
+        end
+
+        # Defines a rule for an optional field (no error when missing)
+        #
+        # @param keys [Array<Symbol>] single or full path
+        # @param block [Proc] validation rule
+        def optional(*keys, &block)
+          init_accu
+          @rules << Rule.new(@nested + keys, :optional, block).freeze
+        end
+
+        # Defines a virtual rule that validates the whole data rather than a single key. Unlike
+        # `required`/`optional`, the block receives the full data and returns its own errors.
+        #
+        # @param block [Proc] validation rule, called with `(data, errors, contract)`. It must return either a
+        #   non-Array (`true`/`nil`/`false`) for "no errors", or an `Array` of `[path, message]` error pairs (where
+        #   `path` is itself an `Array` of symbols).
+        #
+        # @note The returned error `Array` is owned by the contract: `#call` prepends the current scope onto each pair
+        #   in place and collects them, so a rule must return a freshly built `Array` on every call.
+        def virtual(&block)
+          init_accu
+          @rules << Rule.new([], :virtual, block).freeze
+        end
+
+        private
+
+        # Initializes nestings and rules building accumulator
+        def init_accu
+          @nested ||= []
+          @rules ||= []
+        end
+      end
+
+      # Runs the validation
+      #
+      # @param data [Hash] hash with data we want to validate
+      # @param scope [Array<String, Integer>] scope of this contract (if any) or empty array if no parent scope is
+      #   needed if contract starts from root
+      # @return [Result] validation result
+      def call(data, scope: EMPTY_ARRAY)
+        errors = []
+
+        # A non-Hash root has no keys to dig, so every required rule reports missing
+        data_is_hash = data.is_a?(Hash)
+
+        self.class.rules.each do |rule|
+          if rule.type == :virtual
+            result = rule.validator.call(data, errors, self)
+
+            # Any non-Array result (true, but also nil/false) means "no errors"
+            next unless result.is_a?(Array)
+
+            result.each do |sub_result|
+              sub_result[0] = scope + sub_result[0]
+            end
+
+            errors.push(*result)
+          else
+            for_checking = data_is_hash ? dig(data, rule.path) : DIG_MISS
+
+            if DIG_MISS.equal?(for_checking)
+              errors << [scope + rule.path, :missing] if rule.type == :required
+            else
+              result = rule.validator.call(for_checking, data, errors, self)
+
+              next if result == true
+
+              errors << [scope + rule.path, result || :format]
+            end
+          end
+        end
+
+        return Result.success if errors.empty?
+
+        Result.new(errors, self)
+      end
+
+      # Runs the validation like `#call`, but raises instead of returning a failed `Result`
+      #
+      # @param data [Hash] data for validation
+      # @param error_class [Class] error class that should be used when validation fails
+      # @param scope [Array<String, Integer>] scope of this contract (if any) or empty array if no parent scope is
+      #   needed if contract starts from root
+      # @return [Boolean] true
+      # @raise [StandardError] any error provided in the error_class that inherits from the standard error
+      def validate!(data, error_class, scope: EMPTY_ARRAY)
+        result = call(data, scope: scope)
+
+        return true if result.success?
+
+        raise error_class, result.errors
+      end
+
+      private
+
+      # Tries to dig for a given key in a hash and returns it, or DIG_MISS when not found (dig returns nil and we
+      # don't know if it wasn't the digged key value)
+      #
+      # @param data [Hash]
+      # @param keys [Array<Symbol>]
+      # @return [DIG_MISS, Object] found element or DIG_MISS indicating that not found
+      def dig(data, keys)
+        case keys.length
+        when 1
+          data.fetch(keys[0], DIG_MISS)
+        when 2
+          mid = data.fetch(keys[0], DIG_MISS)
+
+          return DIG_MISS if DIG_MISS.equal?(mid)
+          return DIG_MISS unless mid.is_a?(Hash)
+
+          mid.fetch(keys[1], DIG_MISS)
+        else
+          current = data
+
+          keys.each do |nesting|
+            return DIG_MISS unless current.is_a?(Hash)
+
+            current = current.fetch(nesting, DIG_MISS)
+
+            return DIG_MISS if DIG_MISS.equal?(current)
+          end
+
+          current
+        end
+      end
+    end
+  end
+end

data/lib/pgmq/contractable/result.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contractable
+    # Representation of a validation result with resolved error messages
+    class Result
+      # @return [Hash{Symbol => String}] error messages keyed by the dot-joined path of the invalid value
+      attr_reader :errors
+
+      # Empty hash that we use when no errors to save on allocations
+      EMPTY_HASH = {}.freeze
+
+      private_constant :EMPTY_HASH
+
+      class << self
+        # @return [Result] cached frozen success result to avoid allocating a new Result on every successful contract
+        #   validation. Safe because successful results are identical regardless of contract.
+        def success
+          @success ||= new([], nil).freeze
+        end
+      end
+
+      # Builds a result object and remaps (if needed) error keys to proper error messages
+      #
+      # @param errors [Array<Array>] array with sub-arrays with paths and error keys
+      # @param contract [Object] contract that generated the error
+      def initialize(errors, contract)
+        # Short track to skip object allocation for the happy path
+        if errors.empty?
+          @errors = EMPTY_HASH
+          return
+        end
+
+        hashed = {}
+
+        errors.each do |error|
+          scope = error.first.map(&:to_s).join(".").to_sym
+
+          # This will allow for usage of custom messages instead of yaml keys if needed
+          hashed[scope] = if error.last.is_a?(String)
+            error.last
+          else
+            build_message(contract, scope, error.last)
+          end
+        end
+
+        @errors = hashed
+      end
+
+      # @return [Boolean] true if no errors
+      def success?
+        errors.empty?
+      end
+
+      private
+
+      # Builds message based on the error messages and scope
+      #
+      # @param contract [Object] contract for which we build the result
+      # @param scope [Symbol] path to the key that has an error
+      # @param error_key [Symbol] error key for yaml errors lookup
+      # @return [String] error message
+      def build_message(contract, scope, error_key)
+        messages = contract.class.error_messages
+
+        scope_parts = scope.to_s.split(".")
+
+        # Try full scope first, then progressively remove from beginning, so dynamic scopes (for example a batch
+        # index in 'messages.3.message') fall back to the key's own message
+        (0..scope_parts.length).each do |i|
+          current_scope_parts = scope_parts[i..]
+
+          key = if current_scope_parts.empty?
+            error_key.to_s
+          else
+            "#{current_scope_parts.join(".")}_#{error_key}"
+          end
+
+          return messages[key] if messages.key?(key)
+        end
+
+        # If nothing found, raise the original error
+        messages.fetch(error_key.to_s) do
+          messages.fetch("#{scope}_#{error_key}")
+        end
+      end
+    end
+  end
+end

data/lib/pgmq/contractable/rule.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contractable
+    # Representation of a single validation rule
+    Rule = Struct.new(:path, :type, :validator)
+  end
+end

data/lib/pgmq/contractable.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module PGMQ
+  # Contract layer for validating user input before it reaches PostgreSQL
+  #
+  # Extracted from karafka-core's contractable layer, without its dependencies. It allows for nested validations,
+  # virtual rules and YAML based error messages.
+  #
+  # @note It is thread-safe to run but validations definitions should happen before threads are used.
+  module Contractable
+  end
+end

data/lib/pgmq/contracts/autovacuum.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contracts
+    # Validates the storage settings of {PGMQ::Client::Autovacuum#tune_autovacuum}
+    #
+    # Parameter names cannot be bound, so only the known names may reach the ALTER TABLE statement, and only values
+    # that coerce to a number.
+    class Autovacuum < Contractable::Contract
+      # Storage parameters that can be set: the union of the queue and archive default keys
+      STORAGE_PARAMS = (
+        Client::Autovacuum::DEFAULT_QUEUE_SETTINGS.keys + Client::Autovacuum::DEFAULT_ARCHIVE_SETTINGS.keys
+      ).uniq.freeze
+
+      # Storage parameters whose values are Floats; everything else is an Integer
+      FLOAT_SETTINGS = %i[
+        autovacuum_vacuum_scale_factor
+        autovacuum_analyze_scale_factor
+      ].freeze
+
+      self.error_messages = Contracts.error_messages("autovacuum")
+
+      %i[queue_settings archive_settings].each do |key|
+        required(key) do |settings|
+          next :format unless (settings.keys - STORAGE_PARAMS).empty?
+          next :values unless settings.all? { |name, value| numeric_setting?(name, value) }
+
+          true
+        end
+      end
+
+      class << self
+        # @param name [Symbol] storage parameter name
+        # @param value [Object] value to check
+        # @return [Boolean] true when the value coerces to the Float or Integer the parameter needs
+        def numeric_setting?(name, value)
+          if FLOAT_SETTINGS.include?(name)
+            !Float(value, exception: false).nil?
+          else
+            !Integer(value, exception: false).nil?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/pgmq/contracts/batch.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contracts
+    # Validates the messages and headers arrays of a batch produce
+    class Batch < Contractable::Contract
+      self.error_messages = Contracts.error_messages("batch")
+
+      required(:messages) { |messages| messages.is_a?(Array) }
+      optional(:headers) { |headers| headers.nil? || headers.is_a?(Array) }
+
+      virtual do |data, errors|
+        next true unless errors.empty?
+        next true if data[:headers].nil?
+        next true if data[:headers].length == data[:messages].length
+
+        [[[:headers], :length]]
+      end
+    end
+  end
+end

data/lib/pgmq/contracts/connection.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contracts
+    # Validates the configuration of {PGMQ::Connection}
+    #
+    # Each key is optional, so one contract serves the constructor and the class-level settings.
+    class Connection < Contractable::Contract
+      self.error_messages = Contracts.error_messages("connection")
+
+      optional(:conn_params) do |conn_params|
+        next :empty if conn_params.nil?
+        next true if conn_params.respond_to?(:call) || conn_params.is_a?(String)
+
+        conn_params.is_a?(Hash) && !conn_params.empty?
+      end
+
+      optional(:reconnectable_error_patterns) do |patterns|
+        Array(patterns).all? { |pattern| pattern.is_a?(String) || pattern.is_a?(Regexp) }
+      end
+
+      optional(:reconnectable_error_classes) do |classes|
+        Array(classes).all? { |klass| klass.is_a?(Class) && klass <= Exception }
+      end
+    end
+  end
+end

data/lib/pgmq/contracts/consumer.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contracts
+    # Validates the options of {PGMQ::Consumer}
+    #
+    # Queue names themselves are validated by {PGMQ::QueueName}.
+    class Consumer < Contractable::Contract
+      # Actions that can be taken on a processed message
+      ON_SUCCESS_ACTIONS = %i[delete archive].freeze
+
+      self.error_messages = Contracts.error_messages("consumer")
+
+      required(:queue_names) do |queue_names|
+        next :empty if queue_names.empty?
+        next :too_many if queue_names.size > MultiQueue::MAX_QUEUES
+        # An invalid name is reported by PGMQ::QueueName.validate! with its own error class
+        next true unless queue_names.all? { |name| QueueName.valid?(name) }
+        # pgmq folds queue names to lower case, so names differing only in case refer to the same queue
+        next :unique unless queue_names.map { |name| name.to_s.downcase }.uniq.size == queue_names.size
+
+        true
+      end
+
+      required(:on_success) { |on_success| ON_SUCCESS_ACTIONS.include?(on_success) }
+
+      %i[vt batch_size max_poll_seconds poll_interval_ms].each do |key|
+        required(key) { |value| value.is_a?(Integer) && value.positive? }
+      end
+
+      optional(:max_attempts) do |max_attempts|
+        max_attempts.nil? || (max_attempts.is_a?(Integer) && max_attempts.positive?)
+      end
+
+      # Every read raises read_ct, so with larger batches the messages released after a failure would have their
+      # unprocessed read counted as a failed attempt
+      virtual do |data, errors|
+        next true unless errors.empty?
+        next true if data[:max_attempts].nil? || data[:batch_size] == 1
+
+        [[[:max_attempts], :batch_size]]
+      end
+    end
+  end
+end

data/lib/pgmq/contracts/message.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module PGMQ
+  module Contracts
+    # Validates a message and its headers before they are bound
+    #
+    # The pg gem reads a Hash bind parameter as a parameter spec and binds `nil` as SQL NULL, so a non-String value
+    # could be stored as NULL without an error.
+    class Message < Contractable::Contract
+      self.error_messages = Contracts.error_messages("message")
+
+      required(:message) { |message| message.is_a?(String) }
+      optional(:headers) { |headers| headers.nil? || headers.is_a?(String) }
+    end
+  end
+end