laravel-cloud-rails 0.1.1

data/lib/laravel_cloud/queue/dispatcher.rb

@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module LaravelCloud
+  module Queue
+    # Pushes jobs to SQS for the Active Job adapter.
+    class Dispatcher
+      # SQS limits: per-message delay, visibility timeout (a retry's backoff),
+      # and SendMessageBatch size.
+      MAX_DELAY_SECONDS = 900
+      MAX_VISIBILITY_SECONDS = 43_200
+      MAX_BATCH_MESSAGES = 10
+      MAX_BATCH_BYTES = 1_048_576
+
+      def initialize(runtime)
+        @runtime = runtime
+      end
+
+      # `delay` is in seconds (a Float, from `wait:` / `wait_until:`).
+      def dispatch(job, delay: 0)
+        delivery = Delivery.current
+
+        if delivery&.retry_of?(job)
+          retry_job(job, delay, delivery)
+        else
+          push(job, queue_for(job), delay)
+        end
+      end
+
+      # `ActiveJob.perform_all_later`. Every job is validated before anything is
+      # sent, so an invalid one raises with nothing half-sent; a job SQS rejects
+      # gets an `enqueue_error`. Returns the number enqueued.
+      def dispatch_all(jobs)
+        entries = jobs.map do |job|
+          queue = queue_for(job)
+          [job, queue, message_for(job, queue, delay_for(job))]
+        end
+
+        entries.group_by { |_, queue, _| queue }.sum do |queue, queue_entries|
+          queue_url = require_queue_url(queue)
+
+          batches(queue_entries).sum { |batch| send_batch(queue, queue_url, batch) }
+        end
+      end
+
+      private
+
+      def config
+        @runtime.config
+      end
+
+      # A `retry_on` re-enqueue of the running job. Rather than pushing a copy,
+      # the original message is released for SQS to redeliver after the backoff:
+      # no duplicate, backoffs up to 12 hours, and it works on FIFO queues.
+      def retry_job(job, delay, delivery)
+        # Round up: truncating a sub-second backoff to 0 would make a hot retry
+        # loop. Long polynomial backoffs are clamped to the 12-hour ceiling.
+        seconds = delay.to_f.ceil.clamp(0, MAX_VISIBILITY_SECONDS)
+
+        # Recorded first, so a failed cross-queue push below still retries here.
+        delivery.release!(seconds)
+        job.provider_job_id = delivery.message_id
+
+        # `retry_on ..., queue:` moves the job, which a release can't do: push
+        # it there as a new message and delete the original. Comparing with the
+        # queue name it arrived with means only an explicit move gets here.
+        return if job.queue_name.to_s == delivery.job_queue_name.to_s
+
+        push(job, queue_for(job), delay)
+        delivery.supersede!
+      end
+
+      def push(job, queue, delay)
+        result = send_message(queue, message_for(job, queue, delay))
+
+        job.provider_job_id = result.message_id
+      end
+
+      def send_message(queue, message)
+        queue_url = require_queue_url(queue)
+
+        result = begin
+          @runtime.sqs.send_message(message.merge(queue_url: queue_url))
+        rescue StandardError => e
+          raise ActiveJob::EnqueueError, "Failed to push to the Laravel Cloud managed queue \"#{queue}\": #{e.message}"
+        end
+
+        @runtime.observer.queued(queue)
+        result
+      end
+
+      def send_batch(queue, queue_url, batch)
+        by_id = batch.each_with_index.to_h { |(job, _, _), index| [index.to_s, job] }
+        messages = batch.each_with_index.map { |(_, _, message), index| message.merge(id: index.to_s) }
+
+        result = begin
+          @runtime.sqs.send_message_batch(queue_url: queue_url, entries: messages)
+        rescue StandardError => e
+          error = ActiveJob::EnqueueError.new("Failed to push to the Laravel Cloud managed queue \"#{queue}\": #{e.message}")
+          by_id.each_value { |job| reject(job, error) }
+          return 0
+        end
+
+        result.successful.each do |entry|
+          job = by_id.fetch(entry.id)
+          job.provider_job_id = entry.message_id
+          job.successfully_enqueued = true
+          @runtime.observer.queued(queue)
+        end
+
+        result.failed.each do |entry|
+          reject(by_id.fetch(entry.id), ActiveJob::EnqueueError.new(
+            "The Laravel Cloud managed queue \"#{queue}\" rejected the job: #{entry.code} #{entry.message}",
+          ))
+        end
+
+        result.successful.size
+      end
+
+      def reject(job, error)
+        job.successfully_enqueued = false
+        job.enqueue_error = error
+      end
+
+      # Split one queue's messages by SendMessageBatch's limits.
+      def batches(entries)
+        batches = []
+        bytes = 0
+
+        entries.each do |entry|
+          size = entry.last[:message_body].bytesize
+
+          if batches.empty? || batches.last.size >= MAX_BATCH_MESSAGES || bytes + size > MAX_BATCH_BYTES
+            batches << []
+            bytes = 0
+          end
+
+          batches.last << entry
+          bytes += size
+        end
+
+        batches
+      end
+
+      def message_for(job, queue, delay)
+        group = attribute(job, :cloud_message_group_id)
+        deduplication = attribute(job, :cloud_message_deduplication_id)
+
+        { message_body: Message.encode(job) }.merge(attributes_for(queue, delay, group: group, deduplication: deduplication))
+      end
+
+      # FIFO queues need a group and deduplication ID and reject delays.
+      # Anything a queue can't honour raises rather than being dropped.
+      def attributes_for(queue, delay, group:, deduplication:)
+        if ManagedQueueConfig.fifo?(queue)
+          if delay.to_f.positive?
+            raise InvalidDispatch, "The FIFO queue \"#{queue}\" does not support a per-message delay; " \
+                                   "remove the wait or dispatch to a standard queue."
+          end
+
+          # Laravel's SqsQueue defaults: one group per queue, unique dedupe IDs.
+          { message_group_id: group || queue, message_deduplication_id: deduplication || SecureRandom.uuid }
+        else
+          # Deduplication is FIFO-only; it would silently do nothing here.
+          unless deduplication.nil?
+            raise InvalidDispatch, "A message deduplication ID was set for the non-FIFO queue \"#{queue}\"; " \
+                                   "deduplication requires a \".fifo\" queue."
+          end
+
+          attributes = {}
+
+          # On a standard queue a group is a fair-queues tenant key.
+          attributes[:message_group_id] = group unless group.nil?
+
+          if delay.to_f.positive?
+            seconds = delay.to_f.ceil
+
+            # Raise rather than quietly run the job early.
+            if seconds > MAX_DELAY_SECONDS
+              raise InvalidDispatch, "A delay of #{seconds} seconds exceeds the SQS maximum of #{MAX_DELAY_SECONDS} " \
+                                     "seconds (15 minutes); SQS cannot delay a message beyond 15 minutes."
+            end
+
+            attributes[:delay_seconds] = seconds
+          end
+
+          attributes
+        end
+      end
+
+      def delay_for(job)
+        job.scheduled_at ? job.scheduled_at.to_f - Time.now.to_f : 0
+      end
+
+      def require_queue_url(queue)
+        config.queue_url_for(queue) ||
+          raise(NotConfigured, "No Laravel Cloud managed queue is configured (LARAVEL_CLOUD_MANAGED_QUEUES_CONFIG is missing).")
+      end
+
+      def attribute(job, name)
+        job.respond_to?(name) ? job.public_send(name) : nil
+      end
+
+      # Active Job's default queue maps to the environment's default managed
+      # queue; any other name is the managed queue of that name.
+      def queue_for(job)
+        name = job.queue_name.to_s
+
+        name == job.class.queue_name_from_part(job.class.default_queue_name) ? config.queue : name
+      end
+    end
+  end
+end

data/lib/laravel_cloud/queue/job_class.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module LaravelCloud
+  module Queue
+    # Validates the job class a message names before constantizing it.
+    module JobClass
+      NAME_PATTERN = /\A[A-Z]\w*(::[A-Z]\w*)*\z/
+
+      # Can never run here, so not retried.
+      class Unrunnable < LaravelCloud::Error; end
+
+      # Not defined here, e.g. mid-deploy. Retried, as another worker may have it.
+      class NotDefined < LaravelCloud::Error; end
+
+      def self.resolve(name, allowed: nil)
+        name = name.to_s
+
+        raise Unrunnable, "#{name.inspect} is not a valid job class name." unless NAME_PATTERN.match?(name)
+        raise Unrunnable, "#{name} is not in the allowed job classes." if allowed && !allowed.include?(name)
+
+        klass = begin
+          Object.const_get(name)
+        rescue NameError
+          raise NotDefined, "The job class #{name} is not defined."
+        end
+
+        raise Unrunnable, "#{name} is not an Active Job class." unless klass.is_a?(Class) && klass < ActiveJob::Base
+
+        klass
+      end
+    end
+  end
+end

data/lib/laravel_cloud/queue/managed_queue_config.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LaravelCloud
+  module Queue
+    # Parses LARAVEL_CLOUD_MANAGED_QUEUES_CONFIG, which Laravel Cloud injects:
+    #
+    #   {
+    #     "driver": "cloud",
+    #     "queue": "default",
+    #     "connection": {
+    #       "driver": "sqs",
+    #       "prefix": "https://sqs.us-east-2.amazonaws.com/036991040854",
+    #       "suffix": "-env-a21e326b-...",
+    #       "queue": "default",
+    #       "region": "us-east-2",
+    #       "credentials": "ecs"
+    #     },
+    #     "agent": {
+    #       "enabled": true,
+    #       "socket": "/tmp/cloud-agent.sock"
+    #     }
+    #   }
+    class ManagedQueueConfig
+      DEFAULT_AGENT_SOCKET = "/tmp/cloud-agent.sock"
+      FIFO_SUFFIX = ".fifo"
+
+      attr_reader :queue_url, :region, :credentials, :queue, :agent_socket, :prefix, :suffix
+
+      def self.from_env(env = ENV)
+        from_json(env["LARAVEL_CLOUD_MANAGED_QUEUES_CONFIG"], agent_socket: env["LARAVEL_CLOUD_AGENT_SOCKET"])
+      end
+
+      # Missing or malformed config gives an unconfigured result rather than an
+      # error, so the app still boots.
+      def self.from_json(json, agent_socket: nil)
+        data = parse(json)
+        connection = data["connection"].is_a?(Hash) ? data["connection"] : {}
+
+        agent = data["agent"].is_a?(Hash) ? data["agent"] : {}
+
+        # Agent socket: the config's, then LARAVEL_CLOUD_AGENT_SOCKET, then the default.
+        socket = presence(agent["socket"]) || presence(agent_socket) || DEFAULT_AGENT_SOCKET
+
+        new(
+          prefix: connection["prefix"].to_s.sub(%r{/+\z}, ""),
+          suffix: connection["suffix"].to_s,
+          queue: (connection["queue"] || data["queue"] || "default").to_s,
+          region: connection["region"]&.to_s,
+          credentials: (connection["credentials"] || "default").to_s,
+          agent_socket: socket,
+          agent_enabled: agent["enabled"] == true,
+        )
+      end
+
+      def self.parse(json)
+        return {} if json.nil? || json.empty?
+
+        decoded = JSON.parse(json)
+        decoded.is_a?(Hash) ? decoded : {}
+      rescue JSON::ParserError
+        {}
+      end
+      private_class_method :parse
+
+      def self.presence(value)
+        value.is_a?(String) && !value.empty? ? value : nil
+      end
+      private_class_method :presence
+
+      # ".fifo" is the only marker SQS and Laravel use for FIFO queues.
+      def self.fifo?(queue)
+        queue.to_s.end_with?(FIFO_SUFFIX)
+      end
+
+      def initialize(prefix:, suffix:, queue:, region:, credentials:, agent_socket:, agent_enabled:)
+        @prefix = prefix
+        @suffix = suffix
+        @queue = queue
+        @region = region
+        @credentials = credentials
+        @agent_socket = agent_socket
+        @agent_enabled = agent_enabled
+        @queue_url = queue_url_for(queue)
+      end
+
+      # The SQS URL for a queue name, as Laravel's SqsQueue::suffixQueue builds
+      # it: {prefix}/{queue}{suffix}, with the suffix before ".fifo" on FIFO
+      # queues ({prefix}/{base}{suffix}.fifo). Nil when unconfigured.
+      def queue_url_for(queue)
+        return nil if prefix.empty?
+
+        if self.class.fifo?(queue)
+          "#{prefix}/#{queue.delete_suffix(FIFO_SUFFIX)}#{suffix}#{FIFO_SUFFIX}"
+        else
+          "#{prefix}/#{queue}#{suffix}"
+        end
+      end
+
+      # The queue name for an SQS URL — the inverse of #queue_url_for, and the
+      # name the dashboard groups metrics by (Laravel's Cloud\Queue::normalizeQueue).
+      def normalize_queue(queue)
+        queue = queue.to_s
+        queue = queue.delete_prefix("#{prefix}/") unless prefix.empty?
+
+        return queue if suffix.empty?
+
+        if self.class.fifo?(queue)
+          base = queue.delete_suffix(FIFO_SUFFIX)
+          base.end_with?(suffix) ? "#{base.delete_suffix(suffix)}#{FIFO_SUFFIX}" : queue
+        else
+          queue.delete_suffix(suffix)
+        end
+      end
+
+      def configured?
+        !queue_url.nil?
+      end
+
+      # Whether to receive through the Cloud agent. Trusts the config's flag
+      # rather than probing for the socket, which may appear or vanish.
+      def agent_available?
+        @agent_enabled
+      end
+
+      def ecs_credentials?
+        credentials == "ecs"
+      end
+    end
+  end
+end

data/lib/laravel_cloud/queue/message.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "json"
+
+module LaravelCloud
+  module Queue
+    # The message body: Active Job's serialized job plus the "uuid" and
+    # "displayName" keys of Laravel's payload, which the dashboard names
+    # failed jobs by. Decoding only reads "data".
+    #
+    #   { "uuid": "<job_id>", "displayName": "ProcessReportJob", "data": { ...job.serialize... } }
+    module Message
+      class Undecodable < LaravelCloud::Error; end
+
+      def self.encode(job)
+        JSON.generate(
+          "uuid" => job.job_id,
+          "displayName" => job.class.name,
+          "data" => job.serialize,
+        )
+      end
+
+      # Decode a body into job data, with attempt counters taken from SQS.
+      #
+      # Retries release the original message (see Delivery), so its stored
+      # counters never change and `retry_on` would retry forever. Instead,
+      # delivery N has N-1 prior executions — per message, like Laravel's
+      # SqsJob::attempts(), so a freshly pushed message (a dashboard retry, a
+      # queue move) starts again at 1. `exception_executions` is cleared, which
+      # makes `retry_on` count all deliveries rather than per exception.
+      def self.decode(body, receive_count: 1)
+        payload = JSON.parse(body.to_s)
+        data = payload["data"] if payload.is_a?(Hash)
+
+        unless data.is_a?(Hash) && data["job_class"].is_a?(String)
+          raise Undecodable, "The managed queue message body is not in the expected format."
+        end
+
+        data.merge(
+          "executions" => [receive_count.to_i, 1].max - 1,
+          "exception_executions" => nil,
+        )
+      rescue JSON::ParserError
+        raise Undecodable, "The managed queue message body is not valid JSON."
+      end
+    end
+  end
+end

data/lib/laravel_cloud/queue/message_attributes.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+
+module LaravelCloud
+  module Queue
+    # Message group and deduplication IDs for FIFO and fair queues, declared on
+    # the job class (a value, or a block evaluated on the job):
+    #
+    #   class ProcessOrderJob < ApplicationJob
+    #     include LaravelCloud::Queue::MessageAttributes
+    #
+    #     queue_as "orders.fifo"
+    #
+    #     message_group_id { "customer-#{arguments.first.customer_id}" }
+    #     message_deduplication_id { "order-#{arguments.first.id}" }
+    #   end
+    #
+    # On a FIFO queue the group is the ordering scope; on a standard queue it's
+    # a fair-queues tenant key. Deduplication is FIFO-only.
+    module MessageAttributes
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :cloud_message_group_id_resolver, instance_writer: false
+        class_attribute :cloud_message_deduplication_id_resolver, instance_writer: false
+      end
+
+      class_methods do
+        def message_group_id(value = nil, &block)
+          self.cloud_message_group_id_resolver = block || value
+        end
+
+        def message_deduplication_id(value = nil, &block)
+          self.cloud_message_deduplication_id_resolver = block || value
+        end
+      end
+
+      def cloud_message_group_id
+        resolve_cloud_attribute(cloud_message_group_id_resolver)
+      end
+
+      def cloud_message_deduplication_id
+        resolve_cloud_attribute(cloud_message_deduplication_id_resolver)
+      end
+
+      private
+
+      def resolve_cloud_attribute(resolver)
+        value = resolver.is_a?(Proc) ? instance_exec(&resolver) : resolver
+        value&.to_s
+      end
+    end
+  end
+end

data/lib/laravel_cloud/queue/observer.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "json"
+require "securerandom"
+
+module LaravelCloud
+  module Queue
+    # Emits the dashboard's queue events, in the same format as Laravel's
+    # Cloud\Queue and Cloud\FailedJobProvider:
+    #
+    #   - "queued"     when a job is pushed onto a managed queue
+    #   - "started"    when a worker picks a job up
+    #   - "processed"  when the job succeeds
+    #   - "released"   when the job fails but will be retried
+    #   - "failed"     when the job fails terminally
+    #   - "failed_job" the payload + exception for a terminal failure
+    class Observer
+      # Laravel caps the one-line exception preview at 1001 characters.
+      MAX_EXCEPTION_PREVIEW_CHARS = 1001
+
+      def initialize(events)
+        @events = events
+      end
+
+      def queued(queue)
+        emit_queue_event("queued", queue, now)
+      end
+
+      def started(queue, at)
+        emit_queue_event("started", queue, at)
+      end
+
+      def processed(queue, started_at, finished_at)
+        emit_queue_event("processed", queue, finished_at, "duration_ms" => duration_ms(started_at, finished_at))
+      end
+
+      def released(queue, started_at, finished_at)
+        emit_queue_event("released", queue, finished_at, "duration_ms" => duration_ms(started_at, finished_at))
+      end
+
+      # Matches FailedJobProvider::log exactly — the same fields, and the
+      # failed job before the "failed" event. The dashboard needs every field.
+      def failed(delivery, error, started_at, finished_at)
+        @events.emit(
+          "_cloud_event" => "failed_job",
+          "id" => uuid7(finished_at),
+          "queue" => delivery.queue,
+          "started_at" => format_time(started_at),
+          "attempts" => delivery.receive_count,
+          # Whole and unchanged: Cloud pushes it back as-is to retry the job.
+          "payload" => delivery.body.to_s,
+          "exception_preview" => preview(error),
+          "job_name" => job_name(delivery.body),
+          "exception" => describe(error),
+        )
+
+        emit_queue_event("failed", delivery.queue, finished_at, "duration_ms" => duration_ms(started_at, finished_at))
+      end
+
+      def now
+        Time.now.utc
+      end
+
+      private
+
+      def emit_queue_event(type, queue, at, extra = {})
+        @events.emit({
+          "_cloud_event" => "queue",
+          "timestamp" => format_time(at),
+          "type" => type,
+          "queue" => queue,
+        }.merge(extra))
+      end
+
+      # A UUIDv7 for `time`, like Laravel's Str::uuid7($timestamp). It must be
+      # v7: Cloud derives the failure time from it (UUIDv7ToDateTime).
+      def uuid7(time)
+        milliseconds = (time.to_r * 1000).to_i
+        bytes = [milliseconds >> 16, milliseconds & 0xFFFF].pack("Nn") + SecureRandom.random_bytes(10)
+
+        bytes.setbyte(6, (bytes.getbyte(6) & 0x0F) | 0x70)
+        bytes.setbyte(8, (bytes.getbyte(8) & 0x3F) | 0x80)
+
+        hex = bytes.unpack1("H*")
+        [hex[0, 8], hex[8, 4], hex[12, 4], hex[16, 4], hex[20, 12]].join("-")
+      end
+
+      def format_time(time)
+        time.utc.strftime("%Y-%m-%d %H:%M:%S.%6N")
+      end
+
+      def duration_ms(started_at, finished_at)
+        [((finished_at - started_at) * 1000).round, 0].max
+      end
+
+      # "Class: message in file:line", as Laravel previews it.
+      def preview(error)
+        location = error.backtrace_locations&.first
+        where = location ? " in #{utf8(location.absolute_path || location.path)}:#{location.lineno}" : ""
+        message = utf8(error.message)
+
+        line = message.empty? ? "#{error.class}#{where}" : "#{error.class}: #{message}#{where}"
+        line[0, MAX_EXCEPTION_PREVIEW_CHARS]
+      end
+
+      def job_name(body)
+        payload = JSON.parse(body.to_s)
+        payload.is_a?(Hash) ? payload["displayName"].to_s : ""
+      rescue JSON::ParserError
+        ""
+      end
+
+      # "Class: message" then the backtrace, like PHP's (string) $exception.
+      def describe(error)
+        ["#{error.class}: #{utf8(error.message)}", *Array(error.backtrace).map { |line| utf8(line) }].join("\n")
+      end
+
+      # Exceptions can carry any bytes, and mixing encodings would raise here —
+      # after the message is deleted, losing the failed job. Invalid bytes
+      # become U+FFFD, like PHP's JSON_INVALID_UTF8_SUBSTITUTE.
+      def utf8(value)
+        value.to_s.dup.force_encoding(Encoding::UTF_8).scrub("\uFFFD")
+      end
+    end
+  end
+end

data/lib/laravel_cloud/queue/retry_attempts.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module LaravelCloud
+  module Queue
+    # Records the highest `retry_on` `attempts:` on a job class, so the
+    # worker's delivery cap never cuts a retry short: retries reuse the
+    # message, so each one is another delivery. Active Job doesn't expose this,
+    # so `retry_on` is wrapped; its arguments pass through untouched.
+    module RetryAttempts
+      DEFAULT_ATTEMPTS = 5 # Active Job's own default
+
+      def retry_on(*exceptions, attempts: DEFAULT_ATTEMPTS, **options, &block)
+        limit = attempts == :unlimited ? Float::INFINITY : attempts.to_i
+        self.cloud_retry_attempts = [cloud_retry_attempts, limit].max
+
+        super
+      end
+    end
+
+    ActiveSupport.on_load(:active_job) do
+      # 0 without `retry_on`; Infinity for `attempts: :unlimited`.
+      class_attribute :cloud_retry_attempts, instance_accessor: false, default: 0
+
+      ActiveJob::Exceptions::ClassMethods.prepend(RetryAttempts)
+    end
+  end
+end

data/lib/laravel_cloud/queue/settings.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module LaravelCloud
+  module Queue
+    # Optional settings, set from an initializer:
+    #
+    #   LaravelCloud::Queue.configure do |queue|
+    #     queue.allowed_job_classes = %w[ProcessReportJob SendInvoiceJob]
+    #     queue.on_failure = ->(error, job_data) { Rails.error.report(error) }
+    #   end
+    class Settings
+      # Job class names the worker may run (nil: any Active Job). Checked before
+      # constantizing, so other names never trigger autoloading.
+      attr_reader :allowed_job_classes
+
+      # Called with (error, job_data) when a job fails for good. Errors it
+      # raises are logged and ignored.
+      attr_accessor :on_failure
+
+      def initialize
+        @allowed_job_classes = nil
+        @on_failure = nil
+      end
+
+      def allowed_job_classes=(classes)
+        @allowed_job_classes = classes.nil? ? nil : Array(classes).map(&:to_s)
+      end
+    end
+  end
+end