pgbus 0.19.2 → 0.19.4

data/lib/pgbus/batch/builder.rb

@@ -59,6 +59,7 @@ module Pgbus
         raise ArgumentError, UNTAGGED_MESSAGE unless Thread.current[:pgbus_batch_id] == @batch.batch_id
 
         jobs = self.class.job_instances(jobs)
+        Pgbus::ActiveJob::BatchId.capture(jobs, @batch.batch_id)
         ::ActiveJob.perform_all_later(jobs) if jobs.any?
         yield if block_given?
         jobs

data/lib/pgbus/batch.rb

@@ -45,6 +45,19 @@ module Pgbus
     # Thread.current): b.add raises anywhere else, and a plain perform_later
     # from a thread or fiber the block spawns does not join the batch.
     #
+    # Inside a database transaction the batch stays pending until the
+    # transaction commits: a job with enqueue_after_transaction_commit keeps
+    # the batch on its instance (b.add, batch.add and perform_later put it
+    # there) and counts itself in at commit, before the batch starts. On a
+    # rollback no job is sent and, when pgbus shares the application's
+    # database, the batch row is gone with the rest (a separate pgbus
+    # database commits the row on its own connection). A bare
+    # ActiveJob.perform_all_later in the block does not keep it: use b.add.
+    # Add jobs that defer: one sent at once, on a connection other than the
+    # caller's, can run before the batch commits and is then accounted for
+    # only by the stalled-execution sweep. A job instance is one member (one
+    # job_id): enqueue it once.
+    #
     #   Pgbus::Batch.start(description: "Import") do |b|
     #     b.on_complete ImportDoneJob
     #     b.add(*rows.map { |row| ImportRowJob.new(row.id) })
@@ -124,7 +137,7 @@ module Pgbus
       end
       @started = true
       count_jobs(&)
-      start_processing
+      self.class.send(:after_transactions_commit) { start_processing }
       self
     end
 
@@ -139,6 +152,7 @@ module Pgbus
       return jobs if jobs.empty? && block.nil?
 
       enqueue do
+        Pgbus::ActiveJob::BatchId.capture(jobs, batch_id)
         ::ActiveJob.perform_all_later(jobs) if jobs.any?
         yield if block
       end
@@ -473,6 +487,22 @@ module Pgbus
         finish_if_needed(try_finish!(batch_id))
       end
 
+      # Run the end-of-block start (or an open batch's re-check) once every
+      # open transaction has committed, and at once when none is open
+      # (issue #553). Jobs with enqueue_after_transaction_commit register
+      # their enqueue as an after_commit callback during the block; Active
+      # Record runs a transaction's callbacks in registration order and
+      # appends a savepoint's to its parent's on release, so every one of
+      # them has counted itself into the still-pending row before this runs.
+      # On a rollback none of them runs, and the row (created in the same
+      # transaction) is gone. Active Record 7.1 has no
+      # after_all_transactions_commit, and Active Job 7.1 never defers.
+      def after_transactions_commit(&)
+        return yield unless ::ActiveRecord.respond_to?(:after_all_transactions_commit)
+
+        ::ActiveRecord.after_all_transactions_commit(&)
+      end
+
       def finish_if_needed(result)
         return result unless result&.fetch(:just_finished, false) && result[:record]
 
@@ -629,8 +659,10 @@ module Pgbus
       raise AlreadyFinished, "Can't add jobs into an already finished batch" if finished?
 
       count_jobs(&)
-      reload
-      self.class.send(:finish_if_needed, BatchEntry.check_finished!(batch_id))
+      self.class.send(:after_transactions_commit) do
+        reload
+        self.class.send(:finish_if_needed, BatchEntry.check_finished!(batch_id))
+      end
       self
     end
 

data/lib/pgbus/configuration/snapshot.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Configuration
+    # The settings a non-main Ractor reads, as one Ractor-shareable value
+    # (issue #527). The configuration itself stays on the main Ractor: it holds
+    # the logger and the app's callables. Pgbus.finalize! and Pgbus.configure
+    # publish a snapshot of it as Pgbus.configuration_snapshot, which any
+    # Ractor can read.
+    #
+    # Only what pgbus's Ractor-safe code reads is here: the queue_prefix, for
+    # the stream-key budget (Streams::Key). A setting joins when such code
+    # needs it.
+    Snapshot = Data.define(:queue_prefix) do
+      # A shareable snapshot of +configuration+. The configuration's own
+      # values are copied, never frozen in place. Nil when a setting is not
+      # plain data (Shareable.data?), which only a configuration that skipped
+      # validation can hold: the caller keeps the snapshot it has.
+      def self.of(configuration)
+        prefix = configuration.queue_prefix
+        return unless Shareable.data?(prefix)
+
+        new(queue_prefix: Shareable.freeze_copy(prefix))
+      end
+    end
+  end
+end

data/lib/pgbus/integrations/appsignal/event_span.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "pgbus/integrations/appsignal/span_subscriber"
+
+module Pgbus
+  module Integrations
+    module Appsignal
+      # The `<Handler>#handle` transaction around pgbus.event_processed, and
+      # the fallback for an error pgbus.event_failed reports from outside it
+      # (an event that could not be built never reaches the span).
+      class EventSpan < SpanSubscriber
+        private
+
+        def frame_key = :pgbus_appsignal_event_span
+
+        def attached_key = :pgbus_appsignal_event_span_error
+
+        def describe(transaction, payload)
+          transaction.set_action_if_nil("#{payload[:handler] || "UnknownHandler"}#handle")
+          apply_queue_start(transaction, payload[:published_at])
+          transaction.add_tags(handler_tags(payload))
+        end
+
+        # The processed count is taken at finish whether or not the handler
+        # raised, as Metrics::Subscriber does: a raised handler counts as
+        # processed and, through pgbus.event_failed, as failed.
+        def finished(payload, duration)
+          tags = { handler: payload[:handler], routing_key: payload[:routing_key] }
+          ::Appsignal.add_distribution_value("pgbus_event_duration_ms", duration, tags)
+          ::Appsignal.increment_counter("pgbus_event_count", 1, tags.merge(status: "processed"))
+        end
+
+        def handler_tags(payload)
+          {
+            "handler" => payload[:handler],
+            "routing_key" => payload[:routing_key],
+            "attempts" => payload[:read_ct]
+          }.compact
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/integrations/appsignal/job_span.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "pgbus/integrations/appsignal/span_subscriber"
+
+module Pgbus
+  module Integrations
+    module Appsignal
+      # The `<JobClass>#perform` transaction around pgbus.executor.execute
+      # (deserialize + perform + archive), and the fallback for an error
+      # pgbus.job_failed reports from outside it.
+      class JobSpan < SpanSubscriber
+        private
+
+        def frame_key = :pgbus_appsignal_job_span
+
+        def attached_key = :pgbus_appsignal_job_span_error
+
+        def describe(transaction, payload)
+          transaction.set_action_if_nil("#{payload[:job_class] || "UnknownJob"}#perform")
+          apply_queue_start(transaction, payload[:enqueued_at])
+          transaction.add_tags(job_tags(payload))
+          transaction.add_params_if_nil { { arguments: payload[:arguments] } } if payload.key?(:arguments)
+        end
+
+        def finished(payload, duration)
+          ::Appsignal.add_distribution_value(
+            "pgbus_job_duration_ms",
+            duration,
+            { queue: payload[:queue], job_class: payload[:job_class] }
+          )
+        end
+
+        def job_tags(payload)
+          tags = {
+            "queue" => payload[:queue],
+            "job_class" => payload[:job_class],
+            "attempts" => payload[:read_ct]
+          }
+          tags["active_job_id"] = payload[:job_id] if payload[:job_id]
+          tags["provider_job_id"] = payload[:provider_job_id] if payload[:provider_job_id]
+          tags["request_id"] = payload[:provider_job_id] || payload[:job_id]
+          tags.compact
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/integrations/appsignal/span_subscriber.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Pgbus
+  module Integrations
+    module Appsignal
+      # The AppSignal transaction of one unit of work: a job (JobSpan, on
+      # pgbus.executor.execute) or an event handler (EventSpan, on
+      # pgbus.event_processed).
+      #
+      # An ActiveSupport::Notifications listener that responds to #start and
+      # #finish is called on both sides of the instrumented block (the Evented
+      # subscriber). A finish-only block subscriber runs after the block has
+      # returned, so the transaction it opened was never current while the job
+      # ran: job code had nothing to attach to, and pgbus.job_failed, fired
+      # later, found no transaction to put its error on (issue #552). Here
+      # #start opens the transaction before the block runs, so job and handler
+      # code, and the Rails error reporter, find it current; #finish records
+      # the exception the block raised (the instrumenter puts it in
+      # payload[:exception_object]) and completes the transaction.
+      #
+      # Like AppSignal's own ActiveJob hook, a transaction that is already
+      # current is reused and left to its owner to complete.
+      #
+      # The frame lives in Thread.current[], which is fiber-local, so start and
+      # finish pair up per job under execution_mode: :async, the same way
+      # AppSignal keeps its current transaction. A span opened inside another
+      # of its kind (a handler calling another handler's #process) keeps the
+      # enclosing frame and restores it at finish, so the outer span still
+      # completes its transaction. An instance holds no state of its own.
+      # Nothing an AppSignal call raises reaches the job: it is logged at warn
+      # level.
+      #
+      # Subclasses define #frame_key and #attached_key (the fiber-local slots),
+      # #describe(transaction, payload) (action, queue start, tags) and
+      # #finished(payload, duration_ms) (the span's metrics).
+      class SpanSubscriber
+        BACKGROUND_JOB = "background_job"
+
+        # owned: this span created the transaction, so it completes it.
+        # previous: the enclosing span's frame, restored at finish.
+        Frame = Struct.new(:started_at, :owned, :previous)
+
+        def start(_name, _id, payload)
+          frame = Thread.current[frame_key] = Frame.new(monotonic_now, false, Thread.current[frame_key])
+          guard do
+            frame.owned = !::Appsignal::Transaction.current?
+            transaction = frame.owned ? ::Appsignal::Transaction.create(BACKGROUND_JOB) : ::Appsignal::Transaction.current
+            describe(transaction, payload)
+          end
+        end
+
+        def finish(_name, _id, payload)
+          frame = Thread.current[frame_key]
+          guard do
+            Thread.current[attached_key] = attach(payload[:exception_object])
+            finished(payload, monotonic_now - frame.started_at)
+          end
+        ensure
+          guard do
+            Thread.current[frame_key] = frame.previous
+            ::Appsignal::Transaction.complete_current! if frame.owned
+          end
+        end
+
+        # pgbus.job_failed and pgbus.event_failed fire after #finish. An error
+        # the span recorded is already on its transaction. One raised before
+        # the span opened (a message body that is not JSON, an event that
+        # cannot be built, a dead-letter move that failed) had no transaction,
+        # so it is reported on one of its own, named and tagged like the span's
+        # (or added to a transaction that is current, which is left as it is).
+        def report_unattached(error, payload)
+          attached = Thread.current[attached_key]
+          Thread.current[attached_key] = nil
+          return if error.nil? || error.equal?(attached)
+          return ::Appsignal::Transaction.current.set_error(error) if ::Appsignal::Transaction.current?
+
+          ::Appsignal.report_error(error) do |transaction|
+            transaction.set_namespace(BACKGROUND_JOB)
+            describe(transaction, payload)
+          end
+        end
+
+        private
+
+        # Returns the exception it recorded, nil when there was none.
+        def attach(error)
+          return unless error && ::Appsignal::Transaction.current?
+
+          ::Appsignal::Transaction.current.set_error(error)
+          error
+        end
+
+        # AppSignal expects queue-start as Unix epoch milliseconds. Pgbus
+        # carries it as either an ISO-8601 String or a Time — both happen
+        # in practice (executor passes the JSON string, handler passes a
+        # parsed Time).
+        def apply_queue_start(transaction, value)
+          return unless value
+
+          millis =
+            case value
+            when Time
+              (value.to_f * 1_000).to_i
+            when String
+              (Time.parse(value).to_f * 1_000).to_i
+            when Numeric
+              value.to_i
+            end
+          transaction.set_queue_start(millis) if millis
+        rescue ArgumentError
+          # Unparseable timestamp — skip rather than blow up.
+        end
+
+        def guard
+          yield
+        rescue StandardError => e
+          Pgbus.logger.warn { "[Pgbus::AppSignal] subscriber error: #{e.class}: #{e.message}" }
+        end
+
+        def monotonic_now
+          ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :float_millisecond)
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/integrations/appsignal/subscriber.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
-require "time"
+require "pgbus/integrations/appsignal/job_span"
+require "pgbus/integrations/appsignal/event_span"
 
 module Pgbus
   module Integrations
@@ -10,16 +11,23 @@ module Pgbus
       #
       # Job and event-handler events open a BACKGROUND_JOB transaction so they
       # appear under AppSignal's "Performance > Background jobs" view, with
-      # action `<JobClass>#perform` or `<HandlerClass>#handle`. All other
-      # events are reported as counters or distributions only.
+      # action `<JobClass>#perform` or `<HandlerClass>#handle`. The transaction
+      # is opened when pgbus.executor.execute / pgbus.event_processed starts and
+      # completed when it finishes (JobSpan, EventSpan), so it is current while the
+      # job or handler runs and carries the error the execution raised. All
+      # other events are reported as counters or distributions only.
       #
       # All metric names are prefixed `pgbus_`. Tag keys avoid high-cardinality
       # values (no msg_id, no event_id) so AppSignal's metric storage stays
       # efficient.
       module Subscriber
-        BACKGROUND_JOB = "background_job"
         METRIC_PREFIX = "pgbus_"
-        private_constant :BACKGROUND_JOB, :METRIC_PREFIX
+
+        # Built once, at load: a span keeps its state in a fiber-local frame,
+        # never in itself.
+        JOB_SPAN = JobSpan.new.freeze
+        EVENT_SPAN = EventSpan.new.freeze
+        private_constant :METRIC_PREFIX, :JOB_SPAN, :EVENT_SPAN
 
         # Tracked so we can detach in reset! (used by specs). Installed once
         # at boot (the engine's after_initialize); @install_lock makes the
@@ -54,11 +62,11 @@ module Pgbus
 
           def subscribe_all
             [
-              subscribe("pgbus.executor.execute") { |event| on_executor_execute(event) },
+              ActiveSupport::Notifications.subscribe("pgbus.executor.execute", JOB_SPAN),
               subscribe("pgbus.job_completed") { |event| on_job_completed(event) },
               subscribe("pgbus.job_failed") { |event| on_job_failed(event) },
               subscribe("pgbus.job_dead_lettered") { |event| on_job_dead_lettered(event) },
-              subscribe("pgbus.event_processed") { |event| on_event_processed(event) },
+              ActiveSupport::Notifications.subscribe("pgbus.event_processed", EVENT_SPAN),
               subscribe("pgbus.event_failed") { |event| on_event_failed(event) },
               subscribe("pgbus.event_dead_lettered") { |event| on_event_dead_lettered(event) },
               subscribe("pgbus.client.send_message") { |event| on_send_message(event) },
@@ -98,22 +106,6 @@ module Pgbus
 
           # ── Job execution ───────────────────────────────────────────────
 
-          def on_executor_execute(event)
-            payload = event.payload
-            transaction = ::Appsignal::Transaction.create(BACKGROUND_JOB)
-            transaction.set_action_if_nil("#{payload[:job_class] || "UnknownJob"}#perform")
-            apply_queue_start(transaction, payload[:enqueued_at])
-            transaction.add_tags(job_tags(payload))
-            transaction.add_params_if_nil { { arguments: payload[:arguments] } }
-            ::Appsignal.add_distribution_value(
-              "#{METRIC_PREFIX}job_duration_ms",
-              event.duration,
-              { queue: payload[:queue], job_class: payload[:job_class] }
-            )
-          ensure
-            ::Appsignal::Transaction.complete_current!
-          end
-
           def on_job_completed(event)
             payload = event.payload
             ::Appsignal.increment_counter(
@@ -123,15 +115,18 @@ module Pgbus
             )
           end
 
+          # The counter has its own guard so a raise there cannot keep the
+          # error from being reported.
           def on_job_failed(event)
             payload = event.payload
-            ::Appsignal.increment_counter(
-              "#{METRIC_PREFIX}queue_job_count",
-              1,
-              { queue: payload[:queue], job_class: payload[:job_class], status: "failed" }
-            )
-            err = payload[:exception_object]
-            ::Appsignal.set_error(err) if err && ::Appsignal.respond_to?(:set_error)
+            safely do
+              ::Appsignal.increment_counter(
+                "#{METRIC_PREFIX}queue_job_count",
+                1,
+                { queue: payload[:queue], job_class: payload[:job_class], status: "failed" }
+              )
+            end
+            JOB_SPAN.report_unattached(payload[:exception_object], payload)
           end
 
           def on_job_dead_lettered(event)
@@ -145,40 +140,23 @@ module Pgbus
 
           # ── Event handler ───────────────────────────────────────────────
 
-          def on_event_processed(event)
-            payload = event.payload
-            transaction = ::Appsignal::Transaction.create(BACKGROUND_JOB)
-            transaction.set_action_if_nil("#{payload[:handler] || "UnknownHandler"}#handle")
-            apply_queue_start(transaction, payload[:published_at])
-            transaction.add_tags(handler_tags(payload))
-            ::Appsignal.add_distribution_value(
-              "#{METRIC_PREFIX}event_duration_ms",
-              event.duration,
-              { handler: payload[:handler], routing_key: payload[:routing_key] }
-            )
-            ::Appsignal.increment_counter(
-              "#{METRIC_PREFIX}event_count",
-              1,
-              { handler: payload[:handler], routing_key: payload[:routing_key], status: "processed" }
-            )
-          ensure
-            ::Appsignal::Transaction.complete_current!
-          end
-
+          # The counter has its own guard so a raise there cannot keep the
+          # error from being reported.
           def on_event_failed(event)
             payload = event.payload
-            ::Appsignal.increment_counter(
-              "#{METRIC_PREFIX}event_count",
-              1,
-              { handler: payload[:handler], routing_key: payload[:routing_key], status: "failed" }
-            )
-            err = payload[:exception_object]
-            ::Appsignal.set_error(err) if err && ::Appsignal.respond_to?(:set_error)
+            safely do
+              ::Appsignal.increment_counter(
+                "#{METRIC_PREFIX}event_count",
+                1,
+                { handler: payload[:handler], routing_key: payload[:routing_key], status: "failed" }
+              )
+            end
+            EVENT_SPAN.report_unattached(payload[:exception_object], payload)
           end
 
           # A counter only, like on_job_dead_lettered: nothing is raised at
-          # move time, and the handler's error already went out through
-          # pgbus.event_failed on the attempt that raised (issue #543). A body
+          # move time, and the handler's error already went out on the
+          # attempt that raised (issues #543, #552). A body
           # that is not JSON leaves handler and routing_key nil; compact drops
           # those tags, as Metrics::Subscriber does.
           def on_event_dead_lettered(event)
@@ -295,49 +273,6 @@ module Pgbus
               { reason: payload[:reason], kind: "consumer" }
             )
           end
-
-          # ── Helpers ─────────────────────────────────────────────────────
-
-          # AppSignal expects queue-start as Unix epoch milliseconds. Pgbus
-          # carries it as either an ISO-8601 String or a Time — both happen
-          # in practice (executor passes the JSON string, handler passes a
-          # parsed Time).
-          def apply_queue_start(transaction, value)
-            return unless value
-
-            millis =
-              case value
-              when Time
-                (value.to_f * 1_000).to_i
-              when String
-                (Time.parse(value).to_f * 1_000).to_i
-              when Numeric
-                value.to_i
-              end
-            transaction.set_queue_start(millis) if millis
-          rescue ArgumentError
-            # Unparseable timestamp — skip rather than blow up.
-          end
-
-          def job_tags(payload)
-            tags = {
-              "queue" => payload[:queue],
-              "job_class" => payload[:job_class],
-              "attempts" => payload[:read_ct]
-            }
-            tags["active_job_id"] = payload[:job_id] if payload[:job_id]
-            tags["provider_job_id"] = payload[:provider_job_id] if payload[:provider_job_id]
-            tags["request_id"] = payload[:provider_job_id] || payload[:job_id]
-            tags.compact
-          end
-
-          def handler_tags(payload)
-            {
-              "handler" => payload[:handler],
-              "routing_key" => payload[:routing_key],
-              "attempts" => payload[:read_ct]
-            }.compact
-          end
         end
       end
     end

data/lib/pgbus/shareable.rb

@@ -2,7 +2,8 @@
 
 module Pgbus
   # Deep-frozen, Ractor-shareable copies of plain data, for the settings
-  # Configuration#finalize! freezes at boot (issue #527).
+  # Configuration#finalize! freezes at boot and the snapshot non-main Ractors
+  # read (issue #527).
   module Shareable
     # Leaves that are plain data. Modules and classes count too: they are
     # shareable by nature.
@@ -39,6 +40,15 @@ module Pgbus
       end
     end
 
+    # True on the main Ractor, the only one that can read Pgbus.configuration,
+    # and wherever the engine has no Ractors. Ractor.main? is Ruby 3.4+.
+    def main_ractor?
+      return true unless defined?(::Ractor)
+      return ::Ractor.main? if ::Ractor.respond_to?(:main?)
+
+      ::Ractor.current.equal?(::Ractor.main)
+    end
+
     # A deep-frozen copy of plain data (see data?), Ractor-shareable where the
     # engine has Ractors. The caller's own objects are copied, never frozen in
     # place.

data/lib/pgbus/streams/key.rb

@@ -105,7 +105,7 @@ module Pgbus
         raise ArgumentError,
               "stream_key #{key.inspect} is #{key.length} chars, " \
               "exceeds pgbus budget of #{budget} " \
-              "(queue_prefix=#{Pgbus.configuration.queue_prefix.inspect}, " \
+              "(queue_prefix=#{queue_prefix.inspect}, " \
               "pgbus_max_queue_name_length=#{QueueNameValidator::MAX_QUEUE_NAME_LENGTH}). " \
               "Shorten the streamables or use Pgbus::Streams::Streamable on the model."
       end
@@ -182,10 +182,19 @@ module Pgbus
       # override `config.queue_prefix` get the correct budget
       # automatically.
       def queue_name_budget
-        QueueNameValidator::MAX_QUEUE_NAME_LENGTH -
-          Pgbus.configuration.queue_prefix.length - 1
+        QueueNameValidator::MAX_QUEUE_NAME_LENGTH - queue_prefix.length - 1
       end
 
+      # The live configuration's queue_prefix on the main Ractor. A non-main
+      # Ractor cannot read Pgbus.configuration (it holds the logger and the
+      # app's callables), so there it is the published snapshot's: the
+      # prefix as the last Pgbus.finalize! or Pgbus.configure left it.
+      def queue_prefix
+        config = Shareable.main_ractor? ? Pgbus.configuration : Pgbus.configuration_snapshot
+        config.queue_prefix
+      end
+      private_class_method :queue_prefix
+
       # Raises if a normalized fragment contains the `:` separator.
       # Kept private-ish via module_function so the guard is shared
       # between stream_key and any future composer without becoming

data/lib/pgbus/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Pgbus
-  VERSION = "0.19.2"
+  VERSION = "0.19.4"
 end