pgbus 0.21.4 → 0.23.0

data/lib/pgbus/active_job/quiet_logs.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Pgbus
+  module ActiveJob
+    # config.quiet_active_job_logs (issue #618): a worker process stops
+    # ActiveJob's per-job "Performing …" / "Performed …" info lines by
+    # unsubscribing ActiveJob::LogSubscriber from perform_start.active_job and
+    # perform.active_job. The perform listener also writes ActiveJob's
+    # "Error performing …" error line, for a job that raised (with its
+    # backtrace) and for one a before_perform callback halted, so those go too.
+    # An exception that escapes ActiveJob's handlers is still logged by
+    # ErrorReporter (class and message at error) and recorded in
+    # pgbus_failed_events; one retry_on catches still logs ActiveJob's
+    # "Retrying …" line (and "Stopped retrying …" at error when it gives up);
+    # a halted job logs nothing.
+    # The other ActiveJob events (Enqueued, Retrying, Stopped retrying,
+    # Discarded) still log as the host configured them, and the web process is
+    # untouched. Swapping
+    # ActiveJob::Base.logger instead would reach the web process too, and by
+    # default that logger is Rails.logger.
+    module QuietLogs
+      EVENTS = %w[perform_start.active_job perform.active_job].freeze
+
+      # Built at load; the applied notifiers are read and written under it.
+      @mutex = Mutex.new
+      @applied = {}.compare_by_identity
+
+      class << self
+        # Unsubscribes the LogSubscriber listeners once per notifier. Returns
+        # true when it did, false when that notifier was already done.
+        def apply!(notifier: ::ActiveSupport::Notifications.notifier)
+          @mutex.synchronize do
+            return false if @applied.key?(notifier)
+
+            @applied[notifier] = true
+            # all_listeners_for, not listeners_for: the latter skips a
+            # silenced subscriber (its logger above info), which would come
+            # back the moment the level drops.
+            EVENTS.each do |name|
+              notifier.all_listeners_for(name).each do |listener|
+                notifier.unsubscribe(listener) if log_subscriber?(listener)
+              end
+            end
+            true
+          end
+        end
+
+        private
+
+        # Fanout::Subscribers::Evented exposes #delegate on Rails 7.1 and later.
+        def log_subscriber?(listener)
+          listener.respond_to?(:delegate) && listener.delegate.is_a?(::ActiveJob::LogSubscriber)
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/autovacuum_tuning.rb

@@ -66,8 +66,8 @@ module Pgbus
             q RECORD;
           BEGIN
             FOR q IN SELECT queue_name FROM pgmq.meta LOOP
-              EXECUTE format('ALTER TABLE pgmq.q_%I SET (#{settings_clause(QUEUE_SETTINGS)})', q.queue_name);
-              EXECUTE format('ALTER TABLE pgmq.a_%I SET (#{settings_clause(ARCHIVE_SETTINGS)})', q.queue_name);
+              EXECUTE format('ALTER TABLE pgmq.%I SET (#{settings_clause(QUEUE_SETTINGS)})', 'q_' || lower(q.queue_name));
+              EXECUTE format('ALTER TABLE pgmq.%I SET (#{settings_clause(ARCHIVE_SETTINGS)})', 'a_' || lower(q.queue_name));
             END LOOP;
           END $$;
         SQL

data/lib/pgbus/batch/sweep.rb

@@ -47,6 +47,8 @@ module Pgbus
 
           dlq = client.dead_letter_physical_name(row.queue_name, prefixed: false)
           return :failed if client.message_with_job_id?(dlq, job_id: row.job_id, prefixed: false) == true
+          # completed_jobs :delete (issue #607) leaves no archive row to find.
+          return :completed if Pgbus.configuration.completed_jobs == :delete
 
           return :completed if client.message_archived?(row.queue_name, msg_id: row.msg_id, prefixed: false) == true
 

data/lib/pgbus/batch.rb

@@ -425,8 +425,11 @@ module Pgbus
     end
 
     def self.backfill_execution(payload, msg_id, queue_name)
-      return unless executions_migrated?
       return unless payload && msg_id
+      # Only batch members have an execution row: without this, every plain
+      # enqueue paid a no-op UPDATE round trip after the send (issue #603).
+      return unless payload[METADATA_KEY]
+      return unless executions_migrated?
 
       job_id = payload["job_id"]
       return unless job_id

data/lib/pgbus/cli/dlq.rb

@@ -70,7 +70,8 @@ module Pgbus
         if data_source.retry_dlq_message(detail[:queue_name], detail[:msg_id])
           puts "Re-enqueued msg_id #{detail[:msg_id]} to #{origin_queue(detail[:queue_name])}."
         else
-          warn "Failed to retry msg_id #{detail[:msg_id]}."
+          warn "Failed to retry msg_id #{detail[:msg_id]}: it is gone, the retry failed, " \
+               "another copy of the job holds its uniqueness key, or its concurrency limit is full."
           exit 1
         end
       end

data/lib/pgbus/client/async_commit.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Client
+    # Asynchronous completion commit (issue #619,
+    # `config.completion_synchronous_commit = false`). A completion statement
+    # (archive, delete and their batches) runs in its own transaction with
+    # synchronous_commit off, so the job is acknowledged without waiting for
+    # its WAL flush. The four queries go out in one libpq pipeline with one
+    # sync, so it is still one round trip (pg writes each query to the socket
+    # as it is sent, but waits for a reply only once, after the sync):
+    #
+    #   BEGIN
+    #   SET LOCAL synchronous_commit TO off
+    #   <the prepared completion statement>   <- the result the caller gets
+    #   COMMIT
+    #
+    # SET LOCAL ends with the transaction, so the pooled connection never
+    # keeps the setting, and the statement stays prepared (no planning cost
+    # back, issue #605). Only a job's completion asks for it (the executor and
+    # the completer pass job_completion: true; the event consumer and the
+    # dashboard keep the durable commit), and NativeStatements#exec_native
+    # calls it only outside a caller's transaction: a completion inside one is
+    # the caller's to commit.
+    #
+    # A failed query aborts the rest of the pipeline and leaves the explicit
+    # BEGIN open in a failed state: the connection leaves pipeline mode, the
+    # transaction is rolled back, and the error is raised for run_native's
+    # re-run and short-row checks. A statement not yet prepared on this session
+    # runs through prepare_and_exec outside the pipeline, committed
+    # synchronously, once; the next call pipelines again.
+    module AsyncCommit
+      ASYNC_COMMIT_BEGIN = "BEGIN"
+      ASYNC_COMMIT_OFF = "SET LOCAL synchronous_commit TO off"
+      ASYNC_COMMIT_COMMIT = "COMMIT"
+      NO_PARAMS = [].freeze
+
+      private
+
+      def exec_async_commit(conn, statement, params, prepared)
+        results = pipeline_async_commit(conn, statement, params, prepared)
+        failed = results.find { |result| result.result_status == PG::PGRES_FATAL_ERROR }
+        unless failed
+          NativeStatements.note_executed if prepared
+          return results[2]
+        end
+
+        rollback_async_commit(conn)
+        begin
+          failed.check
+        rescue PG::InvalidSqlStatementName
+          raise unless prepared
+
+          prepare_and_exec(conn, statement, params)
+        end
+      end
+
+      # Sends the four queries and one sync, and reads every result up to the
+      # sync: each query answers its result and then nil.
+      def pipeline_async_commit(conn, statement, params, prepared)
+        conn.enter_pipeline_mode
+        begin
+          conn.send_query_params(ASYNC_COMMIT_BEGIN, NO_PARAMS)
+          conn.send_query_params(ASYNC_COMMIT_OFF, NO_PARAMS)
+          if prepared
+            conn.send_query_prepared(statement.name, params)
+          else
+            conn.send_query_params(statement.sql, params)
+          end
+          conn.send_query_params(ASYNC_COMMIT_COMMIT, NO_PARAMS)
+          conn.pipeline_sync
+          read_pipeline(conn)
+        ensure
+          leave_pipeline(conn)
+        end
+      end
+
+      def read_pipeline(conn)
+        results = []
+        ended = false
+        loop do
+          result = conn.get_result
+          if result.nil?
+            # One nil closes each query's results; two in a row means libpq
+            # has nothing more, so the sync will not come. That session's
+            # state cannot be trusted: pgmq-ruby re-runs on a fresh one.
+            if ended
+              discard_session(conn, "A completion pipeline ended before its sync")
+              raise PG::ConnectionBad, "pipeline ended before its sync"
+            end
+
+            ended = true
+            next
+          end
+          ended = false
+          break if result.result_status == PG::PGRES_PIPELINE_SYNC
+
+          results << result
+        end
+        results
+      end
+
+      # A connection back in the pool in pipeline mode would break the next
+      # caller. A lost one is left as it is: pgmq-ruby resets a bad connection
+      # at its next checkout, and a reset leaves pipeline mode. One that cannot
+      # leave (results left unread by an interrupted read) is reset here.
+      def leave_pipeline(conn)
+        return if conn.pipeline_status == PG::PQ_PIPELINE_OFF || conn.status == PG::CONNECTION_BAD
+
+        conn.exit_pipeline_mode
+      rescue PG::Error => e
+        discard_session(conn, "Could not leave pipeline mode after a completion (#{e.message})")
+      end
+
+      # Resets a session whose pipeline state cannot be trusted. A reset that
+      # fails (the server is gone) is logged and left there, so the error the
+      # caller is already raising is the one that surfaces; pgmq-ruby resets a
+      # bad connection at its next checkout.
+      # A reset reconnects with the same conninfo (the read bounds ride in it)
+      # but does not re-run the connect lambda, so the :session-mode SETs are
+      # applied again here.
+      def discard_session(conn, reason)
+        Pgbus.logger.warn { "[Pgbus::Client] #{reason}, resetting the connection" }
+        begin
+          conn.reset
+        rescue PG::Error => e
+          Pgbus.logger.warn { "[Pgbus::Client] Resetting the connection failed: #{e.message}" }
+          return
+        end
+        reapply_session_gucs(conn)
+      end
+
+      def reapply_session_gucs(conn)
+        apply_session_gucs(conn, @session_gucs) if @session_gucs
+      rescue PG::Error => e
+        Pgbus.logger.warn do
+          "[Pgbus::Client] Re-applying the session's database.yml variables after the reset failed: #{e.message}"
+        end
+      end
+
+      # The failed BEGIN block (INERROR) is still open after the sync. A failed
+      # COMMIT has already ended it (IDLE); a lost connection reports UNKNOWN
+      # and has nothing to roll back.
+      def rollback_async_commit(conn)
+        conn.exec("ROLLBACK") if [PG::PQTRANS_INERROR, PG::PQTRANS_INTRANS].include?(conn.transaction_status)
+      end
+    end
+  end
+end