pgbus 0.19.0 → 0.19.2

data/config/locales/pt.yml

@@ -236,6 +236,102 @@ pt:
         payload: Carga útil
         retry: Tentar novamente
         title: 'Mensagem DLQ #%{id}'
+    deploy_tasks:
+      banner:
+        heading: Transição
+        lagging_html:
+          one: 'Aguardando a transição: 1 processo ainda não executa <code>%{revision}</code>.'
+          other: 'Aguardando a transição: %{count} processos ainda não executam <code>%{revision}</code>.'
+        lagging_more: "…e mais %{count}"
+        lagging_process: "%{kind}@%{hostname} pid %{pid} em %{revision}"
+        live_count: "%{count} em %{revision}"
+        live_heading: 'Processos ativos:'
+        no_revision_hint_html: Defina <code>config.app_revision</code>, ou uma de <code>PGBUS_REVISION</code>, <code>DASH_VERSION</code> ou <code>KAMAL_VERSION</code>, ou inclua um arquivo <code>REVISION</code>. Executar agora continua funcionando.
+        no_revision_html: 'Revisão da aplicação não configurada: as tarefas de deploy aguardam até que esteja.'
+        no_revision_label: sem revisão
+        no_scheduler_html: 'Nenhum agendador em execução: as tarefas de deploy só são disparadas pelo agendador. Este veredito usa a revisão do próprio painel.'
+        no_workers_html: 'Aguardando a transição: ainda nenhum worker em <code>%{revision}</code>.'
+        ready_html: 'Transição concluída: cada processo ativo executa <code>%{revision}</code>. Tarefas armadas executam no próximo tick do agendador.'
+        unavailable_html: Não foi possível ler o estado da transição.
+      default_queue: default
+      disable: Desativar
+      enable: Ativar
+      enqueue:
+        enqueued: Tarefa de deploy enfileirada
+        failed: 'A tarefa de deploy não foi enfileirada: veja o log para o motivo'
+      every_deploy: Em todo deploy
+      index:
+        task_count:
+          one: "%{count} tarefa declarada"
+          other: "%{count} tarefas declaradas"
+        title: Tarefas de deploy
+      not_found: Tarefa de deploy não encontrada
+      reasons:
+        disabled_armed_html: 'Desativada com uma execução de transição pendente: ative-a para deixá-la executar.'
+        disabled_html: 'Desativada: o agendador a ignora.'
+        due_html: 'Pronta: executa no próximo tick do agendador.'
+        manual_html: "· executada %{ago} (manual)"
+        never_html: Nunca executada após uma transição.
+        ran_html: Executada %{ago} em <code>%{revision}</code> (transição).
+        waiting_lagging_html:
+          one: 'Aguardando a transição: 1 processo ainda não executa <code>%{revision}</code>.'
+          other: 'Aguardando a transição: %{count} processos ainda não executam <code>%{revision}</code>.'
+        waiting_no_revision_html: 'Aguardando: a revisão da aplicação não está configurada.'
+        waiting_no_scheduler_html: 'Aguardando: nenhum agendador em execução.'
+        waiting_no_workers_html: Aguardando um worker em <code>%{revision}</code>.
+        waiting_unknown_html: 'Aguardando: não foi possível ler o estado da transição.'
+      run_now: Executar agora
+      show:
+        armed_html: "%{armed_for}, armada %{ago}"
+        back: Voltar
+        configuration: Configuração
+        labels:
+          arguments: Argumentos
+          armed: Execução pendente
+          command: Comando
+          description: Descrição
+          job_class: Classe do job
+          priority: Prioridade
+          queue: Fila
+          source: Origem
+        last_run: Última execução
+        no_runs: Nenhuma execução registrada ainda
+        recent_runs: Execuções recentes
+        run_headers:
+          enqueued_at: Enfileirada em
+          queue: Fila
+          revision: Revisão
+          triggered_by: Disparada por
+          version: Versão
+        source_config: Arquivo de configuração
+        source_dynamic: Dinâmica
+        state: Estado
+        version: Versão
+      states:
+        disabled: Desativada
+        due: Pronta
+        never: Nunca executada
+        ran: Executada
+        waiting: Aguardando
+      tasks_table:
+        empty_hint_html: Declare tarefas pontuais em <code>config/deploy_tasks.yml</code>; <code>rails generate pgbus:add_deploy_tasks</code> cria um arquivo inicial.
+        empty_title: Nenhuma tarefa de deploy declarada
+        headers:
+          actions: Ações
+          last_run: Última execução
+          queue: Fila
+          state: Estado
+          task: Tarefa
+          version: Versão
+        never: Nunca
+      toggle:
+        disabled: Tarefa de deploy desativada
+        enabled: Tarefa de deploy ativada
+        failed: Falha ao alternar a tarefa de deploy
+      triggered_by:
+        manual: Manual
+        rollover: Transição
+      version: v%{version}
     dialogs:
       cancel: Cancelar
       confirm: Confirmar
@@ -617,6 +713,7 @@ pt:
       nav:
         batches: Lotes
         dashboard: Painel
+        deploy_tasks: Deploy
         dlq: DLQ
         events: Eventos
         insights: Insights

data/config/locales/sv.yml

@@ -236,6 +236,102 @@ sv:
         payload: Innehåll
         retry: Försök igen
         title: 'DLQ-meddelande #%{id}'
+    deploy_tasks:
+      banner:
+        heading: Utrullning
+        lagging_html:
+          one: 'Väntar på utrullning: 1 process kör ännu inte <code>%{revision}</code>.'
+          other: 'Väntar på utrullning: %{count} processer kör ännu inte <code>%{revision}</code>.'
+        lagging_more: "…och %{count} till"
+        lagging_process: "%{kind}@%{hostname} pid %{pid} på %{revision}"
+        live_count: "%{count} på %{revision}"
+        live_heading: 'Aktiva processer:'
+        no_revision_hint_html: Sätt <code>config.app_revision</code>, eller någon av <code>PGBUS_REVISION</code>, <code>DASH_VERSION</code> eller <code>KAMAL_VERSION</code>, eller leverera en <code>REVISION</code>-fil. Kör nu fungerar fortfarande.
+        no_revision_html: 'Appens revision är inte konfigurerad: deploy-uppgifter väntar tills den är det.'
+        no_revision_label: ingen revision
+        no_scheduler_html: 'Ingen schemaläggare körs: deploy-uppgifter startas bara av schemaläggaren. Bedömningen använder instrumentpanelens egen revision.'
+        no_workers_html: 'Väntar på utrullning: ingen worker på <code>%{revision}</code> ännu.'
+        ready_html: 'Utrullad: varje aktiv process kör <code>%{revision}</code>. Väntande uppgifter körs vid schemaläggarens nästa tick.'
+        unavailable_html: Utrullningsstatusen kunde inte läsas.
+      default_queue: default
+      disable: Inaktivera
+      enable: Aktivera
+      enqueue:
+        enqueued: Deploy-uppgiften köad
+        failed: 'Deploy-uppgiften köades inte: se loggen för orsaken'
+      every_deploy: Vid varje deploy
+      index:
+        task_count:
+          one: "%{count} uppgift deklarerad"
+          other: "%{count} uppgifter deklarerade"
+        title: Deploy-uppgifter
+      not_found: Deploy-uppgiften hittades inte
+      reasons:
+        disabled_armed_html: 'Inaktiverad med en väntande utrullningskörning: aktivera den för att låta den köras.'
+        disabled_html: 'Inaktiverad: schemaläggaren hoppar över den.'
+        due_html: 'Redo: körs vid schemaläggarens nästa tick.'
+        manual_html: "· kördes %{ago} (manuellt)"
+        never_html: Har aldrig körts efter en utrullning.
+        ran_html: Kördes %{ago} på <code>%{revision}</code> (utrullning).
+        waiting_lagging_html:
+          one: 'Väntar på utrullning: 1 process kör ännu inte <code>%{revision}</code>.'
+          other: 'Väntar på utrullning: %{count} processer kör ännu inte <code>%{revision}</code>.'
+        waiting_no_revision_html: 'Väntar: appens revision är inte konfigurerad.'
+        waiting_no_scheduler_html: 'Väntar: ingen schemaläggare körs.'
+        waiting_no_workers_html: Väntar på en worker på <code>%{revision}</code>.
+        waiting_unknown_html: 'Väntar: utrullningsstatusen kunde inte läsas.'
+      run_now: Kör nu
+      show:
+        armed_html: "%{armed_for}, förberedd %{ago}"
+        back: Tillbaka
+        configuration: Konfiguration
+        labels:
+          arguments: Argument
+          armed: Väntande körning
+          command: Kommando
+          description: Beskrivning
+          job_class: Jobbklass
+          priority: Prioritet
+          queue: Kö
+          source: Källa
+        last_run: Senaste körning
+        no_runs: Inga körningar registrerade ännu
+        recent_runs: Senaste körningar
+        run_headers:
+          enqueued_at: Köad
+          queue: Kö
+          revision: Revision
+          triggered_by: Startad av
+          version: Version
+        source_config: Konfigurationsfil
+        source_dynamic: Dynamisk
+        state: Tillstånd
+        version: Version
+      states:
+        disabled: Inaktiverad
+        due: Redo
+        never: Aldrig körd
+        ran: Körd
+        waiting: Väntar
+      tasks_table:
+        empty_hint_html: Deklarera engångsuppgifter i <code>config/deploy_tasks.yml</code>; <code>rails generate pgbus:add_deploy_tasks</code> skapar en startfil.
+        empty_title: Inga deploy-uppgifter deklarerade
+        headers:
+          actions: Åtgärder
+          last_run: Senaste körning
+          queue: Kö
+          state: Tillstånd
+          task: Uppgift
+          version: Version
+        never: Aldrig
+      toggle:
+        disabled: Deploy-uppgiften inaktiverad
+        enabled: Deploy-uppgiften aktiverad
+        failed: Det gick inte att växla deploy-uppgiften
+      triggered_by:
+        manual: Manuellt
+        rollover: Utrullning
+      version: v%{version}
     dialogs:
       cancel: Avbryt
       confirm: Bekräfta
@@ -617,6 +713,7 @@ sv:
       nav:
         batches: Grupper
         dashboard: Instrumentpanel
+        deploy_tasks: Deploy
         dlq: DLQ
         events: Händelser
         insights: Insikter

data/config/routes.rb

@@ -45,6 +45,13 @@ Pgbus::Engine.routes.draw do
     end
   end
 
+  resources :deploy_tasks, only: %i[index show] do
+    member do
+      post :toggle
+      post :enqueue
+    end
+  end
+
   resources :batches, only: %i[index show]
   resources :processes, only: [:index]
 

data/lib/generators/pgbus/add_batch_callback_jobs_generator.rb

@@ -33,7 +33,10 @@ module Pgbus
         say "  2. Restart pgbus: bin/pgbus start"
         say ""
         say "You can now pass a configured ActiveJob instance to a batch:"
-        say "  Pgbus::Batch.new(on_finish: ReportJob.new.set(queue: :critical, wait: 5.minutes))"
+        say "  Pgbus.batch do |b|"
+        say "    b.on_complete ReportJob.new.set(queue: :critical, wait: 5.minutes)"
+        say "    b.add(*rows.map { |row| ImportRowJob.new(row.id) })"
+        say "  end"
         say ""
       end
 

data/lib/generators/pgbus/add_deploy_tasks_generator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+require_relative "migration_path"
+
+module Pgbus
+  module Generators
+    class AddDeployTasksGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+      include MigrationPath
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add deploy tasks tables (one-time jobs run after a rollover) to an existing Pgbus installation"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        migration_template "add_deploy_tasks_tables.rb.erb",
+                           File.join(pgbus_migrate_path, "add_pgbus_deploy_tasks_tables.rb")
+      end
+
+      def create_deploy_tasks_config
+        template "deploy_tasks.yml.erb", "config/deploy_tasks.yml"
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus deploy tasks installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{migrate_command_suffix}"
+        say "  2. Edit config/deploy_tasks.yml to declare your one-time tasks"
+        say "  3. Make sure every pgbus process knows the app revision (config.app_revision,"
+        say "     PGBUS_REVISION, DASH_VERSION, KAMAL_VERSION or a REVISION file)"
+        say "  4. Restart pgbus: bin/pgbus start"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+    end
+  end
+end

data/lib/generators/pgbus/templates/add_deploy_tasks_tables.rb.erb

@@ -0,0 +1,43 @@
+class AddPgbusDeployTasksTables < ActiveRecord::Migration<%= migration_version %>
+  # if_not_exists throughout: pgbus:update queues this migration when either
+  # table is missing, so a half-present schema must not fail it.
+  def change
+    create_table :pgbus_deploy_tasks, if_not_exists: true do |t|
+      t.string :key, null: false
+      t.string :class_name
+      t.string :command, limit: 2048
+      t.text :arguments
+      t.string :queue_name
+      t.integer :priority, default: 0
+      t.string :version, null: false, default: "1"
+      t.boolean :every_deploy, null: false, default: false
+      t.boolean :static, null: false, default: true
+      t.boolean :enabled, null: false, default: true
+      t.text :description
+      t.datetime :armed_at
+      t.string :armed_for
+      t.timestamps
+    end
+
+    add_index :pgbus_deploy_tasks, :key,
+      unique: true, name: "idx_pgbus_deploy_tasks_key", if_not_exists: true
+    add_index :pgbus_deploy_tasks, :armed_at,
+      where: "armed_at IS NOT NULL", name: "idx_pgbus_deploy_tasks_armed", if_not_exists: true
+
+    create_table :pgbus_deploy_task_runs, if_not_exists: true do |t|
+      t.string :task_key, null: false
+      t.string :version, null: false
+      t.string :revision
+      t.string :triggered_by, null: false
+      t.string :queue_name
+      t.bigint :msg_id
+      t.datetime :run_at, null: false
+      t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    add_index :pgbus_deploy_task_runs, [:task_key, :run_at],
+      name: "idx_pgbus_deploy_task_runs_task", if_not_exists: true
+    add_index :pgbus_deploy_task_runs, :run_at,
+      name: "idx_pgbus_deploy_task_runs_cleanup", if_not_exists: true
+  end
+end

data/lib/generators/pgbus/templates/deploy_tasks.yml.erb

@@ -0,0 +1,42 @@
+# Pgbus Deploy Tasks Configuration
+#
+# One-time jobs that run once the whole fleet has rolled over to a new app
+# revision: a backfill, a data fix, a cache warm. Same shape as
+# config/recurring.yml, without schedule:. Supports environment-scoped or
+# flat configuration.
+#
+# The scheduler enqueues a task only when every live pgbus process reports
+# the scheduler's own app revision and at least one worker on it is alive,
+# so a task never runs on old code during the deploy overlap window. The
+# revision comes from config.app_revision, then PGBUS_REVISION, DASH_VERSION,
+# KAMAL_VERSION, then the REVISION file at Rails.root. Without one, deploy
+# tasks wait (the dashboard says so) and only "Run now" enqueues them.
+#
+# Format:
+#   task_key:
+#     class: JobClassName      # ActiveJob class to run (or use command:)
+#     version: 1               # Runs once per version; bump it to run again after the next deploy
+#     every_deploy: false      # true: run after every rollover instead (version is ignored)
+#     queue: default           # Optional: queue name (default: pgbus default queue)
+#     args: [arg1, arg2]       # Optional: job arguments
+#     priority: 0              # Optional: numeric priority
+#     description: "text"      # Optional: human-readable description
+#
+# Command-based tasks (no job class needed):
+#   warm_fragment_caches:
+#     command: "FragmentCacheWarmer.run"
+#     every_deploy: true
+#
+# Examples:
+#
+# production:
+#   backfill_order_totals:
+#     class: BackfillOrderTotalsJob
+#     version: 2
+#     queue: maintenance
+#     args: [{ batch_size: 500 }]
+#     description: Recompute totals after the currency migration
+#
+#   warm_fragment_caches:
+#     command: "FragmentCacheWarmer.run"
+#     every_deploy: true

data/lib/generators/pgbus/templates/migration.rb.erb

@@ -170,6 +170,47 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
     add_index :pgbus_recurring_executions, :run_at,
       name: "idx_pgbus_recurring_executions_cleanup"
 
+    # Deploy tasks (synced from config/deploy_tasks.yml): one-time jobs the
+    # scheduler enqueues once the fleet has rolled over to a new revision.
+    # armed_at/armed_for is the compare-and-set claim; runs are the history.
+    create_table :pgbus_deploy_tasks do |t|
+      t.string :key, null: false
+      t.string :class_name
+      t.string :command, limit: 2048
+      t.text :arguments
+      t.string :queue_name
+      t.integer :priority, default: 0
+      t.string :version, null: false, default: "1"
+      t.boolean :every_deploy, null: false, default: false
+      t.boolean :static, null: false, default: true
+      t.boolean :enabled, null: false, default: true
+      t.text :description
+      t.datetime :armed_at
+      t.string :armed_for
+      t.timestamps
+    end
+
+    add_index :pgbus_deploy_tasks, :key,
+      unique: true, name: "idx_pgbus_deploy_tasks_key"
+    add_index :pgbus_deploy_tasks, :armed_at,
+      where: "armed_at IS NOT NULL", name: "idx_pgbus_deploy_tasks_armed"
+
+    create_table :pgbus_deploy_task_runs do |t|
+      t.string :task_key, null: false
+      t.string :version, null: false
+      t.string :revision
+      t.string :triggered_by, null: false
+      t.string :queue_name
+      t.bigint :msg_id
+      t.datetime :run_at, null: false
+      t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    add_index :pgbus_deploy_task_runs, [:task_key, :run_at],
+      name: "idx_pgbus_deploy_task_runs_task"
+    add_index :pgbus_deploy_task_runs, :run_at,
+      name: "idx_pgbus_deploy_task_runs_cleanup"
+
     # Stream queue registry — records which PGMQ queues back streams (vs jobs).
     # Stream and job queues share the pgbus_ namespace, so maintenance
     # (per-stream retention, orphan sweep) and wildcard workers rely on this
@@ -204,6 +245,8 @@ class CreatePgbusTables < ActiveRecord::Migration<%= migration_version %>
     execute "SELECT pgmq.drop_queue('pgbus_default')"
 
     drop_table :pgbus_stream_queues
+    drop_table :pgbus_deploy_task_runs
+    drop_table :pgbus_deploy_tasks
     drop_table :pgbus_recurring_executions
     drop_table :pgbus_recurring_tasks
     drop_table :pgbus_batch_executions

data/lib/pgbus/active_job/batch_id.rb

@@ -8,9 +8,9 @@ module Pgbus
     #
     # * +batch_id+ — the batch this job is a *member* of. Assigned by the
     #   executor from the payload's +pgbus_batch_id+ before +perform+, so a
-    #   running job can call +batch.enqueue+ to add siblings.
+    #   running job can call +batch.add+ to add siblings.
     # * +callback_batch_id+ — the batch this job *reports on*. Set on
-    #   +on_finish+/+on_success+/+on_failure+ jobs at fire time. A callback is
+    #   +on_complete+/+on_success+/+on_failure+ jobs at fire time. A callback is
     #   never a member of the batch it reports on, so its +batch_id+ is nil.
     #
     # Both round-trip through +serialize+/+deserialize+, and both are omitted
@@ -36,8 +36,14 @@ module Pgbus
         self.callback_batch_id = job_data["callback_batch_id"]
       end
 
-      # The batch this job reports on, or is a member of. nil when neither.
-      def batch
+      # Without a block: the batch this job reports on, or is a member of; nil
+      # when neither. With a block: open a new, independent batch through
+      # Pgbus.batch (this job's own membership is untouched).
+      #
+      #   batch(description: "Import") { |b| b.add(*rows.map { |r| RowJob.new(r.id) }) }
+      def batch(**options, &block)
+        return Pgbus.batch(**options, &block) if block
+        raise ArgumentError, "batch(...) with options needs a block: batch(description: ...) { |b| ... }" if options.any?
         return @batch if defined?(@batch)
 
         id = callback_batch_id || batch_id

data/lib/pgbus/batch/builder.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Batch
+    # What Pgbus.batch (and +batch { |b| ... }+ inside a job) yields: callback
+    # setters and +add+ on a batch that is open for the length of the block.
+    #
+    #   Pgbus.batch(description: "Import #{source.id}") do |b|
+    #     b.on_complete ImportDoneJob.new.set(queue: :critical)
+    #     b.on_failure  ImportFailedJob
+    #     b.add(*source.rows.map { |row| ImportRowJob.new(row.id) })
+    #   end
+    #
+    # The batch row exists (and a uniqueness key is held) before the block
+    # runs, and it stays pending until the block returns, so the order of the
+    # calls does not matter and a job that completes during the block cannot
+    # finish the batch early. Once the block has returned the builder is
+    # closed: add more jobs through +batch.add+ on the returned batch.
+    class Builder
+      CLOSED_MESSAGE = "the batch builder only works inside its Pgbus.batch block; " \
+                       "add more jobs later with batch.add on the batch Pgbus.batch returned"
+      UNTAGGED_MESSAGE = "b.add ran on a thread or fiber outside its Pgbus.batch block (or inside another " \
+                         "batch's block), where its jobs would not join the batch; call b.add from the block itself"
+
+      # Flatten +jobs+ and check that each one is an ActiveJob instance.
+      def self.job_instances(jobs)
+        jobs = jobs.flatten
+        invalid = jobs.grep_v(::ActiveJob::Base)
+        return jobs if invalid.empty?
+
+        raise ArgumentError,
+              "a batch takes job instances (MyJob.new(args), optionally .set(wait:, queue:)), got #{invalid.first.inspect}"
+      end
+
+      def initialize(batch)
+        @batch = batch
+        @closed = false
+      end
+
+      def on_complete(job = nil, &) = set_callback(:on_complete, job, &)
+
+      def on_success(job = nil, &) = set_callback(:on_success, job, &)
+
+      def on_failure(job = nil, &) = set_callback(:on_failure, job, &)
+
+      # Deprecated alias of #on_complete until 1.0.
+      def on_finish(job = nil, &)
+        Batch.warn_on_finish_deprecated
+        set_callback(:on_complete, job, &)
+      end
+
+      # Enqueue job instances into the batch with one bulk send (and anything
+      # the block enqueues). Returns the jobs. Membership is the batch tag the
+      # block holds (Thread.current, so fiber-local), so an add from another
+      # thread or fiber, or from inside a nested batch's block, raises rather
+      # than send jobs that would not be counted.
+      def add(*jobs)
+        raise ArgumentError, CLOSED_MESSAGE if @closed
+        raise ArgumentError, UNTAGGED_MESSAGE unless Thread.current[:pgbus_batch_id] == @batch.batch_id
+
+        jobs = self.class.job_instances(jobs)
+        ::ActiveJob.perform_all_later(jobs) if jobs.any?
+        yield if block_given?
+        jobs
+      end
+
+      # Called by Batch.start when the block returns or raises.
+      def close
+        @closed = true
+      end
+
+      private
+
+      def set_callback(slot, job, &block)
+        raise ArgumentError, CLOSED_MESSAGE if @closed
+
+        unless block.nil? && callback_job?(job)
+          raise ArgumentError,
+                "#{slot} takes an ActiveJob class or a configured instance (MyJob.new.set(queue: :critical)), " \
+                "not #{block ? "a block" : job.inspect}: callbacks are stored on the batch row"
+        end
+
+        @batch.send(:write_callback, slot, job)
+        self
+      end
+
+      def callback_job?(job)
+        (job.is_a?(Class) && job < ::ActiveJob::Base) || job.is_a?(::ActiveJob::Base)
+      end
+    end
+  end
+end