pgbus 0.1.4 → 0.1.6

data/lib/generators/pgbus/add_job_locks_generator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddJobLocksGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add job locks table for uniqueness guarantees"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_job_locks.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_job_locks.rb"
+        else
+          migration_template "add_job_locks.rb.erb",
+                             "db/migrate/add_pgbus_job_locks.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus job locks table installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. Add `ensures_uniqueness` to your job classes (DSL is auto-included)"
+        say "  3. Restart pgbus: bin/pgbus start"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/add_job_stats_generator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddJobStatsGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add job stats table for dashboard insights and performance tracking"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_job_stats.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_job_stats.rb"
+        else
+          migration_template "add_job_stats.rb.erb",
+                             "db/migrate/add_pgbus_job_stats.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus job stats table installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. Stats collection is enabled by default"
+        say "  3. View insights at /pgbus/insights"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/add_outbox_generator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddOutboxGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add outbox table for transactional event publishing"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_outbox.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_outbox.rb"
+        else
+          migration_template "add_outbox.rb.erb",
+                             "db/migrate/add_pgbus_outbox.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus outbox installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. Enable in config: config.outbox_enabled = true"
+        say "  3. Restart pgbus: bin/pgbus start"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/add_queue_states_generator.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module Pgbus
+  module Generators
+    class AddQueueStatesGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      desc "Add queue states table for pause/resume and circuit breaker support"
+
+      class_option :database,
+                   type: :string,
+                   default: nil,
+                   desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
+
+      def create_migration_file
+        if separate_database?
+          migration_template "add_queue_states.rb.erb",
+                             "db/pgbus_migrate/add_pgbus_queue_states.rb"
+        else
+          migration_template "add_queue_states.rb.erb",
+                             "db/migrate/add_pgbus_queue_states.rb"
+        end
+      end
+
+      def display_post_install
+        say ""
+        say "Pgbus queue states table installed!", :green
+        say ""
+        say "Next steps:"
+        say "  1. Run: rails db:migrate#{":#{options[:database]}" if separate_database?}"
+        say "  2. Restart pgbus: bin/pgbus start"
+        say ""
+      end
+
+      private
+
+      def migration_version
+        "[#{ActiveRecord::Migration.current_version}]"
+      end
+
+      def separate_database?
+        options[:database].present?
+      end
+    end
+  end
+end

data/lib/generators/pgbus/add_recurring_generator.rb

@@ -17,7 +17,7 @@ module Pgbus
                    default: nil,
                    desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
 
-      def create_migration
+      def create_migration_file
         if separate_database?
           migration_template "add_recurring_tables.rb.erb",
                              "db/pgbus_migrate/add_pgbus_recurring_tables.rb"

data/lib/generators/pgbus/install_generator.rb

@@ -24,7 +24,7 @@ module Pgbus
                    desc: "Use a separate database for pgbus tables (e.g. --database=pgbus). " \
                          "Migrations go to db/pgbus_migrate/ and schema to db/pgbus_schema.rb"
 
-      def create_migration
+      def create_migration_file
         if separate_database?
           migration_template "migration.rb.erb",
                              "db/pgbus_migrate/create_pgbus_tables.rb"

data/lib/generators/pgbus/templates/add_job_locks.rb.erb

@@ -0,0 +1,21 @@
+class AddPgbusJobLocks < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pgbus_job_locks do |t|
+      t.string :lock_key, null: false
+      t.string :job_class, null: false
+      t.string :job_id
+      t.string :state, null: false, default: "queued"
+      t.integer :owner_pid
+      t.string :owner_hostname
+      t.datetime :locked_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+      t.datetime :expires_at, null: false
+    end
+
+    add_index :pgbus_job_locks, :lock_key,
+      unique: true, name: "idx_pgbus_job_locks_key"
+    add_index :pgbus_job_locks, :expires_at,
+      name: "idx_pgbus_job_locks_expires"
+    add_index :pgbus_job_locks, [:state, :owner_pid],
+      name: "idx_pgbus_job_locks_reaper"
+  end
+end

data/lib/generators/pgbus/templates/add_job_stats.rb.erb

@@ -0,0 +1,18 @@
+class AddPgbusJobStats < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pgbus_job_stats do |t|
+      t.string :job_class, null: false
+      t.string :queue_name, null: false
+      t.string :status, null: false
+      t.integer :duration_ms, null: false, default: 0
+      t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    add_index :pgbus_job_stats, :created_at,
+      name: "idx_pgbus_job_stats_time"
+    add_index :pgbus_job_stats, [:job_class, :created_at],
+      name: "idx_pgbus_job_stats_class_time"
+    add_index :pgbus_job_stats, [:status, :created_at],
+      name: "idx_pgbus_job_stats_status_time"
+  end
+end

data/lib/generators/pgbus/templates/add_outbox.rb.erb

@@ -0,0 +1,25 @@
+class AddPgbusOutbox < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pgbus_outbox_entries do |t|
+      t.string :queue_name
+      t.string :routing_key
+      t.jsonb :payload, null: false
+      t.jsonb :headers
+      t.integer :priority, default: 0
+      t.integer :delay, default: 0
+      t.datetime :published_at
+      t.datetime :created_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
+    end
+
+    add_check_constraint :pgbus_outbox_entries,
+      "(queue_name IS NOT NULL) <> (routing_key IS NOT NULL)",
+      name: "chk_pgbus_outbox_destination"
+
+    add_index :pgbus_outbox_entries, :published_at,
+      where: "published_at IS NULL",
+      name: "idx_pgbus_outbox_unpublished"
+    add_index :pgbus_outbox_entries, :published_at,
+      where: "published_at IS NOT NULL",
+      name: "idx_pgbus_outbox_cleanup"
+  end
+end

data/lib/generators/pgbus/templates/add_queue_states.rb.erb

@@ -0,0 +1,16 @@
+class AddPgbusQueueStates < ActiveRecord::Migration<%= migration_version %>
+  def change
+    create_table :pgbus_queue_states do |t|
+      t.string :queue_name, null: false
+      t.boolean :paused, null: false, default: false
+      t.string :paused_reason
+      t.datetime :paused_at
+      t.integer :circuit_breaker_trip_count, default: 0
+      t.datetime :circuit_breaker_resume_at
+      t.timestamps
+    end
+
+    add_index :pgbus_queue_states, :queue_name,
+      unique: true, name: "idx_pgbus_queue_states_queue_name"
+  end
+end

data/lib/generators/pgbus/upgrade_pgmq_generator.rb

@@ -17,7 +17,7 @@ module Pgbus
                    default: nil,
                    desc: "Use a separate database for pgbus tables (e.g. --database=pgbus)"
 
-      def create_migration
+      def create_migration_file
         if separate_database?
           migration_template "upgrade_pgmq.rb.erb",
                              "db/pgbus_migrate/upgrade_pgmq_to_v#{target_version_slug}.rb"

data/lib/pgbus/active_job/adapter.rb

@@ -9,8 +9,11 @@ module Pgbus
         queue = active_job.queue_name || Pgbus.configuration.default_queue
         payload_hash = Serializer.serialize_job_hash(active_job)
         payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
+        payload_hash = Uniqueness.inject_metadata(active_job, payload_hash)
         payload_hash = inject_batch_metadata(payload_hash)
 
+        return active_job if uniqueness_rejected?(active_job, payload_hash)
+
         enqueue_with_concurrency(active_job, queue, payload_hash)
       end
 
@@ -18,14 +21,27 @@ module Pgbus
         queue = active_job.queue_name || Pgbus.configuration.default_queue
         payload_hash = Serializer.serialize_job_hash(active_job)
         payload_hash = Concurrency.inject_metadata(active_job, payload_hash)
+        payload_hash = Uniqueness.inject_metadata(active_job, payload_hash)
         payload_hash = inject_batch_metadata(payload_hash)
         delay = [(timestamp - Time.now.to_f).ceil, 0].max
 
+        return active_job if uniqueness_rejected?(active_job, payload_hash)
+
         enqueue_with_concurrency(active_job, queue, payload_hash, delay: delay)
       end
 
       def enqueue_all(active_jobs)
-        active_jobs.group_by { |j| j.queue_name || Pgbus.configuration.default_queue }.each do |queue, jobs|
+        # Jobs with uniqueness must go through individual enqueue to acquire locks
+        unique, bulk = active_jobs.partition { |j| Uniqueness.uniqueness_config(j) }
+        unique.each do |j|
+          if j.scheduled_at && j.scheduled_at > Time.now
+            enqueue_at(j, j.scheduled_at.to_f)
+          else
+            enqueue(j)
+          end
+        end
+
+        bulk.group_by { |j| j.queue_name || Pgbus.configuration.default_queue }.each do |queue, jobs|
           enqueue_immediate(queue, jobs.reject { |j| j.scheduled_at && j.scheduled_at > Time.now })
           jobs.select { |j| j.scheduled_at && j.scheduled_at > Time.now }.each { |j| enqueue_at(j, j.scheduled_at.to_f) }
         end
@@ -38,39 +54,50 @@ module Pgbus
       def enqueue_with_concurrency(active_job, queue, payload_hash, delay: 0)
         key = Concurrency.extract_key(payload_hash)
         concurrency = concurrency_config(active_job)
+        priority = active_job.try(:priority)
 
         if key && concurrency
           result = Concurrency::Semaphore.acquire(key, concurrency[:limit], concurrency[:duration])
 
           if result == :acquired
-            msg_id = Pgbus.client.send_message(queue, payload_hash, delay: delay)
+            msg_id = Pgbus.client.send_message(queue, payload_hash, delay: delay, priority: priority)
             active_job.provider_job_id = msg_id
           else
-            handle_conflict(concurrency, active_job, key, queue, payload_hash)
+            handle_conflict(concurrency, active_job, key, queue, payload_hash, priority: priority)
           end
         else
-          msg_id = Pgbus.client.send_message(queue, payload_hash, delay: delay)
+          msg_id = Pgbus.client.send_message(queue, payload_hash, delay: delay, priority: priority)
           active_job.provider_job_id = msg_id
         end
 
+        Thread.current[:pgbus_acquired_uniqueness_key] = nil
         active_job
-      rescue Pgbus::SchemaNotReady => e
-        Pgbus.logger.error { "[Pgbus] #{e.message}" }
-        raise
+      rescue StandardError => e
+        # Roll back the uniqueness lock if enqueue failed
+        rollback_key = Thread.current[:pgbus_acquired_uniqueness_key]
+        if rollback_key
+          begin
+            Uniqueness.release_lock(rollback_key)
+          rescue StandardError => rollback_error
+            Pgbus.logger.warn { "[Pgbus] Lock rollback failed: #{rollback_error.message}" }
+          end
+          Thread.current[:pgbus_acquired_uniqueness_key] = nil
+        end
+        raise e
       end
 
       def concurrency_config(active_job)
         active_job.class.respond_to?(:pgbus_concurrency) && active_job.class.pgbus_concurrency
       end
 
-      def handle_conflict(concurrency, active_job, key, queue, payload_hash)
+      def handle_conflict(concurrency, active_job, key, queue, payload_hash, priority: nil)
         case concurrency[:on_conflict]
         when :block
           Concurrency::BlockedExecution.insert(
             concurrency_key: key,
             queue_name: queue,
             payload: payload_hash,
-            priority: active_job.try(:priority) || 0,
+            priority: priority || Pgbus.configuration.default_priority,
             duration: concurrency[:duration]
           )
         when :discard
@@ -80,6 +107,34 @@ module Pgbus
         end
       end
 
+      def uniqueness_rejected?(active_job, payload_hash)
+        uniqueness_key = Uniqueness.extract_key(payload_hash)
+        return false unless uniqueness_key
+
+        result = Uniqueness.acquire_enqueue_lock(uniqueness_key, active_job)
+
+        # :no_lock means no enqueue-time lock needed (e.g. :while_executing strategy)
+        return false if result == :no_lock
+
+        # Store the acquired key so we can release it if enqueue fails
+        Thread.current[:pgbus_acquired_uniqueness_key] = uniqueness_key if result == :acquired
+        return false if result == :acquired
+
+        config = Uniqueness.uniqueness_config(active_job)
+        case config[:on_conflict]
+        when :reject
+          raise JobNotUnique, "Job #{active_job.class.name} is already locked"
+        when :discard
+          Pgbus.logger.info { "[Pgbus] Discarding duplicate job #{active_job.class.name}" }
+          true
+        when :log
+          Pgbus.logger.warn { "[Pgbus] Duplicate job #{active_job.class.name} detected" }
+          true
+        else
+          true
+        end
+      end
+
       def inject_batch_metadata(payload_hash)
         batch_id = Thread.current[:pgbus_batch_id]
         return payload_hash unless batch_id

data/lib/pgbus/active_job/executor.rb

@@ -10,33 +10,58 @@ module Pgbus
         @config = config
       end
 
-      def execute(message, queue_name)
+      def execute(message, queue_name, source_queue: nil)
+        execution_start = monotonic_now
         payload = JSON.parse(message.message)
         read_count = message.read_ct.to_i
 
         if read_count > config.max_retries
-          handle_dead_letter(message, queue_name, payload)
+          handle_dead_letter(message, queue_name, payload, source_queue: source_queue)
           signal_concurrency(payload)
           signal_batch_discarded(payload)
+          Uniqueness.release_lock(Uniqueness.extract_key(payload))
+          record_stat(payload, queue_name, "dead_lettered", execution_start)
           return :dead_lettered
         end
 
         job_class = payload["job_class"]
+        uniqueness_key = Uniqueness.extract_key(payload)
+        uniqueness_strategy = Uniqueness.extract_strategy(payload)
+        uniqueness_ttl = payload[Uniqueness::TTL_KEY] || Uniqueness::DEFAULT_LOCK_TTL
+
+        if uniqueness_key
+          case uniqueness_strategy
+          when :until_executed
+            # Transition the queued lock to executing state with our PID.
+            # The lock was acquired at enqueue time — now we claim ownership
+            # so the reaper can correlate it with our heartbeat.
+            Uniqueness.claim_for_execution!(uniqueness_key, ttl: uniqueness_ttl)
+          when :while_executing
+            # Acquire the lock now. If another worker is already executing
+            # this job, skip it — VT will expire and it'll be retried.
+            unless Uniqueness.acquire_execution_lock(uniqueness_key, payload)
+              Pgbus.logger.info { "[Pgbus] Skipping duplicate execution for #{job_class}" }
+              return :skipped
+            end
+          end
+        end
 
         job_succeeded = false
 
         Instrumentation.instrument("pgbus.executor.execute", queue: queue_name, job_class: job_class) do
           job = ::ActiveJob::Base.deserialize(payload)
           execute_job(job)
-          client.archive_message(queue_name, message.msg_id.to_i)
+          archive_from(queue_name, message.msg_id.to_i, source_queue: source_queue)
           job_succeeded = true
         end
 
         instrument("pgbus.job_completed", queue: queue_name, job_class: job_class)
+        record_stat(payload, queue_name, "success", execution_start)
         :success
       rescue StandardError => e
         handle_failure(message, queue_name, e)
         instrument("pgbus.job_failed", queue: queue_name, job_class: payload&.dig("job_class"), error: e.class.name)
+        record_stat(payload, queue_name, "failed", execution_start)
         # Don't signal concurrency on transient failure — the job will be retried.
         # Semaphore is released only on success or dead-lettering.
         :failed
@@ -47,6 +72,8 @@ module Pgbus
         if job_succeeded
           signal_concurrency(payload)
           signal_batch_completed(payload)
+          # Release uniqueness lock on successful completion (both strategies)
+          Uniqueness.release_lock(uniqueness_key) if uniqueness_key
         end
       end
 
@@ -60,6 +87,24 @@ module Pgbus
         end
       end
 
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      def record_stat(payload, queue_name, status, start_time)
+        return unless config.stats_enabled
+
+        duration_ms = ((monotonic_now - start_time) * 1000).round
+        JobStat.record!(
+          job_class: payload&.dig("job_class") || "unknown",
+          queue_name: queue_name,
+          status: status,
+          duration_ms: duration_ms
+        )
+      rescue StandardError => e
+        Pgbus.logger.debug { "[Pgbus] Stat recording failed: #{e.message}" }
+      end
+
       def handle_failure(_message, _queue_name, error)
         Pgbus.logger.error { "[Pgbus] Job failed: #{error.class}: #{error.message}" }
         Pgbus.logger.debug { error.backtrace&.join("\n") }
@@ -109,12 +154,29 @@ module Pgbus
         Pgbus.logger.warn { "[Pgbus] Batch discard signal failed: #{e.message}" }
       end
 
-      def handle_dead_letter(message, queue_name, payload)
+      def archive_from(queue_name, msg_id, source_queue: nil)
+        if source_queue
+          client.archive_from_queue(source_queue, msg_id)
+        else
+          client.archive_message(queue_name, msg_id)
+        end
+      end
+
+      def handle_dead_letter(message, queue_name, payload, source_queue: nil)
         Pgbus.logger.warn do
           job_class = payload["job_class"] || "unknown"
           "[Pgbus] Moving job #{job_class} to dead letter queue after #{message.read_ct} attempts"
         end
-        client.move_to_dead_letter(queue_name, message)
+        if source_queue
+          client.ensure_dead_letter_queue(queue_name)
+          dlq_name = config.dead_letter_queue_name(queue_name)
+          client.transaction do |txn|
+            txn.produce(dlq_name, message.message, headers: message.headers)
+            txn.delete(source_queue, message.msg_id.to_i)
+          end
+        else
+          client.move_to_dead_letter(queue_name, message)
+        end
       end
     end
   end