bsv-wallet-postgres 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d007064fafd9853cdc8696661ef969ddef47c0eeb9a1c0451acc7793a67fd1f1
-  data.tar.gz: 21335a50ff36fbaf390ddfe558e5efbb67514ae4da5189f0b6b9c0a2a4b7c2e6
+  metadata.gz: 69980501d8a03bd6552190800aba40a65f3c34d2b6633a3a59d58030a11d54e3
+  data.tar.gz: 7ee7f4c26fc294d9f227cdae83e3c9d9dcc7cce64592dda289db8c223c6b34ec
 SHA512:
-  metadata.gz: bd2578b3303a8b2a1407c51366aa9a30e78d28e38875fb5f229d5408a76f02a6ecbcaddfa3e1a01d770257e0c3e3f18ea15a257097cd5c5cc5444b4d4501d382
-  data.tar.gz: e296a4026b3bddedd41cd572ad627c279ee9aaf16ca4d7ee061c11718a1b81e27cb76e9583dda0012bf55cdfaf7c28a34791c28306951802879c76463efd7157
+  metadata.gz: 5fa70d4686c6f55d782578f9f7c2b6bb290c113ebdd2a8e05a11d8062e56ba21337a2ce3bf8d96af32a13ac60d05d8328f6608d560f6b2dce3b1514045f6b1ef
+  data.tar.gz: 371d0d3ccc53cf8c247aed4fa449a232b0a9f3ce46e8c7ebfbc311f5f23eae7043a186e0678c14fe5f3693ced12e3458965d3a93e471fcdbc2f46a1661961eac

data/CHANGELOG.md

@@ -5,6 +5,22 @@ All notable changes to the `bsv-wallet-postgres` gem are documented here.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/)
 and this gem adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.4.0 — 2026-04-12
+
+### Added
+- `BSV::Wallet::SolidQueueAdapter` — PostgreSQL-backed async broadcast queue implementing the `BroadcastQueue` interface
+- Migration 006: `wallet_broadcast_jobs` table with `FOR UPDATE SKIP LOCKED` polling support
+- Background worker thread broadcasts transactions and promotes/rolls back wallet state
+- Recovery on restart via stale `locked_at` detection
+- Idempotent enqueue on duplicate txid (crash recovery)
+- `MAX_ATTEMPTS` enforcement (5) prevents infinite retry loops
+- Guard refuses MemoryStore attachment
+
+### Fixed
+- Migration timestamps use `timestamptz` (matching migration 004 pattern)
+- `start()` check-and-set is atomic under mutex (prevents TOCTOU double-spawn)
+- Deserialization failures mark job as failed immediately (no tight retry loop)
+
 ## 0.3.1 — 2026-04-12
 
 ### Fixed

data/lib/bsv/wallet_postgres/migrations/006_create_broadcast_jobs.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Broadcast job queue table for the SolidQueueAdapter.
+#
+# Tracks outbound transactions that need to be broadcast to the network,
+# with state machine, retry counters, and advisory locking support so that
+# multiple worker processes can poll the queue without double-processing.
+#
+# === Columns
+#
+# * +txid+              — TEXT NOT NULL UNIQUE. Prevents duplicate queue entries
+#                         for the same transaction.
+#
+# * +status+            — TEXT NOT NULL DEFAULT 'unsent'. State machine values:
+#                         'unsent', 'sending', 'completed', 'failed'.
+#
+# * +beef_hex+          — TEXT NOT NULL. Serialised BEEF payload to broadcast.
+#
+# * +input_outpoints+   — text[], nullable. Outpoints consumed by the tx, used
+#                         to release UTXOs on success or rollback on failure.
+#
+# * +change_outpoints+  — text[], nullable. Change outpoints created by the tx,
+#                         used to mark outputs spendable after confirmation.
+#
+# * +fund_ref+          — TEXT, nullable. Caller-supplied funding reference for
+#                         observability and reconciliation.
+#
+# * +attempts+          — INTEGER NOT NULL DEFAULT 0. Incremented on each
+#                         broadcast attempt; gates retry logic.
+#
+# * +last_error+        — TEXT, nullable. Message from the last failed attempt.
+#
+# * +locked_at+         — TIMESTAMPTZ, nullable. Set when a worker claims the
+#                         job; cleared on completion or stale-lock recovery.
+#
+# * +created_at+        — TIMESTAMPTZ NOT NULL DEFAULT NOW().
+#
+# * +updated_at+        — TIMESTAMPTZ NOT NULL DEFAULT NOW().
+#
+# === Index
+#
+# A composite index on +(status, locked_at)+ (named +broadcast_jobs_poll_idx+)
+# supports the hot-path poll query:
+#
+#   WHERE status = 'unsent'
+#      OR (status = 'sending' AND locked_at < NOW() - interval '300 seconds'
+#          AND attempts < 5)
+#   ORDER BY created_at
+#   FOR UPDATE SKIP LOCKED
+Sequel.migration do
+  change do
+    create_table(:wallet_broadcast_jobs) do
+      primary_key :id, type: :Bignum
+      String   :txid,             null: false, unique: true
+      String   :status,           null: false, default: 'unsent'
+      String   :beef_hex,         text: true, null: false
+      column   :input_outpoints,  'text[]'
+      column   :change_outpoints, 'text[]'
+      String   :fund_ref
+      Integer  :attempts,         null: false, default: 0
+      String   :last_error,       text: true
+      column :locked_at,   :timestamptz
+      column :created_at,  :timestamptz, null: false, default: Sequel::CURRENT_TIMESTAMP
+      column :updated_at,  :timestamptz, null: false, default: Sequel::CURRENT_TIMESTAMP
+      index %i[status locked_at], name: :broadcast_jobs_poll_idx
+    end
+  end
+end

data/lib/bsv/wallet_postgres/solid_queue_adapter.rb

@@ -0,0 +1,317 @@
+# frozen_string_literal: true
+
+require 'sequel'
+
+module BSV
+  module Wallet
+    # PostgreSQL-backed asynchronous broadcast queue adapter.
+    #
+    # Persists outbound transactions to the +wallet_broadcast_jobs+ table and
+    # processes them in a background worker thread. Designed for multi-process
+    # deployments where +InlineQueue+'s synchronous broadcast is unacceptable.
+    #
+    # The worker uses +SELECT ... FOR UPDATE SKIP LOCKED+ so multiple processes
+    # can run +SolidQueueAdapter+ against the same database safely — only one
+    # worker will claim a given job.
+    #
+    # === Lifecycle
+    #
+    #   adapter = BSV::Wallet::SolidQueueAdapter.new(
+    #     db: sequel_db,
+    #     storage: postgres_store,
+    #     broadcaster: arc_broadcaster
+    #   )
+    #   adapter.start   # spawns background worker thread
+    #   # ... process transactions ...
+    #   adapter.drain   # stop + join (blocks until current poll cycle completes)
+    #
+    # === Recovery
+    #
+    # On restart, the worker's first poll naturally finds stale +sending+ jobs
+    # (those whose +locked_at+ is older than +stale_threshold+ seconds) and
+    # re-broadcasts them. No special recovery code is needed.
+    #
+    # === Drain warning
+    #
+    # +drain+ blocks until the current poll cycle completes. If a job is
+    # mid-broadcast this may take several seconds. Jobs enqueued after
+    # +@running = false+ but before the worker thread exits will remain
+    # +unsent+ until the next +start+.
+    class SolidQueueAdapter
+      include BSV::Wallet::BroadcastQueue
+
+      # Default number of seconds between poll cycles.
+      DEFAULT_POLL_INTERVAL = 8
+
+      # Default number of seconds before a +sending+ job is considered stale
+      # and eligible for retry. Configurable for testability.
+      STALE_THRESHOLD = 300
+
+      # Maximum number of broadcast attempts before a job is abandoned.
+      # After this many failures the job remains +failed+ permanently.
+      MAX_ATTEMPTS = 5
+
+      # @param db [Sequel::Database] a Sequel database handle (shared with PostgresStore)
+      # @param storage [StorageAdapter] wallet storage adapter (must not be MemoryStore)
+      # @param broadcaster [#broadcast] broadcaster object
+      # @param poll_interval [Integer] seconds between worker poll cycles
+      # @param stale_threshold [Integer] seconds before a +sending+ job is retried
+      # @raise [ArgumentError] if +storage+ is a +BSV::Wallet::MemoryStore+
+      # @raise [ArgumentError] if +broadcaster+ is +nil+
+      def initialize(db:, storage:, broadcaster:, poll_interval: DEFAULT_POLL_INTERVAL, stale_threshold: STALE_THRESHOLD)
+        if storage.is_a?(BSV::Wallet::MemoryStore)
+          raise ArgumentError, 'SolidQueueAdapter requires a persistent storage adapter — MemoryStore is not supported'
+        end
+        raise ArgumentError, 'SolidQueueAdapter requires a broadcaster' if broadcaster.nil?
+
+        @db             = db
+        @storage        = storage
+        @broadcaster    = broadcaster
+        @poll_interval  = poll_interval
+        @stale_threshold = stale_threshold
+
+        @mutex         = Mutex.new
+        @running       = false
+        @worker_thread = nil
+      end
+
+      # Returns +true+ — this adapter executes broadcast asynchronously.
+      #
+      # @return [Boolean]
+      def async?
+        true
+      end
+
+      # Persists a transaction to the broadcast job queue and returns immediately.
+      #
+      # Inserts a row into +wallet_broadcast_jobs+ with status +unsent+. If a row
+      # already exists for the same +txid+ (e.g. after a crash and restart), the
+      # +UniqueConstraintViolation+ is rescued and the existing status is returned
+      # instead.
+      #
+      # @param payload [Hash] broadcast payload (see +BroadcastQueue+ module docs)
+      # @return [Hash] +{ txid: String, broadcast_status: 'sending' }+
+      def enqueue(payload)
+        txid             = payload[:txid]
+        beef_binary      = payload[:beef_binary]
+        input_outpoints  = payload[:input_outpoints]
+        change_outpoints = payload[:change_outpoints]
+        fund_ref         = payload[:fund_ref]
+
+        row = {
+          txid: txid,
+          beef_hex: beef_binary.unpack1('H*'),
+          input_outpoints: input_outpoints ? Sequel.pg_array(input_outpoints, :text) : nil,
+          change_outpoints: change_outpoints ? Sequel.pg_array(change_outpoints, :text) : nil,
+          fund_ref: fund_ref,
+          status: 'unsent'
+        }
+
+        begin
+          @db[:wallet_broadcast_jobs].insert(row)
+        rescue Sequel::UniqueConstraintViolation
+          existing_status = @db[:wallet_broadcast_jobs].where(txid: txid).get(:status)
+          return { txid: txid, broadcast_status: existing_status || 'sending' }
+        end
+
+        { txid: txid, broadcast_status: 'sending' }
+      end
+
+      # Returns the broadcast status for a previously enqueued transaction.
+      #
+      # Reads directly from the jobs table — the authoritative source of truth
+      # for async broadcast status.
+      #
+      # @param txid [String] hex transaction identifier
+      # @return [String, nil] status string or +nil+ if no job exists
+      def status(txid)
+        @db[:wallet_broadcast_jobs].where(txid: txid).get(:status)
+      end
+
+      # Spawns the background worker thread.
+      #
+      # Safe to call multiple times — returns immediately if already running.
+      # The check-and-set is atomic under the mutex to prevent two concurrent
+      # +start+ calls from spawning duplicate worker threads.
+      #
+      # @return [void]
+      def start
+        @mutex.synchronize do
+          return if @running || @worker_thread&.alive?
+
+          @running = true
+        end
+        @worker_thread = Thread.new { worker_loop }
+      end
+
+      # Signals the worker to stop after the current poll cycle.
+      #
+      # Non-blocking — returns immediately without waiting for the thread.
+      #
+      # @return [void]
+      def stop
+        @mutex.synchronize { @running = false }
+      end
+
+      # Stops the worker and blocks until the current poll cycle completes.
+      #
+      # Safe to call when +start+ has not been called (+@worker_thread+ is nil).
+      #
+      # @return [void]
+      def drain
+        stop
+        @worker_thread&.join
+      end
+
+      private
+
+      # Main loop for the background worker thread.
+      #
+      # Calls +poll_once+ in a loop, sleeping +poll_interval+ seconds between
+      # cycles. Unexpected errors from +poll_once+ are logged to stderr and the
+      # loop continues — the worker thread must not die silently.
+      def worker_loop
+        while running?
+          begin
+            poll_once
+          rescue StandardError => e
+            warn "[bsv-wallet] SolidQueueAdapter worker error: #{e.message}"
+          end
+          sleep(@poll_interval) if running?
+        end
+      end
+
+      # Thread-safe check of the running flag.
+      #
+      # @return [Boolean]
+      def running?
+        @mutex.synchronize { @running }
+      end
+
+      # Claims and processes a single pending broadcast job.
+      #
+      # Uses +SELECT ... FOR UPDATE SKIP LOCKED+ so concurrent workers do not
+      # double-process the same job. Stale +sending+ jobs (those whose +locked_at+
+      # is older than +stale_threshold+ seconds) are also eligible for retry,
+      # up to +MAX_ATTEMPTS+.
+      #
+      # NOTE: The entire +process_job+ call (including the network broadcast)
+      # runs inside this transaction, holding the row lock and a database
+      # connection for the duration. This is acceptable for a single worker
+      # thread with an 8-second poll interval but would need restructuring
+      # for high-throughput multi-worker deployments.
+      #
+      # @return [void]
+      def poll_once
+        @db.transaction do
+          job = @db[:wallet_broadcast_jobs]
+                .where(Sequel.lit(
+                         "status = 'unsent' OR " \
+                         "(status = 'sending' AND locked_at < NOW() - " \
+                         "#{@stale_threshold.to_i} * interval '1 second' " \
+                         "AND attempts < #{MAX_ATTEMPTS})"
+                       ))
+                .order(:created_at)
+                .limit(1)
+                .for_update
+                .skip_locked
+                .first
+          return unless job
+
+          process_job(job)
+        end
+      end
+
+      # Marks a job as +sending+, broadcasts the transaction, and promotes or
+      # rolls back wallet state based on the outcome.
+      #
+      # Deserialization failures (corrupt +beef_hex+) are caught and mark the
+      # job as +failed+ immediately to prevent infinite retry loops.
+      #
+      # @param job [Hash] row from +wallet_broadcast_jobs+
+      # @return [void]
+      def process_job(job)
+        @db[:wallet_broadcast_jobs].where(id: job[:id]).update(
+          status: 'sending',
+          locked_at: Sequel.lit('NOW()'),
+          attempts: Sequel.lit('attempts + 1'),
+          updated_at: Sequel.lit('NOW()')
+        )
+
+        begin
+          tx = BSV::Transaction::Transaction.from_beef_hex(job[:beef_hex])
+        rescue StandardError => e
+          @db[:wallet_broadcast_jobs].where(id: job[:id]).update(
+            status: 'failed',
+            last_error: "Deserialization failed: #{e.message}",
+            updated_at: Sequel.lit('NOW()')
+          )
+          return
+        end
+
+        input_outpoints  = job[:input_outpoints]&.to_a
+        change_outpoints = job[:change_outpoints]&.to_a
+        txid             = job[:txid]
+        fund_ref         = job[:fund_ref]
+
+        begin
+          @broadcaster.broadcast(tx)
+        rescue StandardError => e
+          if input_outpoints && !input_outpoints.empty?
+            rollback(input_outpoints, change_outpoints, txid, fund_ref)
+          elsif txid
+            @storage.update_action_status(txid, 'failed')
+          end
+          @db[:wallet_broadcast_jobs].where(id: job[:id]).update(
+            status: 'failed',
+            last_error: e.message,
+            updated_at: Sequel.lit('NOW()')
+          )
+          return
+        end
+
+        promote(input_outpoints, change_outpoints, txid)
+        @db[:wallet_broadcast_jobs].where(id: job[:id]).update(
+          status: 'completed',
+          updated_at: Sequel.lit('NOW()')
+        )
+      end
+
+      # Promotes UTXO state after a successful broadcast.
+      #
+      # Marks inputs as +:spent+, change as +:spendable+, and updates the action
+      # status to +completed+. When outpoints are +nil+ (finalize path), UTXO
+      # transitions are skipped.
+      #
+      # @param input_outpoints [Array<String>, nil]
+      # @param change_outpoints [Array<String>, nil]
+      # @param txid [String, nil]
+      def promote(input_outpoints, change_outpoints, txid)
+        Array(input_outpoints).each { |op| @storage.update_output_state(op, :spent) }
+        Array(change_outpoints).each { |op| @storage.update_output_state(op, :spendable) }
+        @storage.update_action_status(txid, 'completed') if txid
+      end
+
+      # Rolls back wallet state after a failed broadcast.
+      #
+      # Releases locked inputs (only those matching +fund_ref+), deletes phantom
+      # change outputs, and marks the action as +failed+.
+      #
+      # @param input_outpoints [Array<String>]
+      # @param change_outpoints [Array<String>]
+      # @param txid [String, nil]
+      # @param fund_ref [String] fund reference used when locking inputs
+      def rollback(input_outpoints, change_outpoints, txid, fund_ref)
+        Array(input_outpoints).each do |op|
+          outputs = @storage.find_outputs({ outpoint: op, include_spent: true, limit: 1, offset: 0 })
+          next if outputs.empty?
+          next unless outputs.first[:pending_reference] == fund_ref
+
+          @storage.update_output_state(op, :spendable)
+        end
+        Array(change_outpoints).each { |op| @storage.delete_output(op) }
+        @storage.update_action_status(txid, 'failed') if txid
+      end
+    end
+  end
+end

data/lib/bsv/wallet_postgres/version.rb

@@ -2,6 +2,6 @@
 
 module BSV
   module WalletPostgres
-    VERSION = '0.3.1'
+    VERSION = '0.4.0'
   end
 end

data/lib/bsv/wallet_postgres.rb

@@ -8,5 +8,6 @@ module BSV
 
   module Wallet
     autoload :PostgresStore, 'bsv/wallet_postgres/postgres_store'
+    autoload :SolidQueueAdapter, 'bsv/wallet_postgres/solid_queue_adapter'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bsv-wallet-postgres
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Simon Bettison
@@ -72,7 +72,9 @@ files:
 - lib/bsv/wallet_postgres/migrations/003_add_wallet_settings.rb
 - lib/bsv/wallet_postgres/migrations/004_add_pending_metadata.rb
 - lib/bsv/wallet_postgres/migrations/005_add_txid_unique_index.rb
+- lib/bsv/wallet_postgres/migrations/006_create_broadcast_jobs.rb
 - lib/bsv/wallet_postgres/postgres_store.rb
+- lib/bsv/wallet_postgres/solid_queue_adapter.rb
 - lib/bsv/wallet_postgres/version.rb
 homepage: https://github.com/sgbett/bsv-ruby-sdk
 licenses: