@aztec/validator-ha-signer 0.0.1-commit.001888fc

package/src/factory.ts

@@ -0,0 +1,139 @@
+/**
+ * Factory functions for creating validator HA signers
+ */
+import { DateProvider } from '@aztec/foundation/timer';
+import { createStore } from '@aztec/kv-store/lmdb-v2';
+import type { LocalSignerConfig, ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
+import { getTelemetryClient } from '@aztec/telemetry-client';
+
+import { Pool } from 'pg';
+
+import { LmdbSlashingProtectionDatabase } from './db/lmdb.js';
+import { PostgresSlashingProtectionDatabase } from './db/postgres.js';
+import { HASignerMetrics } from './metrics.js';
+import type { CreateHASignerDeps, CreateLocalSignerWithProtectionDeps, SlashingProtectionDatabase } from './types.js';
+import { ValidatorHASigner } from './validator_ha_signer.js';
+
+/**
+ * Create a validator HA signer with PostgreSQL backend
+ *
+ * After creating the signer, call `signer.start()` to begin background
+ * cleanup tasks. Call `signer.stop()` during graceful shutdown.
+ *
+ * Example with manual migrations (recommended for production):
+ * ```bash
+ * # Run migrations separately
+ * yarn migrate:up
+ * ```
+ *
+ * ```typescript
+ * const { signer, db } = await createHASigner({
+ *   databaseUrl: process.env.DATABASE_URL,
+ *   nodeId: 'validator-node-1',
+ *   pollingIntervalMs: 100,
+ *   signingTimeoutMs: 3000,
+ * });
+ * signer.start(); // Start background cleanup
+ *
+ * // ... use signer ...
+ *
+ * await signer.stop(); // On shutdown
+ * ```
+ *
+ * Note: Migrations must be run separately using `aztec migrate-ha-db up` before
+ * creating the signer. The factory will verify the schema is initialized via `db.initialize()`.
+ *
+ * @param config - Configuration for the HA signer
+ * @param deps - Optional dependencies (e.g., for testing)
+ * @returns An object containing the signer and database instances
+ */
+export async function createHASigner(
+  config: ValidatorHASignerConfig,
+  deps?: CreateHASignerDeps,
+): Promise<{
+  signer: ValidatorHASigner;
+  db: SlashingProtectionDatabase;
+}> {
+  const { databaseUrl, poolMaxCount, poolMinCount, poolIdleTimeoutMs, poolConnectionTimeoutMs, ...signerConfig } =
+    config;
+
+  if (!databaseUrl) {
+    throw new Error('databaseUrl is required for createHASigner');
+  }
+
+  const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
+  const dateProvider = deps?.dateProvider ?? new DateProvider();
+
+  // Create connection pool (or use provided pool)
+  let pool: Pool;
+  if (!deps?.pool) {
+    pool = new Pool({
+      connectionString: databaseUrl,
+      max: poolMaxCount ?? 10,
+      min: poolMinCount ?? 0,
+      idleTimeoutMillis: poolIdleTimeoutMs ?? 10_000,
+      connectionTimeoutMillis: poolConnectionTimeoutMs ?? 0,
+    });
+  } else {
+    pool = deps.pool;
+  }
+
+  // Create database instance
+  const db = new PostgresSlashingProtectionDatabase(pool);
+
+  // Verify database schema is initialized and version matches
+  await db.initialize();
+
+  // Create metrics
+  const metrics = new HASignerMetrics(telemetryClient, signerConfig.nodeId);
+
+  // Create signer
+  const signer = new ValidatorHASigner(db, signerConfig, { metrics, dateProvider });
+
+  return { signer, db };
+}
+
+/**
+ * Create a local (single-node) signing protection signer backed by LMDB.
+ *
+ * This provides double-signing protection for nodes that are NOT running in a
+ * high-availability (multi-node) setup. It prevents a proposer from sending two
+ * proposals for the same slot if the node crashes and restarts mid-proposal.
+ *
+ * When `config.dataDirectory` is set, the protection database is persisted to disk
+ * and survives crashes/restarts. When unset, an ephemeral in-memory store is
+ * used which protects within a single run but not across restarts.
+ *
+ * @param config - Local signer config
+ * @param deps - Optional dependencies (telemetry, date provider).
+ * @returns An object containing the signer and database instances.
+ */
+export async function createLocalSignerWithProtection(
+  config: LocalSignerConfig,
+  deps?: CreateLocalSignerWithProtectionDeps,
+): Promise<{
+  signer: ValidatorHASigner;
+  db: SlashingProtectionDatabase;
+}> {
+  const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
+  const dateProvider = deps?.dateProvider ?? new DateProvider();
+
+  const kvStore = await createStore('signing-protection', LmdbSlashingProtectionDatabase.SCHEMA_VERSION, {
+    dataDirectory: config.dataDirectory,
+    dataStoreMapSizeKb: config.signingProtectionMapSizeKb ?? config.dataStoreMapSizeKb,
+    l1Contracts: config.l1Contracts,
+  });
+
+  const db = new LmdbSlashingProtectionDatabase(kvStore, dateProvider);
+
+  const signerConfig = {
+    ...config,
+    nodeId: config.nodeId || 'local',
+  };
+
+  const metrics = new HASignerMetrics(telemetryClient, signerConfig.nodeId, 'LocalSigningProtectionMetrics');
+
+  const signer = new ValidatorHASigner(db, signerConfig, { metrics, dateProvider });
+
+  return { signer, db };
+}

package/src/metrics.ts

@@ -0,0 +1,138 @@
+import {
+  Attributes,
+  type Histogram,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+  createUpDownCounterWithDefault,
+} from '@aztec/telemetry-client';
+
+export type HACleanupType = 'stuck' | 'old' | 'outdated_rollup';
+
+/**
+ * Metrics for HA signer tracking signing operations, lock acquisition, and cleanup.
+ */
+export class HASignerMetrics {
+  // Signing lifecycle metrics
+  private signingDuration: Histogram;
+  private signingSuccessCount: UpDownCounter;
+  private dutyAlreadySignedCount: UpDownCounter;
+  private slashingProtectionCount: UpDownCounter;
+  private signingErrorCount: UpDownCounter;
+
+  // Lock acquisition metrics
+  private lockAcquiredCount: UpDownCounter;
+
+  // Cleanup metrics
+  private cleanupStuckDutiesCount: UpDownCounter;
+  private cleanupOldDutiesCount: UpDownCounter;
+  private cleanupOutdatedRollupDutiesCount: UpDownCounter;
+
+  constructor(
+    client: TelemetryClient,
+    private nodeId: string,
+    name = 'HASignerMetrics',
+  ) {
+    const meter = client.getMeter(name);
+
+    // Signing lifecycle
+    this.signingDuration = meter.createHistogram(Metrics.HA_SIGNER_SIGNING_DURATION);
+    this.signingSuccessCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_SUCCESS_COUNT);
+    this.dutyAlreadySignedCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_DUTY_ALREADY_SIGNED_COUNT);
+    this.slashingProtectionCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SLASHING_PROTECTION_COUNT);
+    this.signingErrorCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_ERROR_COUNT);
+
+    // Lock acquisition
+    this.lockAcquiredCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_LOCK_ACQUIRED_COUNT);
+
+    // Cleanup
+    this.cleanupStuckDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_STUCK_DUTIES_COUNT);
+    this.cleanupOldDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_OLD_DUTIES_COUNT);
+    this.cleanupOutdatedRollupDutiesCount = createUpDownCounterWithDefault(
+      meter,
+      Metrics.HA_SIGNER_CLEANUP_OUTDATED_ROLLUP_DUTIES_COUNT,
+    );
+  }
+
+  /**
+   * Record a successful signing operation.
+   * @param dutyType - The type of duty signed
+   * @param durationMs - Duration from start of signWithProtection to completion
+   */
+  public recordSigningSuccess(dutyType: string, durationMs: number): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.signingSuccessCount.add(1, attributes);
+    this.signingDuration.record(durationMs, attributes);
+  }
+
+  /**
+   * Record a DutyAlreadySignedError (expected in HA; another node signed first).
+   * @param dutyType - The type of duty
+   */
+  public recordDutyAlreadySigned(dutyType: string): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.dutyAlreadySignedCount.add(1, attributes);
+  }
+
+  /**
+   * Record a SlashingProtectionError (attempted to sign different data for same duty).
+   * @param dutyType - The type of duty
+   */
+  public recordSlashingProtection(dutyType: string): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.slashingProtectionCount.add(1, attributes);
+  }
+
+  /**
+   * Record a signing function failure (lock will be deleted for retry).
+   * @param dutyType - The type of duty
+   */
+  public recordSigningError(dutyType: string): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.signingErrorCount.add(1, attributes);
+  }
+
+  /**
+   * Record lock acquisition.
+   * @param acquired - Whether a new lock was acquired (true) or existing record found (false)
+   */
+  public recordLockAcquire(acquired: boolean): void {
+    if (acquired) {
+      const attributes = {
+        [Attributes.HA_NODE_ID]: this.nodeId,
+      };
+      this.lockAcquiredCount.add(1, attributes);
+    }
+  }
+
+  /**
+   * Record cleanup metrics.
+   * @param type - Type of cleanup
+   * @param count - Number of duties cleaned up
+   */
+  public recordCleanup(type: HACleanupType, count: number): void {
+    const attributes = {
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+
+    if (type === 'stuck') {
+      this.cleanupStuckDutiesCount.add(count, attributes);
+    } else if (type === 'old') {
+      this.cleanupOldDutiesCount.add(count, attributes);
+    } else if (type === 'outdated_rollup') {
+      this.cleanupOutdatedRollupDutiesCount.add(count, attributes);
+    }
+  }
+}

package/src/migrations.ts

@@ -0,0 +1,75 @@
+/**
+ * Programmatic migration runner
+ */
+import { createLogger } from '@aztec/foundation/log';
+
+import { readdirSync } from 'fs';
+import { runner } from 'node-pg-migrate';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+export interface RunMigrationsOptions {
+  /** Migration direction ('up' to apply, 'down' to rollback). Defaults to 'up'. */
+  direction?: 'up' | 'down';
+  /** Enable verbose output. Defaults to false. */
+  verbose?: boolean;
+}
+
+/**
+ * Run database migrations programmatically
+ *
+ * @param databaseUrl - PostgreSQL connection string
+ * @param options - Migration options (direction, verbose)
+ * @returns Array of applied migration names
+ */
+export async function runMigrations(databaseUrl: string, options: RunMigrationsOptions = {}): Promise<string[]> {
+  const direction = options.direction ?? 'up';
+  const verbose = options.verbose ?? false;
+
+  const log = createLogger('validator-ha-signer:migrations');
+
+  const migrationsDir = join(__dirname, 'db', 'migrations');
+
+  try {
+    log.info(`Running migrations ${direction}...`);
+
+    // Filter out .d.ts and .d.ts.map files - node-pg-migrate only needs .js files
+    const migrationFiles = readdirSync(migrationsDir);
+    const jsMigrationFiles = migrationFiles.filter(
+      file => file.endsWith('.js') && !file.endsWith('.d.ts') && !file.endsWith('.d.ts.map'),
+    );
+
+    if (jsMigrationFiles.length === 0) {
+      log.info('No migration files found');
+      return [];
+    }
+
+    const appliedMigrations = await runner({
+      databaseUrl,
+      dir: migrationsDir,
+      direction,
+      migrationsTable: 'pgmigrations',
+      count: direction === 'down' ? 1 : Infinity,
+      verbose,
+      log: msg => (verbose ? log.info(msg) : log.debug(msg)),
+      // Ignore TypeScript declaration files - node-pg-migrate will try to import them otherwise
+      ignorePattern: '.*\\.d\\.(ts|js)$|.*\\.d\\.ts\\.map$',
+    });
+
+    if (appliedMigrations.length === 0) {
+      log.info('No migrations to apply - schema is up to date');
+    } else {
+      log.info(`Applied ${appliedMigrations.length} migration(s)`, {
+        migrations: appliedMigrations.map(m => m.name),
+      });
+    }
+
+    return appliedMigrations.map(m => m.name);
+  } catch (error: any) {
+    log.error('Migration failed', error);
+    throw error;
+  }
+}

package/src/slashing_protection_service.ts

@@ -0,0 +1,308 @@
+/**
+ * Slashing Protection Service
+ *
+ * Provides distributed locking and slashing protection for validator duties.
+ * Uses an external database to coordinate across multiple validator nodes.
+ */
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { RunningPromise } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { BaseSignerConfig } from '@aztec/stdlib/ha-signing';
+
+import {
+  type CheckAndRecordParams,
+  type DeleteDutyParams,
+  DutyStatus,
+  type RecordSuccessParams,
+  getBlockIndexFromDutyIdentifier,
+} from './db/types.js';
+import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
+import type { HASignerMetrics } from './metrics.js';
+import type { SlashingProtectionDatabase } from './types.js';
+
+export interface SlashingProtectionServiceDeps {
+  metrics: HASignerMetrics;
+  dateProvider: DateProvider;
+}
+
+/**
+ * Slashing Protection Service
+ *
+ * This service ensures that a validator only signs one block/attestation per slot,
+ * even when running multiple redundant nodes (HA setup).
+ *
+ * All nodes in the HA setup try to sign - the first one wins, others get
+ * DutyAlreadySignedError (normal) or SlashingProtectionError (if different data).
+ *
+ * Flow:
+ * 1. checkAndRecord() - Atomically try to acquire lock via tryInsertOrGetExisting
+ * 2. Caller performs the signing operation
+ * 3. recordSuccess() - Update to 'signed' status with signature
+ *    OR deleteDuty() - Delete the record to allow retry
+ */
+export class SlashingProtectionService {
+  private readonly log: Logger;
+  private readonly pollingIntervalMs: number;
+  private readonly signingTimeoutMs: number;
+  private readonly maxStuckDutiesAgeMs: number;
+
+  private readonly metrics: HASignerMetrics;
+  private readonly dateProvider: DateProvider;
+
+  private cleanupRunningPromise: RunningPromise;
+  private lastOldDutiesCleanupAtMs?: number;
+
+  constructor(
+    private readonly db: SlashingProtectionDatabase,
+    private readonly config: BaseSignerConfig,
+    deps: SlashingProtectionServiceDeps,
+  ) {
+    this.log = createLogger('slashing-protection');
+    this.pollingIntervalMs = config.pollingIntervalMs;
+    this.signingTimeoutMs = config.signingTimeoutMs;
+    // Default to 144s (2x 72s Aztec slot duration) if not explicitly configured
+    this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
+
+    this.cleanupRunningPromise = new RunningPromise(this.cleanup.bind(this), this.log, this.maxStuckDutiesAgeMs);
+    this.metrics = deps.metrics;
+    this.dateProvider = deps.dateProvider;
+  }
+
+  /**
+   * Check if a duty can be performed and acquire the lock if so.
+   *
+   * This method uses an atomic insert-or-get operation.
+   * It will:
+   * 1. Try to insert a new record with 'signing' status
+   * 2. If insert succeeds, we acquired the lock - return the lockToken
+   * 3. If a record exists, handle based on status:
+   *    - SIGNED: Throw appropriate error (already signed or slashing protection)
+   *    - SIGNING: Wait and poll until status changes, then handle result
+   *
+   * @returns The lockToken that must be used for recordSuccess/deleteDuty
+   * @throws DutyAlreadySignedError if the duty was already completed
+   * @throws SlashingProtectionError if attempting to sign different data for same slot/duty
+   */
+  async checkAndRecord(params: CheckAndRecordParams): Promise<string> {
+    const { validatorAddress, slot, dutyType, messageHash, nodeId } = params;
+    const startTime = this.dateProvider.now();
+
+    this.log.debug(`Checking duty: ${dutyType} for slot ${slot}`, {
+      validatorAddress: validatorAddress.toString(),
+      nodeId,
+    });
+
+    while (true) {
+      // insert if not present, get existing if present
+      const { isNew, record } = await this.db.tryInsertOrGetExisting(params);
+
+      if (isNew) {
+        // We successfully acquired the lock
+        this.log.verbose(`Acquired lock for duty ${dutyType} at slot ${slot}`, {
+          validatorAddress: validatorAddress.toString(),
+          nodeId,
+        });
+        this.metrics.recordLockAcquire(true);
+        return record.lockToken;
+      }
+
+      // Record already exists - handle based on status
+      if (record.status === DutyStatus.SIGNED) {
+        // Duty was already signed - check if same or different data
+        if (record.messageHash !== messageHash) {
+          this.log.verbose(`Slashing protection triggered for duty ${dutyType} at slot ${slot}`, {
+            validatorAddress: validatorAddress.toString(),
+            existingMessageHash: record.messageHash,
+            attemptedMessageHash: messageHash,
+            existingNodeId: record.nodeId,
+            attemptingNodeId: nodeId,
+          });
+          this.metrics.recordSlashingProtection(dutyType);
+          throw new SlashingProtectionError(
+            slot,
+            dutyType,
+            record.blockIndexWithinCheckpoint,
+            record.messageHash,
+            messageHash,
+            record.nodeId,
+          );
+        }
+        this.metrics.recordDutyAlreadySigned(dutyType);
+        throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
+      } else if (record.status === DutyStatus.SIGNING) {
+        // Another node is currently signing - check for timeout
+        if (this.dateProvider.now() - startTime > this.signingTimeoutMs) {
+          this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
+            validatorAddress: validatorAddress.toString(),
+            timeoutMs: this.signingTimeoutMs,
+            signingNodeId: record.nodeId,
+          });
+          this.metrics.recordDutyAlreadySigned(dutyType);
+          throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, 'unknown (timeout)');
+        }
+
+        // Wait and poll
+        this.log.debug(`Waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
+          validatorAddress: validatorAddress.toString(),
+          signingNodeId: record.nodeId,
+        });
+        await sleep(this.pollingIntervalMs);
+        // Loop continues - next iteration will check status again
+      } else {
+        throw new Error(`Unknown duty status: ${record.status}`);
+      }
+    }
+  }
+
+  /**
+   * Record a successful signing operation.
+   * Updates the duty status to 'signed' and stores the signature.
+   * Only succeeds if the lockToken matches (caller must be the one who created the duty).
+   *
+   * @returns true if the update succeeded, false if token didn't match
+   */
+  async recordSuccess(params: RecordSuccessParams): Promise<boolean> {
+    const { rollupAddress, validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
+    const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
+
+    const success = await this.db.updateDutySigned(
+      rollupAddress,
+      validatorAddress,
+      slot,
+      dutyType,
+      signature.toString(),
+      lockToken,
+      blockIndexWithinCheckpoint,
+    );
+
+    if (success) {
+      this.log.verbose(`Recorded successful signing for duty ${dutyType} at slot ${slot}`, {
+        validatorAddress: validatorAddress.toString(),
+        nodeId,
+      });
+    } else {
+      this.log.warn(`Failed to record successful signing for duty ${dutyType} at slot ${slot}: invalid token`, {
+        validatorAddress: validatorAddress.toString(),
+        nodeId,
+      });
+    }
+
+    return success;
+  }
+
+  /**
+   * Delete a duty record after a failed signing operation.
+   * Removes the record to allow another node/attempt to retry.
+   * Only succeeds if the lockToken matches (caller must be the one who created the duty).
+   *
+   * @returns true if the delete succeeded, false if token didn't match
+   */
+  async deleteDuty(params: DeleteDutyParams): Promise<boolean> {
+    const { rollupAddress, validatorAddress, slot, dutyType, lockToken } = params;
+    const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
+
+    const success = await this.db.deleteDuty(
+      rollupAddress,
+      validatorAddress,
+      slot,
+      dutyType,
+      lockToken,
+      blockIndexWithinCheckpoint,
+    );
+
+    if (success) {
+      this.log.info(`Deleted duty ${dutyType} at slot ${slot} to allow retry`, {
+        validatorAddress: validatorAddress.toString(),
+      });
+    } else {
+      this.log.warn(`Failed to delete duty ${dutyType} at slot ${slot}: invalid token`, {
+        validatorAddress: validatorAddress.toString(),
+      });
+    }
+
+    return success;
+  }
+
+  /**
+   * Get the node ID for this service
+   */
+  get nodeId(): string {
+    return this.config.nodeId;
+  }
+
+  /**
+   * Start running tasks.
+   * Cleanup runs immediately on start to recover from any previous crashes.
+   */
+  /**
+   * Start the background cleanup task.
+   * Also performs one-time cleanup of duties with outdated rollup addresses.
+   */
+  async start() {
+    // One-time cleanup at startup: remove duties from previous rollup versions
+    const numOutdatedRollupDuties = await this.db.cleanupOutdatedRollupDuties(this.config.l1Contracts.rollupAddress);
+    if (numOutdatedRollupDuties > 0) {
+      this.log.info(`Cleaned up ${numOutdatedRollupDuties} duties with outdated rollup address at startup`, {
+        currentRollupAddress: this.config.l1Contracts.rollupAddress.toString(),
+      });
+      this.metrics.recordCleanup('outdated_rollup', numOutdatedRollupDuties);
+    }
+
+    this.cleanupRunningPromise.start();
+    this.log.info('Slashing protection service started', { nodeId: this.config.nodeId });
+  }
+
+  /**
+   * Stop the background cleanup task.
+   */
+  async stop() {
+    await this.cleanupRunningPromise.stop();
+    this.log.info('Slashing protection service stopped', { nodeId: this.config.nodeId });
+  }
+
+  /**
+   * Close the database connection.
+   * Should be called after stop() during graceful shutdown.
+   */
+  async close() {
+    await this.db.close();
+    this.log.info('Slashing protection database connection closed');
+  }
+
+  /**
+   * Periodic cleanup of stuck duties and optionally old signed duties.
+   * Runs in the background via RunningPromise.
+   */
+  private async cleanup() {
+    // 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status)
+    const numStuckDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
+    if (numStuckDuties > 0) {
+      this.log.verbose(`Cleaned up ${numStuckDuties} stuck duties`, {
+        nodeId: this.config.nodeId,
+        maxStuckDutiesAgeMs: this.maxStuckDutiesAgeMs,
+      });
+      this.metrics.recordCleanup('stuck', numStuckDuties);
+    }
+
+    // 2. Clean up old signed duties if configured
+    // we shouldn't run this as often as stuck duty cleanup.
+    if (this.config.cleanupOldDutiesAfterHours !== undefined) {
+      const maxAgeMs = this.config.cleanupOldDutiesAfterHours * 60 * 60 * 1000;
+      const nowMs = this.dateProvider.now();
+      const shouldRun =
+        this.lastOldDutiesCleanupAtMs === undefined || nowMs - this.lastOldDutiesCleanupAtMs >= maxAgeMs;
+      if (shouldRun) {
+        const numOldDuties = await this.db.cleanupOldDuties(maxAgeMs);
+        this.lastOldDutiesCleanupAtMs = nowMs;
+        if (numOldDuties > 0) {
+          this.log.verbose(`Cleaned up ${numOldDuties} old signed duties`, {
+            cleanupOldDutiesAfterHours: this.config.cleanupOldDutiesAfterHours,
+            maxAgeMs,
+          });
+          this.metrics.recordCleanup('old', numOldDuties);
+        }
+      }
+    }
+  }
+}