@aztec/validator-ha-signer 0.0.1-commit.7d4e6cd → 0.0.1-commit.8227e42

package/src/factory.ts

@@ -1,10 +1,14 @@
 /**
  * Factory functions for creating validator HA signers
  */
+import { DateProvider } from '@aztec/foundation/timer';
+import type { ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
+import { getTelemetryClient } from '@aztec/telemetry-client';
+
 import { Pool } from 'pg';
 
-import type { CreateHASignerConfig } from './config.js';
 import { PostgresSlashingProtectionDatabase } from './db/postgres.js';
+import { HASignerMetrics } from './metrics.js';
 import type { CreateHASignerDeps, SlashingProtectionDatabase } from './types.js';
 import { ValidatorHASigner } from './validator_ha_signer.js';
 
@@ -23,7 +27,7 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
  * ```typescript
  * const { signer, db } = await createHASigner({
  *   databaseUrl: process.env.DATABASE_URL,
- *   enabled: true,
+ *   haSigningEnabled: true,
  *   nodeId: 'validator-node-1',
  *   pollingIntervalMs: 100,
  *   signingTimeoutMs: 3000,
@@ -35,23 +39,15 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
  * await signer.stop(); // On shutdown
  * ```
  *
- * Example with automatic migrations (simpler for dev/testing):
- * ```typescript
- * const { signer, db } = await createHASigner({
- *   databaseUrl: process.env.DATABASE_URL,
- *   enabled: true,
- *   nodeId: 'validator-node-1',
- *   runMigrations: true, // Auto-run migrations on startup
- * });
- * signer.start();
- * ```
+ * Note: Migrations must be run separately using `aztec migrate-ha-db up` before
+ * creating the signer. The factory will verify the schema is initialized via `db.initialize()`.
  *
  * @param config - Configuration for the HA signer
  * @param deps - Optional dependencies (e.g., for testing)
  * @returns An object containing the signer and database instances
  */
 export async function createHASigner(
-  config: CreateHASignerConfig,
+  config: ValidatorHASignerConfig,
   deps?: CreateHASignerDeps,
 ): Promise<{
   signer: ValidatorHASigner;
@@ -60,6 +56,13 @@ export async function createHASigner(
   const { databaseUrl, poolMaxCount, poolMinCount, poolIdleTimeoutMs, poolConnectionTimeoutMs, ...signerConfig } =
     config;
 
+  if (!databaseUrl) {
+    throw new Error('databaseUrl is required for createHASigner');
+  }
+
+  const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
+  const dateProvider = deps?.dateProvider ?? new DateProvider();
+
   // Create connection pool (or use provided pool)
   let pool: Pool;
   if (!deps?.pool) {
@@ -80,8 +83,11 @@ export async function createHASigner(
   // Verify database schema is initialized and version matches
   await db.initialize();
 
+  // Create metrics
+  const metrics = new HASignerMetrics(telemetryClient, signerConfig.nodeId);
+
   // Create signer
-  const signer = new ValidatorHASigner(db, { ...signerConfig, databaseUrl });
+  const signer = new ValidatorHASigner(db, { ...signerConfig, databaseUrl }, { metrics, dateProvider });
 
   return { signer, db };
 }

package/src/metrics.ts

@@ -0,0 +1,138 @@
+import {
+  Attributes,
+  type Histogram,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+  createUpDownCounterWithDefault,
+} from '@aztec/telemetry-client';
+
+export type HACleanupType = 'stuck' | 'old' | 'outdated_rollup';
+
+/**
+ * Metrics for HA signer tracking signing operations, lock acquisition, and cleanup.
+ */
+export class HASignerMetrics {
+  // Signing lifecycle metrics
+  private signingDuration: Histogram;
+  private signingSuccessCount: UpDownCounter;
+  private dutyAlreadySignedCount: UpDownCounter;
+  private slashingProtectionCount: UpDownCounter;
+  private signingErrorCount: UpDownCounter;
+
+  // Lock acquisition metrics
+  private lockAcquiredCount: UpDownCounter;
+
+  // Cleanup metrics
+  private cleanupStuckDutiesCount: UpDownCounter;
+  private cleanupOldDutiesCount: UpDownCounter;
+  private cleanupOutdatedRollupDutiesCount: UpDownCounter;
+
+  constructor(
+    client: TelemetryClient,
+    private nodeId: string,
+    name = 'HASignerMetrics',
+  ) {
+    const meter = client.getMeter(name);
+
+    // Signing lifecycle
+    this.signingDuration = meter.createHistogram(Metrics.HA_SIGNER_SIGNING_DURATION);
+    this.signingSuccessCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_SUCCESS_COUNT);
+    this.dutyAlreadySignedCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_DUTY_ALREADY_SIGNED_COUNT);
+    this.slashingProtectionCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SLASHING_PROTECTION_COUNT);
+    this.signingErrorCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_ERROR_COUNT);
+
+    // Lock acquisition
+    this.lockAcquiredCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_LOCK_ACQUIRED_COUNT);
+
+    // Cleanup
+    this.cleanupStuckDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_STUCK_DUTIES_COUNT);
+    this.cleanupOldDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_OLD_DUTIES_COUNT);
+    this.cleanupOutdatedRollupDutiesCount = createUpDownCounterWithDefault(
+      meter,
+      Metrics.HA_SIGNER_CLEANUP_OUTDATED_ROLLUP_DUTIES_COUNT,
+    );
+  }
+
+  /**
+   * Record a successful signing operation.
+   * @param dutyType - The type of duty signed
+   * @param durationMs - Duration from start of signWithProtection to completion
+   */
+  public recordSigningSuccess(dutyType: string, durationMs: number): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.signingSuccessCount.add(1, attributes);
+    this.signingDuration.record(durationMs, attributes);
+  }
+
+  /**
+   * Record a DutyAlreadySignedError (expected in HA; another node signed first).
+   * @param dutyType - The type of duty
+   */
+  public recordDutyAlreadySigned(dutyType: string): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.dutyAlreadySignedCount.add(1, attributes);
+  }
+
+  /**
+   * Record a SlashingProtectionError (attempted to sign different data for same duty).
+   * @param dutyType - The type of duty
+   */
+  public recordSlashingProtection(dutyType: string): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.slashingProtectionCount.add(1, attributes);
+  }
+
+  /**
+   * Record a signing function failure (lock will be deleted for retry).
+   * @param dutyType - The type of duty
+   */
+  public recordSigningError(dutyType: string): void {
+    const attributes = {
+      [Attributes.HA_DUTY_TYPE]: dutyType,
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+    this.signingErrorCount.add(1, attributes);
+  }
+
+  /**
+   * Record lock acquisition.
+   * @param acquired - Whether a new lock was acquired (true) or existing record found (false)
+   */
+  public recordLockAcquire(acquired: boolean): void {
+    if (acquired) {
+      const attributes = {
+        [Attributes.HA_NODE_ID]: this.nodeId,
+      };
+      this.lockAcquiredCount.add(1, attributes);
+    }
+  }
+
+  /**
+   * Record cleanup metrics.
+   * @param type - Type of cleanup
+   * @param count - Number of duties cleaned up
+   */
+  public recordCleanup(type: HACleanupType, count: number): void {
+    const attributes = {
+      [Attributes.HA_NODE_ID]: this.nodeId,
+    };
+
+    if (type === 'stuck') {
+      this.cleanupStuckDutiesCount.add(count, attributes);
+    } else if (type === 'old') {
+      this.cleanupOldDutiesCount.add(count, attributes);
+    } else if (type === 'outdated_rollup') {
+      this.cleanupOutdatedRollupDutiesCount.add(count, attributes);
+    }
+  }
+}

package/src/migrations.ts

@@ -3,6 +3,7 @@
  */
 import { createLogger } from '@aztec/foundation/log';
 
+import { readdirSync } from 'fs';
 import { runner } from 'node-pg-migrate';
 import { dirname, join } from 'path';
 import { fileURLToPath } from 'url';
@@ -30,17 +31,32 @@ export async function runMigrations(databaseUrl: string, options: RunMigrationsO
 
   const log = createLogger('validator-ha-signer:migrations');
 
+  const migrationsDir = join(__dirname, 'db', 'migrations');
+
   try {
     log.info(`Running migrations ${direction}...`);
 
+    // Filter out .d.ts and .d.ts.map files - node-pg-migrate only needs .js files
+    const migrationFiles = readdirSync(migrationsDir);
+    const jsMigrationFiles = migrationFiles.filter(
+      file => file.endsWith('.js') && !file.endsWith('.d.ts') && !file.endsWith('.d.ts.map'),
+    );
+
+    if (jsMigrationFiles.length === 0) {
+      log.info('No migration files found');
+      return [];
+    }
+
     const appliedMigrations = await runner({
       databaseUrl,
-      dir: join(__dirname, 'db', 'migrations'),
+      dir: migrationsDir,
       direction,
       migrationsTable: 'pgmigrations',
       count: direction === 'down' ? 1 : Infinity,
       verbose,
       log: msg => (verbose ? log.info(msg) : log.debug(msg)),
+      // Ignore TypeScript declaration files - node-pg-migrate will try to import them otherwise
+      ignorePattern: '.*\\.d\\.(ts|js)$|.*\\.d\\.ts\\.map$',
     });
 
     if (appliedMigrations.length === 0) {

package/src/slashing_protection_service.ts

@@ -7,10 +7,24 @@
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/promise';
 import { sleep } from '@aztec/foundation/sleep';
+import type { DateProvider } from '@aztec/foundation/timer';
+import type { ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
 
-import { type CheckAndRecordParams, type DeleteDutyParams, DutyStatus, type RecordSuccessParams } from './db/types.js';
+import {
+  type CheckAndRecordParams,
+  type DeleteDutyParams,
+  DutyStatus,
+  type RecordSuccessParams,
+  getBlockIndexFromDutyIdentifier,
+} from './db/types.js';
 import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
-import type { SlashingProtectionConfig, SlashingProtectionDatabase } from './types.js';
+import type { HASignerMetrics } from './metrics.js';
+import type { SlashingProtectionDatabase } from './types.js';
+
+export interface SlashingProtectionServiceDeps {
+  metrics: HASignerMetrics;
+  dateProvider: DateProvider;
+}
 
 /**
  * Slashing Protection Service
@@ -31,22 +45,28 @@ export class SlashingProtectionService {
   private readonly log: Logger;
   private readonly pollingIntervalMs: number;
   private readonly signingTimeoutMs: number;
+  private readonly maxStuckDutiesAgeMs: number;
+
+  private readonly metrics: HASignerMetrics;
+  private readonly dateProvider: DateProvider;
 
   private cleanupRunningPromise: RunningPromise;
+  private lastOldDutiesCleanupAtMs?: number;
 
   constructor(
     private readonly db: SlashingProtectionDatabase,
-    private readonly config: SlashingProtectionConfig,
+    private readonly config: ValidatorHASignerConfig,
+    deps: SlashingProtectionServiceDeps,
   ) {
     this.log = createLogger('slashing-protection');
     this.pollingIntervalMs = config.pollingIntervalMs;
     this.signingTimeoutMs = config.signingTimeoutMs;
+    // Default to 144s (2x 72s Aztec slot duration) if not explicitly configured
+    this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
 
-    this.cleanupRunningPromise = new RunningPromise(
-      this.cleanupStuckDuties.bind(this),
-      this.log,
-      this.config.maxStuckDutiesAgeMs,
-    );
+    this.cleanupRunningPromise = new RunningPromise(this.cleanup.bind(this), this.log, this.maxStuckDutiesAgeMs);
+    this.metrics = deps.metrics;
+    this.dateProvider = deps.dateProvider;
   }
 
   /**
@@ -58,7 +78,6 @@ export class SlashingProtectionService {
    * 2. If insert succeeds, we acquired the lock - return the lockToken
    * 3. If a record exists, handle based on status:
    *    - SIGNED: Throw appropriate error (already signed or slashing protection)
-   *    - FAILED: Delete the failed record
    *    - SIGNING: Wait and poll until status changes, then handle result
    *
    * @returns The lockToken that must be used for recordSuccess/deleteDuty
@@ -67,7 +86,7 @@ export class SlashingProtectionService {
    */
   async checkAndRecord(params: CheckAndRecordParams): Promise<string> {
     const { validatorAddress, slot, dutyType, messageHash, nodeId } = params;
-    const startTime = Date.now();
+    const startTime = this.dateProvider.now();
 
     this.log.debug(`Checking duty: ${dutyType} for slot ${slot}`, {
       validatorAddress: validatorAddress.toString(),
@@ -84,6 +103,7 @@ export class SlashingProtectionService {
           validatorAddress: validatorAddress.toString(),
           nodeId,
         });
+        this.metrics.recordLockAcquire(true);
         return record.lockToken;
       }
 
@@ -98,18 +118,28 @@ export class SlashingProtectionService {
             existingNodeId: record.nodeId,
             attemptingNodeId: nodeId,
           });
-          throw new SlashingProtectionError(slot, dutyType, record.messageHash, messageHash);
+          this.metrics.recordSlashingProtection(dutyType);
+          throw new SlashingProtectionError(
+            slot,
+            dutyType,
+            record.blockIndexWithinCheckpoint,
+            record.messageHash,
+            messageHash,
+            record.nodeId,
+          );
         }
-        throw new DutyAlreadySignedError(slot, dutyType, record.nodeId);
+        this.metrics.recordDutyAlreadySigned(dutyType);
+        throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
       } else if (record.status === DutyStatus.SIGNING) {
         // Another node is currently signing - check for timeout
-        if (Date.now() - startTime > this.signingTimeoutMs) {
+        if (this.dateProvider.now() - startTime > this.signingTimeoutMs) {
           this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
             validatorAddress: validatorAddress.toString(),
             timeoutMs: this.signingTimeoutMs,
             signingNodeId: record.nodeId,
           });
-          throw new DutyAlreadySignedError(slot, dutyType, 'unknown (timeout)');
+          this.metrics.recordDutyAlreadySigned(dutyType);
+          throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, 'unknown (timeout)');
         }
 
         // Wait and poll
@@ -133,9 +163,18 @@ export class SlashingProtectionService {
    * @returns true if the update succeeded, false if token didn't match
    */
   async recordSuccess(params: RecordSuccessParams): Promise<boolean> {
-    const { validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
+    const { rollupAddress, validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
+    const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
 
-    const success = await this.db.updateDutySigned(validatorAddress, slot, dutyType, signature.toString(), lockToken);
+    const success = await this.db.updateDutySigned(
+      rollupAddress,
+      validatorAddress,
+      slot,
+      dutyType,
+      signature.toString(),
+      lockToken,
+      blockIndexWithinCheckpoint,
+    );
 
     if (success) {
       this.log.info(`Recorded successful signing for duty ${dutyType} at slot ${slot}`, {
@@ -160,9 +199,17 @@ export class SlashingProtectionService {
    * @returns true if the delete succeeded, false if token didn't match
    */
   async deleteDuty(params: DeleteDutyParams): Promise<boolean> {
-    const { validatorAddress, slot, dutyType, lockToken } = params;
+    const { rollupAddress, validatorAddress, slot, dutyType, lockToken } = params;
+    const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
 
-    const success = await this.db.deleteDuty(validatorAddress, slot, dutyType, lockToken);
+    const success = await this.db.deleteDuty(
+      rollupAddress,
+      validatorAddress,
+      slot,
+      dutyType,
+      lockToken,
+      blockIndexWithinCheckpoint,
+    );
 
     if (success) {
       this.log.info(`Deleted duty ${dutyType} at slot ${slot} to allow retry`, {
@@ -188,7 +235,20 @@ export class SlashingProtectionService {
    * Start running tasks.
    * Cleanup runs immediately on start to recover from any previous crashes.
    */
-  start() {
+  /**
+   * Start the background cleanup task.
+   * Also performs one-time cleanup of duties with outdated rollup addresses.
+   */
+  async start() {
+    // One-time cleanup at startup: remove duties from previous rollup versions
+    const numOutdatedRollupDuties = await this.db.cleanupOutdatedRollupDuties(this.config.l1Contracts.rollupAddress);
+    if (numOutdatedRollupDuties > 0) {
+      this.log.info(`Cleaned up ${numOutdatedRollupDuties} duties with outdated rollup address at startup`, {
+        currentRollupAddress: this.config.l1Contracts.rollupAddress.toString(),
+      });
+      this.metrics.recordCleanup('outdated_rollup', numOutdatedRollupDuties);
+    }
+
     this.cleanupRunningPromise.start();
     this.log.info('Slashing protection service started', { nodeId: this.config.nodeId });
   }
@@ -202,15 +262,47 @@ export class SlashingProtectionService {
   }
 
   /**
-   * Cleanup own stuck duties
+   * Close the database connection.
+   * Should be called after stop() during graceful shutdown.
+   */
+  async close() {
+    await this.db.close();
+    this.log.info('Slashing protection database connection closed');
+  }
+
+  /**
+   * Periodic cleanup of stuck duties and optionally old signed duties.
+   * Runs in the background via RunningPromise.
    */
-  private async cleanupStuckDuties() {
-    const numDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.config.maxStuckDutiesAgeMs);
-    if (numDuties > 0) {
-      this.log.info(`Cleaned up ${numDuties} stuck duties`, {
+  private async cleanup() {
+    // 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status)
+    const numStuckDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
+    if (numStuckDuties > 0) {
+      this.log.verbose(`Cleaned up ${numStuckDuties} stuck duties`, {
         nodeId: this.config.nodeId,
-        maxStuckDutiesAgeMs: this.config.maxStuckDutiesAgeMs,
+        maxStuckDutiesAgeMs: this.maxStuckDutiesAgeMs,
       });
+      this.metrics.recordCleanup('stuck', numStuckDuties);
+    }
+
+    // 2. Clean up old signed duties if configured
+    // we shouldn't run this as often as stuck duty cleanup.
+    if (this.config.cleanupOldDutiesAfterHours !== undefined) {
+      const maxAgeMs = this.config.cleanupOldDutiesAfterHours * 60 * 60 * 1000;
+      const nowMs = this.dateProvider.now();
+      const shouldRun =
+        this.lastOldDutiesCleanupAtMs === undefined || nowMs - this.lastOldDutiesCleanupAtMs >= maxAgeMs;
+      if (shouldRun) {
+        const numOldDuties = await this.db.cleanupOldDuties(maxAgeMs);
+        this.lastOldDutiesCleanupAtMs = nowMs;
+        if (numOldDuties > 0) {
+          this.log.verbose(`Cleaned up ${numOldDuties} old signed duties`, {
+            cleanupOldDutiesAfterHours: this.config.cleanupOldDutiesAfterHours,
+            maxAgeMs,
+          });
+          this.metrics.recordCleanup('old', numOldDuties);
+        }
+      }
     }
   }
 }