@aztec/validator-ha-signer 0.0.1-commit.023c3e5

package/src/slashing_protection_service.ts

@@ -0,0 +1,250 @@
+/**
+ * Slashing Protection Service
+ *
+ * Provides distributed locking and slashing protection for validator duties.
+ * Uses an external database to coordinate across multiple validator nodes.
+ */
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import { RunningPromise } from '@aztec/foundation/promise';
+import { sleep } from '@aztec/foundation/sleep';
+
+import {
+  type CheckAndRecordParams,
+  type DeleteDutyParams,
+  DutyStatus,
+  type RecordSuccessParams,
+  getBlockIndexFromDutyIdentifier,
+} from './db/types.js';
+import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
+import type { SlashingProtectionDatabase, ValidatorHASignerConfig } from './types.js';
+
+/**
+ * Slashing Protection Service
+ *
+ * This service ensures that a validator only signs one block/attestation per slot,
+ * even when running multiple redundant nodes (HA setup).
+ *
+ * All nodes in the HA setup try to sign - the first one wins, others get
+ * DutyAlreadySignedError (normal) or SlashingProtectionError (if different data).
+ *
+ * Flow:
+ * 1. checkAndRecord() - Atomically try to acquire lock via tryInsertOrGetExisting
+ * 2. Caller performs the signing operation
+ * 3. recordSuccess() - Update to 'signed' status with signature
+ *    OR deleteDuty() - Delete the record to allow retry
+ */
+export class SlashingProtectionService {
+  private readonly log: Logger;
+  private readonly pollingIntervalMs: number;
+  private readonly signingTimeoutMs: number;
+  private readonly maxStuckDutiesAgeMs: number;
+
+  private cleanupRunningPromise: RunningPromise;
+
+  constructor(
+    private readonly db: SlashingProtectionDatabase,
+    private readonly config: ValidatorHASignerConfig,
+  ) {
+    this.log = createLogger('slashing-protection');
+    this.pollingIntervalMs = config.pollingIntervalMs;
+    this.signingTimeoutMs = config.signingTimeoutMs;
+    // Default to 144s (2x 72s Aztec slot duration) if not explicitly configured
+    this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
+
+    this.cleanupRunningPromise = new RunningPromise(
+      this.cleanupStuckDuties.bind(this),
+      this.log,
+      this.maxStuckDutiesAgeMs,
+    );
+  }
+
+  /**
+   * Check if a duty can be performed and acquire the lock if so.
+   *
+   * This method uses an atomic insert-or-get operation.
+   * It will:
+   * 1. Try to insert a new record with 'signing' status
+   * 2. If insert succeeds, we acquired the lock - return the lockToken
+   * 3. If a record exists, handle based on status:
+   *    - SIGNED: Throw appropriate error (already signed or slashing protection)
+   *    - FAILED: Delete the failed record
+   *    - SIGNING: Wait and poll until status changes, then handle result
+   *
+   * @returns The lockToken that must be used for recordSuccess/deleteDuty
+   * @throws DutyAlreadySignedError if the duty was already completed
+   * @throws SlashingProtectionError if attempting to sign different data for same slot/duty
+   */
+  async checkAndRecord(params: CheckAndRecordParams): Promise<string> {
+    const { validatorAddress, slot, dutyType, messageHash, nodeId } = params;
+    const startTime = Date.now();
+
+    this.log.debug(`Checking duty: ${dutyType} for slot ${slot}`, {
+      validatorAddress: validatorAddress.toString(),
+      nodeId,
+    });
+
+    while (true) {
+      // insert if not present, get existing if present
+      const { isNew, record } = await this.db.tryInsertOrGetExisting(params);
+
+      if (isNew) {
+        // We successfully acquired the lock
+        this.log.info(`Acquired lock for duty ${dutyType} at slot ${slot}`, {
+          validatorAddress: validatorAddress.toString(),
+          nodeId,
+        });
+        return record.lockToken;
+      }
+
+      // Record already exists - handle based on status
+      if (record.status === DutyStatus.SIGNED) {
+        // Duty was already signed - check if same or different data
+        if (record.messageHash !== messageHash) {
+          this.log.verbose(`Slashing protection triggered for duty ${dutyType} at slot ${slot}`, {
+            validatorAddress: validatorAddress.toString(),
+            existingMessageHash: record.messageHash,
+            attemptedMessageHash: messageHash,
+            existingNodeId: record.nodeId,
+            attemptingNodeId: nodeId,
+          });
+          throw new SlashingProtectionError(
+            slot,
+            dutyType,
+            record.blockIndexWithinCheckpoint,
+            record.messageHash,
+            messageHash,
+            record.nodeId,
+          );
+        }
+        throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
+      } else if (record.status === DutyStatus.SIGNING) {
+        // Another node is currently signing - check for timeout
+        if (Date.now() - startTime > this.signingTimeoutMs) {
+          this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
+            validatorAddress: validatorAddress.toString(),
+            timeoutMs: this.signingTimeoutMs,
+            signingNodeId: record.nodeId,
+          });
+          throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, 'unknown (timeout)');
+        }
+
+        // Wait and poll
+        this.log.debug(`Waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
+          validatorAddress: validatorAddress.toString(),
+          signingNodeId: record.nodeId,
+        });
+        await sleep(this.pollingIntervalMs);
+        // Loop continues - next iteration will check status again
+      } else {
+        throw new Error(`Unknown duty status: ${record.status}`);
+      }
+    }
+  }
+
+  /**
+   * Record a successful signing operation.
+   * Updates the duty status to 'signed' and stores the signature.
+   * Only succeeds if the lockToken matches (caller must be the one who created the duty).
+   *
+   * @returns true if the update succeeded, false if token didn't match
+   */
+  async recordSuccess(params: RecordSuccessParams): Promise<boolean> {
+    const { validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
+    const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
+
+    const success = await this.db.updateDutySigned(
+      validatorAddress,
+      slot,
+      dutyType,
+      signature.toString(),
+      lockToken,
+      blockIndexWithinCheckpoint,
+    );
+
+    if (success) {
+      this.log.info(`Recorded successful signing for duty ${dutyType} at slot ${slot}`, {
+        validatorAddress: validatorAddress.toString(),
+        nodeId,
+      });
+    } else {
+      this.log.warn(`Failed to record successful signing for duty ${dutyType} at slot ${slot}: invalid token`, {
+        validatorAddress: validatorAddress.toString(),
+        nodeId,
+      });
+    }
+
+    return success;
+  }
+
+  /**
+   * Delete a duty record after a failed signing operation.
+   * Removes the record to allow another node/attempt to retry.
+   * Only succeeds if the lockToken matches (caller must be the one who created the duty).
+   *
+   * @returns true if the delete succeeded, false if token didn't match
+   */
+  async deleteDuty(params: DeleteDutyParams): Promise<boolean> {
+    const { validatorAddress, slot, dutyType, lockToken } = params;
+    const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
+
+    const success = await this.db.deleteDuty(validatorAddress, slot, dutyType, lockToken, blockIndexWithinCheckpoint);
+
+    if (success) {
+      this.log.info(`Deleted duty ${dutyType} at slot ${slot} to allow retry`, {
+        validatorAddress: validatorAddress.toString(),
+      });
+    } else {
+      this.log.warn(`Failed to delete duty ${dutyType} at slot ${slot}: invalid token`, {
+        validatorAddress: validatorAddress.toString(),
+      });
+    }
+
+    return success;
+  }
+
+  /**
+   * Get the node ID for this service
+   */
+  get nodeId(): string {
+    return this.config.nodeId;
+  }
+
+  /**
+   * Start running tasks.
+   * Cleanup runs immediately on start to recover from any previous crashes.
+   */
+  start() {
+    this.cleanupRunningPromise.start();
+    this.log.info('Slashing protection service started', { nodeId: this.config.nodeId });
+  }
+
+  /**
+   * Stop the background cleanup task.
+   */
+  async stop() {
+    await this.cleanupRunningPromise.stop();
+    this.log.info('Slashing protection service stopped', { nodeId: this.config.nodeId });
+  }
+
+  /**
+   * Close the database connection.
+   * Should be called after stop() during graceful shutdown.
+   */
+  async close() {
+    await this.db.close();
+    this.log.info('Slashing protection database connection closed');
+  }
+
+  /**
+   * Cleanup own stuck duties
+   */
+  private async cleanupStuckDuties() {
+    const numDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
+    if (numDuties > 0) {
+      this.log.info(`Cleaned up ${numDuties} stuck duties`, {
+        nodeId: this.config.nodeId,
+        maxStuckDutiesAgeMs: this.maxStuckDutiesAgeMs,
+      });
+    }
+  }
+}

package/src/test/pglite_pool.ts

@@ -0,0 +1,256 @@
+/**
+ * Vendored pg-compatible Pool/Client wrapper for PGlite.
+ *
+ * Copied from @middle-management/pglite-pg-adapter v0.0.3
+ * https://www.npmjs.com/package/@middle-management/pglite-pg-adapter
+ *
+ * Modifications:
+ * - Converted to ESM and TypeScript
+ * - Uses PGliteInterface instead of PGlite class to avoid TypeScript
+ *   type mismatches from ESM/CJS dual package resolution with private fields
+ * - Simplified rowCount calculation to handle CTEs properly
+ */
+import type { PGliteInterface } from '@electric-sql/pglite';
+import { EventEmitter } from 'events';
+import type { QueryResult, QueryResultRow } from 'pg';
+import { Readable, Writable } from 'stream';
+
+export interface PoolConfig {
+  pglite: PGliteInterface;
+  max?: number;
+  min?: number;
+}
+
+interface ClientConfig {
+  pglite: PGliteInterface;
+  host?: string;
+  port?: number;
+  ssl?: boolean;
+}
+
+export class Client extends EventEmitter {
+  protected pglite: PGliteInterface;
+  protected _connected = false;
+  readonly host: string;
+  readonly port: number;
+  readonly ssl: boolean;
+  readonly connection: object;
+
+  // Stub implementations for pg compatibility
+  readonly copyFrom = (): Writable => new Writable();
+  readonly copyTo = (): Readable => new Readable();
+  readonly pauseDrain = (): void => {};
+  readonly resumeDrain = (): void => {};
+  readonly escapeLiteral = (str: string): string => `'${str.replace(/'/g, "''")}'`;
+  readonly escapeIdentifier = (str: string): string => `"${str.replace(/"/g, '""')}"`;
+  readonly setTypeParser = (): void => {};
+  readonly getTypeParser = (): ((value: string) => unknown) => (value: string) => value;
+
+  constructor(config: ClientConfig) {
+    super();
+    this.pglite = config.pglite;
+    this.host = config.host || 'localhost';
+    this.port = config.port || 5432;
+    this.ssl = typeof config.ssl === 'boolean' ? config.ssl : !!config.ssl;
+    this.connection = {};
+  }
+
+  connect(): Promise<void> {
+    if (this._connected) {
+      return Promise.resolve();
+    }
+    this._connected = true;
+    this.emit('connect');
+    return Promise.resolve();
+  }
+
+  end(): Promise<void> {
+    if (!this._connected) {
+      return Promise.resolve();
+    }
+    this._connected = false;
+    this.emit('end');
+    return Promise.resolve();
+  }
+
+  async query<R extends QueryResultRow = any>(text: string, values?: any[]): Promise<QueryResult<R>> {
+    if (!this._connected) {
+      throw new Error('Client is not connected');
+    }
+    const result = await this.pglite.query<R>(text, values);
+    return this.convertPGliteResult(result);
+  }
+
+  protected convertPGliteResult<R extends QueryResultRow>(result: {
+    rows: R[];
+    fields: Array<{ name: string; dataTypeID: number }>;
+    affectedRows?: number;
+  }): QueryResult<R> {
+    return {
+      command: '',
+      rowCount: 'affectedRows' in result ? (result.affectedRows ?? 0) : result.rows.length,
+      oid: 0,
+      fields: result.fields.map(field => ({
+        name: field.name,
+        tableID: 0,
+        columnID: 0,
+        dataTypeID: field.dataTypeID,
+        dataTypeSize: -1,
+        dataTypeModifier: -1,
+        format: 'text',
+      })),
+      rows: result.rows,
+    };
+  }
+
+  get connected(): boolean {
+    return this._connected;
+  }
+}
+
+export class Pool extends EventEmitter {
+  private clients: PoolClient[] = [];
+  private availableClients: PoolClient[] = [];
+  private waitingQueue: Array<(client: PoolClient) => void> = [];
+  private _ended = false;
+  private pglite: PGliteInterface;
+  private _config: PoolConfig;
+
+  readonly expiredCount = 0;
+  readonly options: PoolConfig;
+
+  constructor(config: PoolConfig) {
+    super();
+    this._config = { max: 10, min: 0, ...config };
+    this.pglite = config.pglite;
+    this.options = config;
+  }
+
+  get totalCount(): number {
+    return this.clients.length;
+  }
+
+  get idleCount(): number {
+    return this.availableClients.length;
+  }
+
+  get waitingCount(): number {
+    return this.waitingQueue.length;
+  }
+
+  get ending(): boolean {
+    return this._ended;
+  }
+
+  get ended(): boolean {
+    return this._ended;
+  }
+
+  connect(): Promise<PoolClient> {
+    if (this._ended) {
+      return Promise.reject(new Error('Pool is ended'));
+    }
+
+    if (this.availableClients.length > 0) {
+      const client = this.availableClients.pop()!;
+      client._markInUse();
+      return Promise.resolve(client);
+    }
+
+    if (this.clients.length < (this._config.max || 10)) {
+      const client = new PoolClient(this.pglite, this);
+      this.clients.push(client);
+      return Promise.resolve(client);
+    }
+
+    return new Promise(resolve => {
+      this.waitingQueue.push(resolve);
+    });
+  }
+
+  async query<R extends QueryResultRow = any>(text: string, values?: any[]): Promise<QueryResult<R>> {
+    const client = await this.connect();
+    try {
+      return await client.query<R>(text, values);
+    } finally {
+      client.release();
+    }
+  }
+
+  releaseClient(client: PoolClient): void {
+    const index = this.clients.indexOf(client);
+    if (index !== -1) {
+      client._markAvailable();
+      if (this.waitingQueue.length > 0) {
+        const resolve = this.waitingQueue.shift()!;
+        client._markInUse();
+        resolve(client);
+      } else {
+        this.availableClients.push(client);
+      }
+    }
+  }
+
+  end(): Promise<void> {
+    this._ended = true;
+    this.clients.forEach(client => client._markReleased());
+    this.clients = [];
+    this.availableClients = [];
+    this.emit('end');
+    return Promise.resolve();
+  }
+}
+
+export class PoolClient extends Client {
+  private pool: Pool;
+  private _released = false;
+  private _inUse = true;
+  private _userReleased = false;
+
+  constructor(pglite: PGliteInterface, pool: Pool) {
+    super({ pglite });
+    this.pool = pool;
+    this._connected = true;
+  }
+
+  override async query<R extends QueryResultRow = any>(text: string, values?: any[]): Promise<QueryResult<R>> {
+    if (this._userReleased && !this._inUse) {
+      throw new Error('Client has been released back to the pool');
+    }
+    const result = await this.pglite.query<R>(text, values);
+    return this.convertPGliteResult(result);
+  }
+
+  release(): void {
+    if (this._released || this._userReleased) {
+      return;
+    }
+    this._userReleased = true;
+    this.pool.releaseClient(this);
+  }
+
+  override end(): Promise<void> {
+    this.release();
+    return Promise.resolve();
+  }
+
+  _markInUse(): void {
+    this._inUse = true;
+    this._userReleased = false;
+  }
+
+  _markAvailable(): void {
+    this._inUse = false;
+    this._userReleased = false;
+  }
+
+  _markReleased(): void {
+    this._released = true;
+    this._inUse = false;
+    this._userReleased = true;
+  }
+
+  override get connected(): boolean {
+    return this._connected && !this._released;
+  }
+}

package/src/types.ts

@@ -0,0 +1,207 @@
+import {
+  BlockNumber,
+  type CheckpointNumber,
+  type IndexWithinCheckpoint,
+  type SlotNumber,
+} from '@aztec/foundation/branded-types';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+
+import type { Pool } from 'pg';
+
+import type { ValidatorHASignerConfig } from './config.js';
+import {
+  type BlockProposalDutyIdentifier,
+  type CheckAndRecordParams,
+  type DeleteDutyParams,
+  type DutyIdentifier,
+  DutyType,
+  type OtherDutyIdentifier,
+  type RecordSuccessParams,
+  type ValidatorDutyRecord,
+} from './db/types.js';
+
+export type {
+  BlockProposalDutyIdentifier,
+  CheckAndRecordParams,
+  DeleteDutyParams,
+  DutyIdentifier,
+  OtherDutyIdentifier,
+  RecordSuccessParams,
+  ValidatorDutyRecord,
+  ValidatorHASignerConfig,
+};
+export { DutyStatus, DutyType, getBlockIndexFromDutyIdentifier, normalizeBlockIndex } from './db/types.js';
+
+/**
+ * Result of tryInsertOrGetExisting operation
+ */
+export interface TryInsertOrGetResult {
+  /** True if we inserted a new record, false if we got an existing record */
+  isNew: boolean;
+  /** The record (either newly inserted or existing) */
+  record: ValidatorDutyRecord;
+}
+
+/**
+ * deps for creating an HA signer
+ */
+export interface CreateHASignerDeps {
+  /**
+   * Optional PostgreSQL connection pool
+   * If provided, databaseUrl and poolConfig are ignored
+   */
+  pool?: Pool;
+}
+
+/**
+ * Base context for signing operations
+ */
+interface BaseSigningContext {
+  /** Slot number for this duty */
+  slot: SlotNumber;
+  /**
+   * Block or checkpoint number for this duty.
+   * For block proposals, this is the block number.
+   * For checkpoint proposals, this is the checkpoint number.
+   */
+  blockNumber: BlockNumber | CheckpointNumber;
+}
+
+/**
+ * Signing context for block proposals.
+ * blockIndexWithinCheckpoint is REQUIRED and must be >= 0.
+ */
+export interface BlockProposalSigningContext extends BaseSigningContext {
+  /** Block index within checkpoint (0, 1, 2...). Required for block proposals. */
+  blockIndexWithinCheckpoint: IndexWithinCheckpoint;
+  dutyType: DutyType.BLOCK_PROPOSAL;
+}
+
+/**
+ * Signing context for non-block-proposal duties that require HA protection.
+ * blockIndexWithinCheckpoint is not applicable (internally always -1).
+ */
+export interface OtherSigningContext extends BaseSigningContext {
+  dutyType: DutyType.CHECKPOINT_PROPOSAL | DutyType.ATTESTATION | DutyType.ATTESTATIONS_AND_SIGNERS;
+}
+
+/**
+ * Signing context for governance/slashing votes which only need slot for HA protection.
+ * blockNumber is not applicable (internally always 0).
+ */
+export interface VoteSigningContext {
+  slot: SlotNumber;
+  dutyType: DutyType.GOVERNANCE_VOTE | DutyType.SLASHING_VOTE;
+}
+
+/**
+ * Signing context for duties which don't require slot/blockNumber
+ * as they don't need HA protection (AUTH_REQUEST, TXS).
+ */
+export interface NoHAProtectionSigningContext {
+  dutyType: DutyType.AUTH_REQUEST | DutyType.TXS;
+}
+
+/**
+ * Signing contexts that require HA protection (excludes AUTH_REQUEST).
+ * Used by the HA signer's signWithProtection method.
+ */
+export type HAProtectedSigningContext = BlockProposalSigningContext | OtherSigningContext | VoteSigningContext;
+
+/**
+ * Type guard to check if a SigningContext requires HA protection.
+ * Returns true for contexts that need HA protection, false for AUTH_REQUEST and TXS.
+ */
+export function isHAProtectedContext(context: SigningContext): context is HAProtectedSigningContext {
+  return context.dutyType !== DutyType.AUTH_REQUEST && context.dutyType !== DutyType.TXS;
+}
+
+/**
+ * Gets the block number from a signing context.
+ * - Vote duties (GOVERNANCE_VOTE, SLASHING_VOTE): returns BlockNumber(0)
+ * - Other duties: returns the blockNumber from the context
+ */
+export function getBlockNumberFromSigningContext(context: HAProtectedSigningContext): BlockNumber | CheckpointNumber {
+  // Check for duty types that have blockNumber
+  if (
+    context.dutyType === DutyType.BLOCK_PROPOSAL ||
+    context.dutyType === DutyType.CHECKPOINT_PROPOSAL ||
+    context.dutyType === DutyType.ATTESTATION ||
+    context.dutyType === DutyType.ATTESTATIONS_AND_SIGNERS
+  ) {
+    return context.blockNumber;
+  }
+  // Vote duties (GOVERNANCE_VOTE, SLASHING_VOTE) don't have blockNumber
+  return BlockNumber(0);
+}
+
+/**
+ * Context required for slashing protection during signing operations.
+ * Uses discriminated union to enforce type safety:
+ * - BLOCK_PROPOSAL duties MUST have blockIndexWithinCheckpoint >= 0
+ * - Other duty types do NOT have blockIndexWithinCheckpoint (internally -1)
+ * - Vote duties only need slot (blockNumber is internally 0)
+ * - AUTH_REQUEST and TXS duties don't need slot/blockNumber (no HA protection needed)
+ */
+export type SigningContext = HAProtectedSigningContext | NoHAProtectionSigningContext;
+
+/**
+ * Database interface for slashing protection operations
+ * This abstraction allows for different database implementations (PostgreSQL, SQLite, etc.)
+ *
+ * The interface is designed around 3 core operations:
+ * 1. tryInsertOrGetExisting - Atomically insert or get existing record (eliminates race conditions)
+ * 2. updateDutySigned - Update to signed status on success
+ * 3. deleteDuty - Delete a duty record on failure
+ */
+export interface SlashingProtectionDatabase {
+  /**
+   * Atomically try to insert a new duty record, or get the existing one if present.
+   *
+   * @returns { isNew: true, record } if we successfully inserted and acquired the lock
+   * @returns { isNew: false, record } if a record already exists (caller should handle based on status)
+   */
+  tryInsertOrGetExisting(params: CheckAndRecordParams): Promise<TryInsertOrGetResult>;
+
+  /**
+   * Update a duty to 'signed' status with the signature.
+   * Only succeeds if the lockToken matches (caller must be the one who created the duty).
+   *
+   * @returns true if the update succeeded, false if token didn't match or duty not found
+   */
+  updateDutySigned(
+    validatorAddress: EthAddress,
+    slot: SlotNumber,
+    dutyType: DutyType,
+    signature: string,
+    lockToken: string,
+    blockIndexWithinCheckpoint: number,
+  ): Promise<boolean>;
+
+  /**
+   * Delete a duty record.
+   * Only succeeds if the lockToken matches (caller must be the one who created the duty).
+   * Used when signing fails to allow another node/attempt to retry.
+   *
+   * @returns true if the delete succeeded, false if token didn't match or duty not found
+   */
+  deleteDuty(
+    validatorAddress: EthAddress,
+    slot: SlotNumber,
+    dutyType: DutyType,
+    lockToken: string,
+    blockIndexWithinCheckpoint: number,
+  ): Promise<boolean>;
+
+  /**
+   * Cleanup own stuck duties
+   * @returns the number of duties cleaned up
+   */
+  cleanupOwnStuckDuties(nodeId: string, maxAgeMs: number): Promise<number>;
+
+  /**
+   * Close the database connection.
+   * Should be called during graceful shutdown.
+   */
+  close(): Promise<void>;
+}