@aztec/validator-ha-signer 0.0.1-commit.23b0eb0 → 0.0.1-commit.27d773e65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dest/db/postgres.d.ts +15 -1
- package/dest/db/postgres.d.ts.map +1 -1
- package/dest/db/postgres.js +24 -3
- package/dest/db/schema.d.ts +13 -6
- package/dest/db/schema.d.ts.map +1 -1
- package/dest/db/schema.js +14 -5
- package/dest/db/types.d.ts +4 -16
- package/dest/db/types.d.ts.map +1 -1
- package/dest/db/types.js +5 -15
- package/dest/factory.d.ts +2 -2
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +11 -1
- package/dest/metrics.d.ts +51 -0
- package/dest/metrics.d.ts.map +1 -0
- package/dest/metrics.js +103 -0
- package/dest/slashing_protection_service.d.ts +19 -6
- package/dest/slashing_protection_service.d.ts.map +1 -1
- package/dest/slashing_protection_service.js +51 -11
- package/dest/types.d.ts +28 -72
- package/dest/types.d.ts.map +1 -1
- package/dest/types.js +3 -20
- package/dest/validator_ha_signer.d.ts +13 -5
- package/dest/validator_ha_signer.d.ts.map +1 -1
- package/dest/validator_ha_signer.js +18 -6
- package/package.json +7 -5
- package/src/db/postgres.ts +25 -2
- package/src/db/schema.ts +16 -5
- package/src/db/types.ts +5 -15
- package/src/factory.ts +13 -2
- package/src/metrics.ts +138 -0
- package/src/slashing_protection_service.ts +65 -15
- package/src/types.ts +45 -107
- package/src/validator_ha_signer.ts +35 -9
- package/dest/config.d.ts +0 -96
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -86
- package/src/config.ts +0 -141
package/src/factory.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Factory functions for creating validator HA signers
|
|
3
3
|
*/
|
|
4
|
+
import { DateProvider } from '@aztec/foundation/timer';
|
|
5
|
+
import type { ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
|
|
6
|
+
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
7
|
+
|
|
4
8
|
import { Pool } from 'pg';
|
|
5
9
|
|
|
6
|
-
import type { ValidatorHASignerConfig } from './config.js';
|
|
7
10
|
import { PostgresSlashingProtectionDatabase } from './db/postgres.js';
|
|
11
|
+
import { HASignerMetrics } from './metrics.js';
|
|
8
12
|
import type { CreateHASignerDeps, SlashingProtectionDatabase } from './types.js';
|
|
9
13
|
import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
10
14
|
|
|
@@ -55,6 +59,10 @@ export async function createHASigner(
|
|
|
55
59
|
if (!databaseUrl) {
|
|
56
60
|
throw new Error('databaseUrl is required for createHASigner');
|
|
57
61
|
}
|
|
62
|
+
|
|
63
|
+
const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
|
|
64
|
+
const dateProvider = deps?.dateProvider ?? new DateProvider();
|
|
65
|
+
|
|
58
66
|
// Create connection pool (or use provided pool)
|
|
59
67
|
let pool: Pool;
|
|
60
68
|
if (!deps?.pool) {
|
|
@@ -75,8 +83,11 @@ export async function createHASigner(
|
|
|
75
83
|
// Verify database schema is initialized and version matches
|
|
76
84
|
await db.initialize();
|
|
77
85
|
|
|
86
|
+
// Create metrics
|
|
87
|
+
const metrics = new HASignerMetrics(telemetryClient, signerConfig.nodeId);
|
|
88
|
+
|
|
78
89
|
// Create signer
|
|
79
|
-
const signer = new ValidatorHASigner(db, { ...signerConfig, databaseUrl });
|
|
90
|
+
const signer = new ValidatorHASigner(db, { ...signerConfig, databaseUrl }, { metrics, dateProvider });
|
|
80
91
|
|
|
81
92
|
return { signer, db };
|
|
82
93
|
}
|
package/src/metrics.ts
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Attributes,
|
|
3
|
+
type Histogram,
|
|
4
|
+
Metrics,
|
|
5
|
+
type TelemetryClient,
|
|
6
|
+
type UpDownCounter,
|
|
7
|
+
createUpDownCounterWithDefault,
|
|
8
|
+
} from '@aztec/telemetry-client';
|
|
9
|
+
|
|
10
|
+
export type HACleanupType = 'stuck' | 'old' | 'outdated_rollup';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Metrics for HA signer tracking signing operations, lock acquisition, and cleanup.
|
|
14
|
+
*/
|
|
15
|
+
export class HASignerMetrics {
|
|
16
|
+
// Signing lifecycle metrics
|
|
17
|
+
private signingDuration: Histogram;
|
|
18
|
+
private signingSuccessCount: UpDownCounter;
|
|
19
|
+
private dutyAlreadySignedCount: UpDownCounter;
|
|
20
|
+
private slashingProtectionCount: UpDownCounter;
|
|
21
|
+
private signingErrorCount: UpDownCounter;
|
|
22
|
+
|
|
23
|
+
// Lock acquisition metrics
|
|
24
|
+
private lockAcquiredCount: UpDownCounter;
|
|
25
|
+
|
|
26
|
+
// Cleanup metrics
|
|
27
|
+
private cleanupStuckDutiesCount: UpDownCounter;
|
|
28
|
+
private cleanupOldDutiesCount: UpDownCounter;
|
|
29
|
+
private cleanupOutdatedRollupDutiesCount: UpDownCounter;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
client: TelemetryClient,
|
|
33
|
+
private nodeId: string,
|
|
34
|
+
name = 'HASignerMetrics',
|
|
35
|
+
) {
|
|
36
|
+
const meter = client.getMeter(name);
|
|
37
|
+
|
|
38
|
+
// Signing lifecycle
|
|
39
|
+
this.signingDuration = meter.createHistogram(Metrics.HA_SIGNER_SIGNING_DURATION);
|
|
40
|
+
this.signingSuccessCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_SUCCESS_COUNT);
|
|
41
|
+
this.dutyAlreadySignedCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_DUTY_ALREADY_SIGNED_COUNT);
|
|
42
|
+
this.slashingProtectionCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SLASHING_PROTECTION_COUNT);
|
|
43
|
+
this.signingErrorCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_ERROR_COUNT);
|
|
44
|
+
|
|
45
|
+
// Lock acquisition
|
|
46
|
+
this.lockAcquiredCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_LOCK_ACQUIRED_COUNT);
|
|
47
|
+
|
|
48
|
+
// Cleanup
|
|
49
|
+
this.cleanupStuckDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_STUCK_DUTIES_COUNT);
|
|
50
|
+
this.cleanupOldDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_OLD_DUTIES_COUNT);
|
|
51
|
+
this.cleanupOutdatedRollupDutiesCount = createUpDownCounterWithDefault(
|
|
52
|
+
meter,
|
|
53
|
+
Metrics.HA_SIGNER_CLEANUP_OUTDATED_ROLLUP_DUTIES_COUNT,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Record a successful signing operation.
|
|
59
|
+
* @param dutyType - The type of duty signed
|
|
60
|
+
* @param durationMs - Duration from start of signWithProtection to completion
|
|
61
|
+
*/
|
|
62
|
+
public recordSigningSuccess(dutyType: string, durationMs: number): void {
|
|
63
|
+
const attributes = {
|
|
64
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
65
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
66
|
+
};
|
|
67
|
+
this.signingSuccessCount.add(1, attributes);
|
|
68
|
+
this.signingDuration.record(durationMs, attributes);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Record a DutyAlreadySignedError (expected in HA; another node signed first).
|
|
73
|
+
* @param dutyType - The type of duty
|
|
74
|
+
*/
|
|
75
|
+
public recordDutyAlreadySigned(dutyType: string): void {
|
|
76
|
+
const attributes = {
|
|
77
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
78
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
79
|
+
};
|
|
80
|
+
this.dutyAlreadySignedCount.add(1, attributes);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Record a SlashingProtectionError (attempted to sign different data for same duty).
|
|
85
|
+
* @param dutyType - The type of duty
|
|
86
|
+
*/
|
|
87
|
+
public recordSlashingProtection(dutyType: string): void {
|
|
88
|
+
const attributes = {
|
|
89
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
90
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
91
|
+
};
|
|
92
|
+
this.slashingProtectionCount.add(1, attributes);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Record a signing function failure (lock will be deleted for retry).
|
|
97
|
+
* @param dutyType - The type of duty
|
|
98
|
+
*/
|
|
99
|
+
public recordSigningError(dutyType: string): void {
|
|
100
|
+
const attributes = {
|
|
101
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
102
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
103
|
+
};
|
|
104
|
+
this.signingErrorCount.add(1, attributes);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Record lock acquisition.
|
|
109
|
+
* @param acquired - Whether a new lock was acquired (true) or existing record found (false)
|
|
110
|
+
*/
|
|
111
|
+
public recordLockAcquire(acquired: boolean): void {
|
|
112
|
+
if (acquired) {
|
|
113
|
+
const attributes = {
|
|
114
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
115
|
+
};
|
|
116
|
+
this.lockAcquiredCount.add(1, attributes);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Record cleanup metrics.
|
|
122
|
+
* @param type - Type of cleanup
|
|
123
|
+
* @param count - Number of duties cleaned up
|
|
124
|
+
*/
|
|
125
|
+
public recordCleanup(type: HACleanupType, count: number): void {
|
|
126
|
+
const attributes = {
|
|
127
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
if (type === 'stuck') {
|
|
131
|
+
this.cleanupStuckDutiesCount.add(count, attributes);
|
|
132
|
+
} else if (type === 'old') {
|
|
133
|
+
this.cleanupOldDutiesCount.add(count, attributes);
|
|
134
|
+
} else if (type === 'outdated_rollup') {
|
|
135
|
+
this.cleanupOutdatedRollupDutiesCount.add(count, attributes);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
@@ -7,6 +7,8 @@
|
|
|
7
7
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
8
8
|
import { RunningPromise } from '@aztec/foundation/promise';
|
|
9
9
|
import { sleep } from '@aztec/foundation/sleep';
|
|
10
|
+
import type { DateProvider } from '@aztec/foundation/timer';
|
|
11
|
+
import type { ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
|
|
10
12
|
|
|
11
13
|
import {
|
|
12
14
|
type CheckAndRecordParams,
|
|
@@ -16,7 +18,13 @@ import {
|
|
|
16
18
|
getBlockIndexFromDutyIdentifier,
|
|
17
19
|
} from './db/types.js';
|
|
18
20
|
import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
19
|
-
import type {
|
|
21
|
+
import type { HASignerMetrics } from './metrics.js';
|
|
22
|
+
import type { SlashingProtectionDatabase } from './types.js';
|
|
23
|
+
|
|
24
|
+
export interface SlashingProtectionServiceDeps {
|
|
25
|
+
metrics: HASignerMetrics;
|
|
26
|
+
dateProvider: DateProvider;
|
|
27
|
+
}
|
|
20
28
|
|
|
21
29
|
/**
|
|
22
30
|
* Slashing Protection Service
|
|
@@ -39,11 +47,16 @@ export class SlashingProtectionService {
|
|
|
39
47
|
private readonly signingTimeoutMs: number;
|
|
40
48
|
private readonly maxStuckDutiesAgeMs: number;
|
|
41
49
|
|
|
50
|
+
private readonly metrics: HASignerMetrics;
|
|
51
|
+
private readonly dateProvider: DateProvider;
|
|
52
|
+
|
|
42
53
|
private cleanupRunningPromise: RunningPromise;
|
|
54
|
+
private lastOldDutiesCleanupAtMs?: number;
|
|
43
55
|
|
|
44
56
|
constructor(
|
|
45
57
|
private readonly db: SlashingProtectionDatabase,
|
|
46
58
|
private readonly config: ValidatorHASignerConfig,
|
|
59
|
+
deps: SlashingProtectionServiceDeps,
|
|
47
60
|
) {
|
|
48
61
|
this.log = createLogger('slashing-protection');
|
|
49
62
|
this.pollingIntervalMs = config.pollingIntervalMs;
|
|
@@ -51,11 +64,9 @@ export class SlashingProtectionService {
|
|
|
51
64
|
// Default to 144s (2x 72s Aztec slot duration) if not explicitly configured
|
|
52
65
|
this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
|
|
53
66
|
|
|
54
|
-
this.cleanupRunningPromise = new RunningPromise(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
this.maxStuckDutiesAgeMs,
|
|
58
|
-
);
|
|
67
|
+
this.cleanupRunningPromise = new RunningPromise(this.cleanup.bind(this), this.log, this.maxStuckDutiesAgeMs);
|
|
68
|
+
this.metrics = deps.metrics;
|
|
69
|
+
this.dateProvider = deps.dateProvider;
|
|
59
70
|
}
|
|
60
71
|
|
|
61
72
|
/**
|
|
@@ -67,7 +78,6 @@ export class SlashingProtectionService {
|
|
|
67
78
|
* 2. If insert succeeds, we acquired the lock - return the lockToken
|
|
68
79
|
* 3. If a record exists, handle based on status:
|
|
69
80
|
* - SIGNED: Throw appropriate error (already signed or slashing protection)
|
|
70
|
-
* - FAILED: Delete the failed record
|
|
71
81
|
* - SIGNING: Wait and poll until status changes, then handle result
|
|
72
82
|
*
|
|
73
83
|
* @returns The lockToken that must be used for recordSuccess/deleteDuty
|
|
@@ -76,7 +86,7 @@ export class SlashingProtectionService {
|
|
|
76
86
|
*/
|
|
77
87
|
async checkAndRecord(params: CheckAndRecordParams): Promise<string> {
|
|
78
88
|
const { validatorAddress, slot, dutyType, messageHash, nodeId } = params;
|
|
79
|
-
const startTime =
|
|
89
|
+
const startTime = this.dateProvider.now();
|
|
80
90
|
|
|
81
91
|
this.log.debug(`Checking duty: ${dutyType} for slot ${slot}`, {
|
|
82
92
|
validatorAddress: validatorAddress.toString(),
|
|
@@ -93,6 +103,7 @@ export class SlashingProtectionService {
|
|
|
93
103
|
validatorAddress: validatorAddress.toString(),
|
|
94
104
|
nodeId,
|
|
95
105
|
});
|
|
106
|
+
this.metrics.recordLockAcquire(true);
|
|
96
107
|
return record.lockToken;
|
|
97
108
|
}
|
|
98
109
|
|
|
@@ -107,6 +118,7 @@ export class SlashingProtectionService {
|
|
|
107
118
|
existingNodeId: record.nodeId,
|
|
108
119
|
attemptingNodeId: nodeId,
|
|
109
120
|
});
|
|
121
|
+
this.metrics.recordSlashingProtection(dutyType);
|
|
110
122
|
throw new SlashingProtectionError(
|
|
111
123
|
slot,
|
|
112
124
|
dutyType,
|
|
@@ -116,15 +128,17 @@ export class SlashingProtectionService {
|
|
|
116
128
|
record.nodeId,
|
|
117
129
|
);
|
|
118
130
|
}
|
|
131
|
+
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
119
132
|
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
|
|
120
133
|
} else if (record.status === DutyStatus.SIGNING) {
|
|
121
134
|
// Another node is currently signing - check for timeout
|
|
122
|
-
if (
|
|
135
|
+
if (this.dateProvider.now() - startTime > this.signingTimeoutMs) {
|
|
123
136
|
this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
|
|
124
137
|
validatorAddress: validatorAddress.toString(),
|
|
125
138
|
timeoutMs: this.signingTimeoutMs,
|
|
126
139
|
signingNodeId: record.nodeId,
|
|
127
140
|
});
|
|
141
|
+
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
128
142
|
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, 'unknown (timeout)');
|
|
129
143
|
}
|
|
130
144
|
|
|
@@ -221,7 +235,20 @@ export class SlashingProtectionService {
|
|
|
221
235
|
* Start running tasks.
|
|
222
236
|
* Cleanup runs immediately on start to recover from any previous crashes.
|
|
223
237
|
*/
|
|
224
|
-
|
|
238
|
+
/**
|
|
239
|
+
* Start the background cleanup task.
|
|
240
|
+
* Also performs one-time cleanup of duties with outdated rollup addresses.
|
|
241
|
+
*/
|
|
242
|
+
async start() {
|
|
243
|
+
// One-time cleanup at startup: remove duties from previous rollup versions
|
|
244
|
+
const numOutdatedRollupDuties = await this.db.cleanupOutdatedRollupDuties(this.config.l1Contracts.rollupAddress);
|
|
245
|
+
if (numOutdatedRollupDuties > 0) {
|
|
246
|
+
this.log.info(`Cleaned up ${numOutdatedRollupDuties} duties with outdated rollup address at startup`, {
|
|
247
|
+
currentRollupAddress: this.config.l1Contracts.rollupAddress.toString(),
|
|
248
|
+
});
|
|
249
|
+
this.metrics.recordCleanup('outdated_rollup', numOutdatedRollupDuties);
|
|
250
|
+
}
|
|
251
|
+
|
|
225
252
|
this.cleanupRunningPromise.start();
|
|
226
253
|
this.log.info('Slashing protection service started', { nodeId: this.config.nodeId });
|
|
227
254
|
}
|
|
@@ -244,15 +271,38 @@ export class SlashingProtectionService {
|
|
|
244
271
|
}
|
|
245
272
|
|
|
246
273
|
/**
|
|
247
|
-
*
|
|
274
|
+
* Periodic cleanup of stuck duties and optionally old signed duties.
|
|
275
|
+
* Runs in the background via RunningPromise.
|
|
248
276
|
*/
|
|
249
|
-
private async
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
277
|
+
private async cleanup() {
|
|
278
|
+
// 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status)
|
|
279
|
+
const numStuckDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
|
|
280
|
+
if (numStuckDuties > 0) {
|
|
281
|
+
this.log.verbose(`Cleaned up ${numStuckDuties} stuck duties`, {
|
|
253
282
|
nodeId: this.config.nodeId,
|
|
254
283
|
maxStuckDutiesAgeMs: this.maxStuckDutiesAgeMs,
|
|
255
284
|
});
|
|
285
|
+
this.metrics.recordCleanup('stuck', numStuckDuties);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// 2. Clean up old signed duties if configured
|
|
289
|
+
// we shouldn't run this as often as stuck duty cleanup.
|
|
290
|
+
if (this.config.cleanupOldDutiesAfterHours !== undefined) {
|
|
291
|
+
const maxAgeMs = this.config.cleanupOldDutiesAfterHours * 60 * 60 * 1000;
|
|
292
|
+
const nowMs = this.dateProvider.now();
|
|
293
|
+
const shouldRun =
|
|
294
|
+
this.lastOldDutiesCleanupAtMs === undefined || nowMs - this.lastOldDutiesCleanupAtMs >= maxAgeMs;
|
|
295
|
+
if (shouldRun) {
|
|
296
|
+
const numOldDuties = await this.db.cleanupOldDuties(maxAgeMs);
|
|
297
|
+
this.lastOldDutiesCleanupAtMs = nowMs;
|
|
298
|
+
if (numOldDuties > 0) {
|
|
299
|
+
this.log.verbose(`Cleaned up ${numOldDuties} old signed duties`, {
|
|
300
|
+
cleanupOldDutiesAfterHours: this.config.cleanupOldDutiesAfterHours,
|
|
301
|
+
maxAgeMs,
|
|
302
|
+
});
|
|
303
|
+
this.metrics.recordCleanup('old', numOldDuties);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
256
306
|
}
|
|
257
307
|
}
|
|
258
308
|
}
|
package/src/types.ts
CHANGED
|
@@ -1,24 +1,27 @@
|
|
|
1
|
-
import {
|
|
2
|
-
BlockNumber,
|
|
3
|
-
type CheckpointNumber,
|
|
4
|
-
type IndexWithinCheckpoint,
|
|
5
|
-
type SlotNumber,
|
|
6
|
-
} from '@aztec/foundation/branded-types';
|
|
1
|
+
import { SlotNumber } from '@aztec/foundation/branded-types';
|
|
7
2
|
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
3
|
+
import { DateProvider } from '@aztec/foundation/timer';
|
|
4
|
+
import {
|
|
5
|
+
DutyType,
|
|
6
|
+
type HAProtectedSigningContext,
|
|
7
|
+
type SigningContext,
|
|
8
|
+
type ValidatorHASignerConfig,
|
|
9
|
+
getBlockNumberFromSigningContext as getBlockNumberFromSigningContextFromStdlib,
|
|
10
|
+
isHAProtectedContext,
|
|
11
|
+
} from '@aztec/stdlib/ha-signing';
|
|
12
|
+
import type { TelemetryClient } from '@aztec/telemetry-client';
|
|
8
13
|
|
|
9
14
|
import type { Pool } from 'pg';
|
|
10
15
|
|
|
11
|
-
import type {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
type RecordSuccessParams,
|
|
21
|
-
type ValidatorDutyRecord,
|
|
16
|
+
import type {
|
|
17
|
+
BlockProposalDutyIdentifier,
|
|
18
|
+
CheckAndRecordParams,
|
|
19
|
+
DeleteDutyParams,
|
|
20
|
+
DutyIdentifier,
|
|
21
|
+
DutyRow,
|
|
22
|
+
OtherDutyIdentifier,
|
|
23
|
+
RecordSuccessParams,
|
|
24
|
+
ValidatorDutyRecord,
|
|
22
25
|
} from './db/types.js';
|
|
23
26
|
|
|
24
27
|
export type {
|
|
@@ -27,12 +30,16 @@ export type {
|
|
|
27
30
|
DeleteDutyParams,
|
|
28
31
|
DutyIdentifier,
|
|
29
32
|
DutyRow,
|
|
33
|
+
HAProtectedSigningContext,
|
|
30
34
|
OtherDutyIdentifier,
|
|
31
35
|
RecordSuccessParams,
|
|
36
|
+
SigningContext,
|
|
32
37
|
ValidatorDutyRecord,
|
|
33
38
|
ValidatorHASignerConfig,
|
|
34
39
|
};
|
|
35
40
|
export { DutyStatus, DutyType, getBlockIndexFromDutyIdentifier, normalizeBlockIndex } from './db/types.js';
|
|
41
|
+
export { isHAProtectedContext };
|
|
42
|
+
export { getBlockNumberFromSigningContextFromStdlib as getBlockNumberFromSigningContext };
|
|
36
43
|
|
|
37
44
|
/**
|
|
38
45
|
* Result of tryInsertOrGetExisting operation
|
|
@@ -53,100 +60,16 @@ export interface CreateHASignerDeps {
|
|
|
53
60
|
* If provided, databaseUrl and poolConfig are ignored
|
|
54
61
|
*/
|
|
55
62
|
pool?: Pool;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
/**
|
|
59
|
-
* Base context for signing operations
|
|
60
|
-
*/
|
|
61
|
-
interface BaseSigningContext {
|
|
62
|
-
/** Slot number for this duty */
|
|
63
|
-
slot: SlotNumber;
|
|
64
63
|
/**
|
|
65
|
-
*
|
|
66
|
-
* For block proposals, this is the block number.
|
|
67
|
-
* For checkpoint proposals, this is the checkpoint number.
|
|
64
|
+
* Optional telemetry client for metrics
|
|
68
65
|
*/
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
* blockIndexWithinCheckpoint is REQUIRED and must be >= 0.
|
|
75
|
-
*/
|
|
76
|
-
export interface BlockProposalSigningContext extends BaseSigningContext {
|
|
77
|
-
/** Block index within checkpoint (0, 1, 2...). Required for block proposals. */
|
|
78
|
-
blockIndexWithinCheckpoint: IndexWithinCheckpoint;
|
|
79
|
-
dutyType: DutyType.BLOCK_PROPOSAL;
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
/**
|
|
83
|
-
* Signing context for non-block-proposal duties that require HA protection.
|
|
84
|
-
* blockIndexWithinCheckpoint is not applicable (internally always -1).
|
|
85
|
-
*/
|
|
86
|
-
export interface OtherSigningContext extends BaseSigningContext {
|
|
87
|
-
dutyType: DutyType.CHECKPOINT_PROPOSAL | DutyType.ATTESTATION | DutyType.ATTESTATIONS_AND_SIGNERS;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
/**
|
|
91
|
-
* Signing context for governance/slashing votes which only need slot for HA protection.
|
|
92
|
-
* blockNumber is not applicable (internally always 0).
|
|
93
|
-
*/
|
|
94
|
-
export interface VoteSigningContext {
|
|
95
|
-
slot: SlotNumber;
|
|
96
|
-
dutyType: DutyType.GOVERNANCE_VOTE | DutyType.SLASHING_VOTE;
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* Signing context for duties which don't require slot/blockNumber
|
|
101
|
-
* as they don't need HA protection (AUTH_REQUEST, TXS).
|
|
102
|
-
*/
|
|
103
|
-
export interface NoHAProtectionSigningContext {
|
|
104
|
-
dutyType: DutyType.AUTH_REQUEST | DutyType.TXS;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/**
|
|
108
|
-
* Signing contexts that require HA protection (excludes AUTH_REQUEST).
|
|
109
|
-
* Used by the HA signer's signWithProtection method.
|
|
110
|
-
*/
|
|
111
|
-
export type HAProtectedSigningContext = BlockProposalSigningContext | OtherSigningContext | VoteSigningContext;
|
|
112
|
-
|
|
113
|
-
/**
|
|
114
|
-
* Type guard to check if a SigningContext requires HA protection.
|
|
115
|
-
* Returns true for contexts that need HA protection, false for AUTH_REQUEST and TXS.
|
|
116
|
-
*/
|
|
117
|
-
export function isHAProtectedContext(context: SigningContext): context is HAProtectedSigningContext {
|
|
118
|
-
return context.dutyType !== DutyType.AUTH_REQUEST && context.dutyType !== DutyType.TXS;
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
/**
|
|
122
|
-
* Gets the block number from a signing context.
|
|
123
|
-
* - Vote duties (GOVERNANCE_VOTE, SLASHING_VOTE): returns BlockNumber(0)
|
|
124
|
-
* - Other duties: returns the blockNumber from the context
|
|
125
|
-
*/
|
|
126
|
-
export function getBlockNumberFromSigningContext(context: HAProtectedSigningContext): BlockNumber | CheckpointNumber {
|
|
127
|
-
// Check for duty types that have blockNumber
|
|
128
|
-
if (
|
|
129
|
-
context.dutyType === DutyType.BLOCK_PROPOSAL ||
|
|
130
|
-
context.dutyType === DutyType.CHECKPOINT_PROPOSAL ||
|
|
131
|
-
context.dutyType === DutyType.ATTESTATION ||
|
|
132
|
-
context.dutyType === DutyType.ATTESTATIONS_AND_SIGNERS
|
|
133
|
-
) {
|
|
134
|
-
return context.blockNumber;
|
|
135
|
-
}
|
|
136
|
-
// Vote duties (GOVERNANCE_VOTE, SLASHING_VOTE) don't have blockNumber
|
|
137
|
-
return BlockNumber(0);
|
|
66
|
+
telemetryClient?: TelemetryClient;
|
|
67
|
+
/**
|
|
68
|
+
* Optional date provider for timestamps
|
|
69
|
+
*/
|
|
70
|
+
dateProvider?: DateProvider;
|
|
138
71
|
}
|
|
139
72
|
|
|
140
|
-
/**
|
|
141
|
-
* Context required for slashing protection during signing operations.
|
|
142
|
-
* Uses discriminated union to enforce type safety:
|
|
143
|
-
* - BLOCK_PROPOSAL duties MUST have blockIndexWithinCheckpoint >= 0
|
|
144
|
-
* - Other duty types do NOT have blockIndexWithinCheckpoint (internally -1)
|
|
145
|
-
* - Vote duties only need slot (blockNumber is internally 0)
|
|
146
|
-
* - AUTH_REQUEST and TXS duties don't need slot/blockNumber (no HA protection needed)
|
|
147
|
-
*/
|
|
148
|
-
export type SigningContext = HAProtectedSigningContext | NoHAProtectionSigningContext;
|
|
149
|
-
|
|
150
73
|
/**
|
|
151
74
|
* Database interface for slashing protection operations
|
|
152
75
|
* This abstraction allows for different database implementations (PostgreSQL, SQLite, etc.)
|
|
@@ -203,6 +126,21 @@ export interface SlashingProtectionDatabase {
|
|
|
203
126
|
*/
|
|
204
127
|
cleanupOwnStuckDuties(nodeId: string, maxAgeMs: number): Promise<number>;
|
|
205
128
|
|
|
129
|
+
/**
|
|
130
|
+
* Cleanup duties with outdated rollup address.
|
|
131
|
+
* Removes all duties where the rollup address doesn't match the current one.
|
|
132
|
+
* Used after a rollup upgrade to clean up duties for the old rollup.
|
|
133
|
+
* @returns the number of duties cleaned up
|
|
134
|
+
*/
|
|
135
|
+
cleanupOutdatedRollupDuties(currentRollupAddress: EthAddress): Promise<number>;
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Cleanup old signed duties.
|
|
139
|
+
* Removes only signed duties older than the specified age.
|
|
140
|
+
* @returns the number of duties cleaned up
|
|
141
|
+
*/
|
|
142
|
+
cleanupOldDuties(maxAgeMs: number): Promise<number>;
|
|
143
|
+
|
|
206
144
|
/**
|
|
207
145
|
* Close the database connection.
|
|
208
146
|
* Should be called during graceful shutdown.
|
|
@@ -9,15 +9,23 @@ import type { Buffer32 } from '@aztec/foundation/buffer';
|
|
|
9
9
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
10
10
|
import type { Signature } from '@aztec/foundation/eth-signature';
|
|
11
11
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
12
|
-
|
|
13
|
-
import type { ValidatorHASignerConfig } from './config.js';
|
|
14
|
-
import { type DutyIdentifier, DutyType } from './db/types.js';
|
|
15
|
-
import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
12
|
+
import type { DateProvider } from '@aztec/foundation/timer';
|
|
16
13
|
import {
|
|
14
|
+
DutyType,
|
|
17
15
|
type HAProtectedSigningContext,
|
|
18
|
-
type
|
|
16
|
+
type ValidatorHASignerConfig,
|
|
19
17
|
getBlockNumberFromSigningContext,
|
|
20
|
-
} from '
|
|
18
|
+
} from '@aztec/stdlib/ha-signing';
|
|
19
|
+
|
|
20
|
+
import type { DutyIdentifier } from './db/types.js';
|
|
21
|
+
import type { HASignerMetrics } from './metrics.js';
|
|
22
|
+
import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
23
|
+
import type { SlashingProtectionDatabase } from './types.js';
|
|
24
|
+
|
|
25
|
+
export interface ValidatorHASignerDeps {
|
|
26
|
+
metrics: HASignerMetrics;
|
|
27
|
+
dateProvider: DateProvider;
|
|
28
|
+
}
|
|
21
29
|
|
|
22
30
|
/**
|
|
23
31
|
* Validator High Availability Signer
|
|
@@ -43,12 +51,19 @@ export class ValidatorHASigner {
|
|
|
43
51
|
private readonly slashingProtection: SlashingProtectionService;
|
|
44
52
|
private readonly rollupAddress: EthAddress;
|
|
45
53
|
|
|
54
|
+
private readonly dateProvider: DateProvider;
|
|
55
|
+
private readonly metrics: HASignerMetrics;
|
|
56
|
+
|
|
46
57
|
constructor(
|
|
47
58
|
db: SlashingProtectionDatabase,
|
|
48
59
|
private readonly config: ValidatorHASignerConfig,
|
|
60
|
+
deps: ValidatorHASignerDeps,
|
|
49
61
|
) {
|
|
50
62
|
this.log = createLogger('validator-ha-signer');
|
|
51
63
|
|
|
64
|
+
this.metrics = deps.metrics;
|
|
65
|
+
this.dateProvider = deps.dateProvider;
|
|
66
|
+
|
|
52
67
|
if (!config.haSigningEnabled) {
|
|
53
68
|
// this shouldn't happen, the validator should use different signer for non-HA setups
|
|
54
69
|
throw new Error('Validator HA Signer is not enabled in config');
|
|
@@ -58,7 +73,10 @@ export class ValidatorHASigner {
|
|
|
58
73
|
throw new Error('NODE_ID is required for high-availability setups');
|
|
59
74
|
}
|
|
60
75
|
this.rollupAddress = config.l1Contracts.rollupAddress;
|
|
61
|
-
this.slashingProtection = new SlashingProtectionService(db, config
|
|
76
|
+
this.slashingProtection = new SlashingProtectionService(db, config, {
|
|
77
|
+
metrics: deps.metrics,
|
|
78
|
+
dateProvider: deps.dateProvider,
|
|
79
|
+
});
|
|
62
80
|
this.log.info('Validator HA Signer initialized with slashing protection', {
|
|
63
81
|
nodeId: config.nodeId,
|
|
64
82
|
rollupAddress: this.rollupAddress.toString(),
|
|
@@ -88,6 +106,9 @@ export class ValidatorHASigner {
|
|
|
88
106
|
context: HAProtectedSigningContext,
|
|
89
107
|
signFn: (messageHash: Buffer32) => Promise<Signature>,
|
|
90
108
|
): Promise<Signature> {
|
|
109
|
+
const startTime = this.dateProvider.now();
|
|
110
|
+
const dutyType = context.dutyType;
|
|
111
|
+
|
|
91
112
|
let dutyIdentifier: DutyIdentifier;
|
|
92
113
|
if (context.dutyType === DutyType.BLOCK_PROPOSAL) {
|
|
93
114
|
dutyIdentifier = {
|
|
@@ -107,6 +128,7 @@ export class ValidatorHASigner {
|
|
|
107
128
|
}
|
|
108
129
|
|
|
109
130
|
// Acquire lock and get the token for ownership verification
|
|
131
|
+
// DutyAlreadySignedError and SlashingProtectionError may be thrown here and are recorded in the service
|
|
110
132
|
const blockNumber = getBlockNumberFromSigningContext(context);
|
|
111
133
|
const lockToken = await this.slashingProtection.checkAndRecord({
|
|
112
134
|
...dutyIdentifier,
|
|
@@ -122,6 +144,7 @@ export class ValidatorHASigner {
|
|
|
122
144
|
} catch (error: any) {
|
|
123
145
|
// Delete duty to allow retry (only succeeds if we own the lock)
|
|
124
146
|
await this.slashingProtection.deleteDuty({ ...dutyIdentifier, lockToken });
|
|
147
|
+
this.metrics.recordSigningError(dutyType);
|
|
125
148
|
throw error;
|
|
126
149
|
}
|
|
127
150
|
|
|
@@ -133,6 +156,9 @@ export class ValidatorHASigner {
|
|
|
133
156
|
lockToken,
|
|
134
157
|
});
|
|
135
158
|
|
|
159
|
+
const duration = this.dateProvider.now() - startTime;
|
|
160
|
+
this.metrics.recordSigningSuccess(dutyType, duration);
|
|
161
|
+
|
|
136
162
|
return signature;
|
|
137
163
|
}
|
|
138
164
|
|
|
@@ -147,8 +173,8 @@ export class ValidatorHASigner {
|
|
|
147
173
|
* Start the HA signer background tasks (cleanup of stuck duties).
|
|
148
174
|
* Should be called after construction and before signing operations.
|
|
149
175
|
*/
|
|
150
|
-
start() {
|
|
151
|
-
this.slashingProtection.start();
|
|
176
|
+
async start() {
|
|
177
|
+
await this.slashingProtection.start();
|
|
152
178
|
}
|
|
153
179
|
|
|
154
180
|
/**
|