@aztec/validator-ha-signer 0.0.1-commit.fffb133c → 0.0.2-commit.217f559981
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dest/db/postgres.d.ts +17 -3
- package/dest/db/postgres.d.ts.map +1 -1
- package/dest/db/postgres.js +32 -5
- package/dest/db/schema.d.ts +17 -10
- package/dest/db/schema.d.ts.map +1 -1
- package/dest/db/schema.js +39 -22
- package/dest/db/types.d.ts +9 -16
- package/dest/db/types.d.ts.map +1 -1
- package/dest/db/types.js +5 -15
- package/dest/factory.d.ts +2 -2
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +11 -1
- package/dest/metrics.d.ts +51 -0
- package/dest/metrics.d.ts.map +1 -0
- package/dest/metrics.js +103 -0
- package/dest/slashing_protection_service.d.ts +19 -6
- package/dest/slashing_protection_service.d.ts.map +1 -1
- package/dest/slashing_protection_service.js +55 -15
- package/dest/types.d.ts +30 -74
- package/dest/types.d.ts.map +1 -1
- package/dest/types.js +3 -20
- package/dest/validator_ha_signer.d.ts +15 -6
- package/dest/validator_ha_signer.d.ts.map +1 -1
- package/dest/validator_ha_signer.js +24 -7
- package/package.json +7 -4
- package/src/db/postgres.ts +33 -2
- package/src/db/schema.ts +41 -22
- package/src/db/types.ts +10 -15
- package/src/factory.ts +13 -2
- package/src/metrics.ts +138 -0
- package/src/slashing_protection_service.ts +76 -18
- package/src/types.ts +48 -106
- package/src/validator_ha_signer.ts +41 -10
- package/dest/config.d.ts +0 -79
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -73
- package/src/config.ts +0 -125
|
@@ -5,9 +5,8 @@
|
|
|
5
5
|
* This ensures that even with multiple validator nodes running, only one
|
|
6
6
|
* node will sign for a given duty (slot + duty type).
|
|
7
7
|
*/ import { createLogger } from '@aztec/foundation/log';
|
|
8
|
-
import { DutyType } from '
|
|
8
|
+
import { DutyType, getBlockNumberFromSigningContext } from '@aztec/stdlib/ha-signing';
|
|
9
9
|
import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
10
|
-
import { getBlockNumberFromSigningContext } from './types.js';
|
|
11
10
|
/**
|
|
12
11
|
* Validator High Availability Signer
|
|
13
12
|
*
|
|
@@ -30,9 +29,14 @@ import { getBlockNumberFromSigningContext } from './types.js';
|
|
|
30
29
|
config;
|
|
31
30
|
log;
|
|
32
31
|
slashingProtection;
|
|
33
|
-
|
|
32
|
+
rollupAddress;
|
|
33
|
+
dateProvider;
|
|
34
|
+
metrics;
|
|
35
|
+
constructor(db, config, deps){
|
|
34
36
|
this.config = config;
|
|
35
37
|
this.log = createLogger('validator-ha-signer');
|
|
38
|
+
this.metrics = deps.metrics;
|
|
39
|
+
this.dateProvider = deps.dateProvider;
|
|
36
40
|
if (!config.haSigningEnabled) {
|
|
37
41
|
// this shouldn't happen, the validator should use different signer for non-HA setups
|
|
38
42
|
throw new Error('Validator HA Signer is not enabled in config');
|
|
@@ -40,9 +44,14 @@ import { getBlockNumberFromSigningContext } from './types.js';
|
|
|
40
44
|
if (!config.nodeId || config.nodeId === '') {
|
|
41
45
|
throw new Error('NODE_ID is required for high-availability setups');
|
|
42
46
|
}
|
|
43
|
-
this.
|
|
47
|
+
this.rollupAddress = config.l1Contracts.rollupAddress;
|
|
48
|
+
this.slashingProtection = new SlashingProtectionService(db, config, {
|
|
49
|
+
metrics: deps.metrics,
|
|
50
|
+
dateProvider: deps.dateProvider
|
|
51
|
+
});
|
|
44
52
|
this.log.info('Validator HA Signer initialized with slashing protection', {
|
|
45
|
-
nodeId: config.nodeId
|
|
53
|
+
nodeId: config.nodeId,
|
|
54
|
+
rollupAddress: this.rollupAddress.toString()
|
|
46
55
|
});
|
|
47
56
|
}
|
|
48
57
|
/**
|
|
@@ -62,9 +71,12 @@ import { getBlockNumberFromSigningContext } from './types.js';
|
|
|
62
71
|
* @throws DutyAlreadySignedError if the duty was already signed (expected in HA)
|
|
63
72
|
* @throws SlashingProtectionError if attempting to sign different data for same slot (expected in HA)
|
|
64
73
|
*/ async signWithProtection(validatorAddress, messageHash, context, signFn) {
|
|
74
|
+
const startTime = this.dateProvider.now();
|
|
75
|
+
const dutyType = context.dutyType;
|
|
65
76
|
let dutyIdentifier;
|
|
66
77
|
if (context.dutyType === DutyType.BLOCK_PROPOSAL) {
|
|
67
78
|
dutyIdentifier = {
|
|
79
|
+
rollupAddress: this.rollupAddress,
|
|
68
80
|
validatorAddress,
|
|
69
81
|
slot: context.slot,
|
|
70
82
|
blockIndexWithinCheckpoint: context.blockIndexWithinCheckpoint,
|
|
@@ -72,12 +84,14 @@ import { getBlockNumberFromSigningContext } from './types.js';
|
|
|
72
84
|
};
|
|
73
85
|
} else {
|
|
74
86
|
dutyIdentifier = {
|
|
87
|
+
rollupAddress: this.rollupAddress,
|
|
75
88
|
validatorAddress,
|
|
76
89
|
slot: context.slot,
|
|
77
90
|
dutyType: context.dutyType
|
|
78
91
|
};
|
|
79
92
|
}
|
|
80
93
|
// Acquire lock and get the token for ownership verification
|
|
94
|
+
// DutyAlreadySignedError and SlashingProtectionError may be thrown here and are recorded in the service
|
|
81
95
|
const blockNumber = getBlockNumberFromSigningContext(context);
|
|
82
96
|
const lockToken = await this.slashingProtection.checkAndRecord({
|
|
83
97
|
...dutyIdentifier,
|
|
@@ -95,6 +109,7 @@ import { getBlockNumberFromSigningContext } from './types.js';
|
|
|
95
109
|
...dutyIdentifier,
|
|
96
110
|
lockToken
|
|
97
111
|
});
|
|
112
|
+
this.metrics.recordSigningError(dutyType);
|
|
98
113
|
throw error;
|
|
99
114
|
}
|
|
100
115
|
// Record success (only succeeds if we own the lock)
|
|
@@ -104,6 +119,8 @@ import { getBlockNumberFromSigningContext } from './types.js';
|
|
|
104
119
|
nodeId: this.config.nodeId,
|
|
105
120
|
lockToken
|
|
106
121
|
});
|
|
122
|
+
const duration = this.dateProvider.now() - startTime;
|
|
123
|
+
this.metrics.recordSigningSuccess(dutyType, duration);
|
|
107
124
|
return signature;
|
|
108
125
|
}
|
|
109
126
|
/**
|
|
@@ -114,8 +131,8 @@ import { getBlockNumberFromSigningContext } from './types.js';
|
|
|
114
131
|
/**
|
|
115
132
|
* Start the HA signer background tasks (cleanup of stuck duties).
|
|
116
133
|
* Should be called after construction and before signing operations.
|
|
117
|
-
*/ start() {
|
|
118
|
-
this.slashingProtection.start();
|
|
134
|
+
*/ async start() {
|
|
135
|
+
await this.slashingProtection.start();
|
|
119
136
|
}
|
|
120
137
|
/**
|
|
121
138
|
* Stop the HA signer background tasks and close database connection.
|
package/package.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/validator-ha-signer",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.2-commit.217f559981",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
|
-
"./config": "./dest/config.js",
|
|
7
6
|
"./db": "./dest/db/index.js",
|
|
8
7
|
"./errors": "./dest/errors.js",
|
|
9
8
|
"./factory": "./dest/factory.js",
|
|
9
|
+
"./metrics": "./dest/metrics.js",
|
|
10
10
|
"./migrations": "./dest/migrations.js",
|
|
11
11
|
"./slashing-protection-service": "./dest/slashing_protection_service.js",
|
|
12
12
|
"./types": "./dest/types.js",
|
|
@@ -15,10 +15,10 @@
|
|
|
15
15
|
},
|
|
16
16
|
"typedocOptions": {
|
|
17
17
|
"entryPoints": [
|
|
18
|
-
"./src/config.ts",
|
|
19
18
|
"./src/db/index.ts",
|
|
20
19
|
"./src/errors.ts",
|
|
21
20
|
"./src/factory.ts",
|
|
21
|
+
"./src/metrics.ts",
|
|
22
22
|
"./src/migrations.ts",
|
|
23
23
|
"./src/slashing_protection_service.ts",
|
|
24
24
|
"./src/types.ts",
|
|
@@ -74,7 +74,10 @@
|
|
|
74
74
|
]
|
|
75
75
|
},
|
|
76
76
|
"dependencies": {
|
|
77
|
-
"@aztec/
|
|
77
|
+
"@aztec/ethereum": "0.0.2-commit.217f559981",
|
|
78
|
+
"@aztec/foundation": "0.0.2-commit.217f559981",
|
|
79
|
+
"@aztec/stdlib": "0.0.2-commit.217f559981",
|
|
80
|
+
"@aztec/telemetry-client": "0.0.2-commit.217f559981",
|
|
78
81
|
"node-pg-migrate": "^8.0.4",
|
|
79
82
|
"pg": "^8.11.3",
|
|
80
83
|
"tslib": "^2.4.0",
|
package/src/db/postgres.ts
CHANGED
|
@@ -11,6 +11,8 @@ import type { QueryResult, QueryResultRow } from 'pg';
|
|
|
11
11
|
|
|
12
12
|
import type { SlashingProtectionDatabase, TryInsertOrGetResult } from '../types.js';
|
|
13
13
|
import {
|
|
14
|
+
CLEANUP_OLD_DUTIES,
|
|
15
|
+
CLEANUP_OUTDATED_ROLLUP_DUTIES,
|
|
14
16
|
CLEANUP_OWN_STUCK_DUTIES,
|
|
15
17
|
DELETE_DUTY,
|
|
16
18
|
INSERT_OR_GET_DUTY,
|
|
@@ -101,6 +103,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
101
103
|
const result = await retry<QueryResult<InsertOrGetRow>>(
|
|
102
104
|
async () => {
|
|
103
105
|
const queryResult: QueryResult<InsertOrGetRow> = await this.pool.query(INSERT_OR_GET_DUTY, [
|
|
106
|
+
params.rollupAddress.toString(),
|
|
104
107
|
params.validatorAddress.toString(),
|
|
105
108
|
params.slot.toString(),
|
|
106
109
|
params.blockNumber.toString(),
|
|
@@ -148,6 +151,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
148
151
|
* @returns true if the update succeeded, false if token didn't match or duty not found
|
|
149
152
|
*/
|
|
150
153
|
async updateDutySigned(
|
|
154
|
+
rollupAddress: EthAddress,
|
|
151
155
|
validatorAddress: EthAddress,
|
|
152
156
|
slot: SlotNumber,
|
|
153
157
|
dutyType: DutyType,
|
|
@@ -157,6 +161,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
157
161
|
): Promise<boolean> {
|
|
158
162
|
const result = await this.pool.query(UPDATE_DUTY_SIGNED, [
|
|
159
163
|
signature,
|
|
164
|
+
rollupAddress.toString(),
|
|
160
165
|
validatorAddress.toString(),
|
|
161
166
|
slot.toString(),
|
|
162
167
|
dutyType,
|
|
@@ -166,6 +171,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
166
171
|
|
|
167
172
|
if (result.rowCount === 0) {
|
|
168
173
|
this.log.warn('Failed to update duty to signed status: invalid token or duty not found', {
|
|
174
|
+
rollupAddress: rollupAddress.toString(),
|
|
169
175
|
validatorAddress: validatorAddress.toString(),
|
|
170
176
|
slot: slot.toString(),
|
|
171
177
|
dutyType,
|
|
@@ -184,6 +190,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
184
190
|
* @returns true if the delete succeeded, false if token didn't match or duty not found
|
|
185
191
|
*/
|
|
186
192
|
async deleteDuty(
|
|
193
|
+
rollupAddress: EthAddress,
|
|
187
194
|
validatorAddress: EthAddress,
|
|
188
195
|
slot: SlotNumber,
|
|
189
196
|
dutyType: DutyType,
|
|
@@ -191,6 +198,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
191
198
|
blockIndexWithinCheckpoint: number,
|
|
192
199
|
): Promise<boolean> {
|
|
193
200
|
const result = await this.pool.query(DELETE_DUTY, [
|
|
201
|
+
rollupAddress.toString(),
|
|
194
202
|
validatorAddress.toString(),
|
|
195
203
|
slot.toString(),
|
|
196
204
|
dutyType,
|
|
@@ -200,6 +208,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
200
208
|
|
|
201
209
|
if (result.rowCount === 0) {
|
|
202
210
|
this.log.warn('Failed to delete duty: invalid token or duty not found', {
|
|
211
|
+
rollupAddress: rollupAddress.toString(),
|
|
203
212
|
validatorAddress: validatorAddress.toString(),
|
|
204
213
|
slot: slot.toString(),
|
|
205
214
|
dutyType,
|
|
@@ -215,6 +224,7 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
215
224
|
*/
|
|
216
225
|
private rowToRecord(row: DutyRow): ValidatorDutyRecord {
|
|
217
226
|
return {
|
|
227
|
+
rollupAddress: EthAddress.fromString(row.rollup_address),
|
|
218
228
|
validatorAddress: EthAddress.fromString(row.validator_address),
|
|
219
229
|
slot: SlotNumber.fromString(row.slot),
|
|
220
230
|
blockNumber: BlockNumber.fromString(row.block_number),
|
|
@@ -244,8 +254,29 @@ export class PostgresSlashingProtectionDatabase implements SlashingProtectionDat
|
|
|
244
254
|
* @returns the number of duties cleaned up
|
|
245
255
|
*/
|
|
246
256
|
async cleanupOwnStuckDuties(nodeId: string, maxAgeMs: number): Promise<number> {
|
|
247
|
-
const
|
|
248
|
-
|
|
257
|
+
const result = await this.pool.query(CLEANUP_OWN_STUCK_DUTIES, [nodeId, maxAgeMs]);
|
|
258
|
+
return result.rowCount ?? 0;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Cleanup duties with outdated rollup address.
|
|
263
|
+
* Removes all duties where the rollup address doesn't match the current one.
|
|
264
|
+
* Used after a rollup upgrade to clean up duties for the old rollup.
|
|
265
|
+
* @returns the number of duties cleaned up
|
|
266
|
+
*/
|
|
267
|
+
async cleanupOutdatedRollupDuties(currentRollupAddress: EthAddress): Promise<number> {
|
|
268
|
+
const result = await this.pool.query(CLEANUP_OUTDATED_ROLLUP_DUTIES, [currentRollupAddress.toString()]);
|
|
269
|
+
return result.rowCount ?? 0;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Cleanup old signed duties.
|
|
274
|
+
* Removes only signed duties older than the specified age.
|
|
275
|
+
* Does not remove 'signing' duties as they may be in progress.
|
|
276
|
+
* @returns the number of duties cleaned up
|
|
277
|
+
*/
|
|
278
|
+
async cleanupOldDuties(maxAgeMs: number): Promise<number> {
|
|
279
|
+
const result = await this.pool.query(CLEANUP_OLD_DUTIES, [maxAgeMs]);
|
|
249
280
|
return result.rowCount ?? 0;
|
|
250
281
|
}
|
|
251
282
|
}
|
package/src/db/schema.ts
CHANGED
|
@@ -16,12 +16,13 @@ export const SCHEMA_VERSION = 1;
|
|
|
16
16
|
*/
|
|
17
17
|
export const CREATE_VALIDATOR_DUTIES_TABLE = `
|
|
18
18
|
CREATE TABLE IF NOT EXISTS validator_duties (
|
|
19
|
+
rollup_address VARCHAR(42) NOT NULL,
|
|
19
20
|
validator_address VARCHAR(42) NOT NULL,
|
|
20
21
|
slot BIGINT NOT NULL,
|
|
21
22
|
block_number BIGINT NOT NULL,
|
|
22
23
|
block_index_within_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
23
24
|
duty_type VARCHAR(30) NOT NULL CHECK (duty_type IN ('BLOCK_PROPOSAL', 'CHECKPOINT_PROPOSAL', 'ATTESTATION', 'ATTESTATIONS_AND_SIGNERS', 'GOVERNANCE_VOTE', 'SLASHING_VOTE')),
|
|
24
|
-
status VARCHAR(20) NOT NULL CHECK (status IN ('signing', 'signed'
|
|
25
|
+
status VARCHAR(20) NOT NULL CHECK (status IN ('signing', 'signed')),
|
|
25
26
|
message_hash VARCHAR(66) NOT NULL,
|
|
26
27
|
signature VARCHAR(132),
|
|
27
28
|
node_id VARCHAR(255) NOT NULL,
|
|
@@ -30,7 +31,7 @@ CREATE TABLE IF NOT EXISTS validator_duties (
|
|
|
30
31
|
completed_at TIMESTAMP,
|
|
31
32
|
error_message TEXT,
|
|
32
33
|
|
|
33
|
-
PRIMARY KEY (validator_address, slot, duty_type, block_index_within_checkpoint),
|
|
34
|
+
PRIMARY KEY (rollup_address, validator_address, slot, duty_type, block_index_within_checkpoint),
|
|
34
35
|
CHECK (completed_at IS NULL OR completed_at >= started_at)
|
|
35
36
|
);
|
|
36
37
|
`;
|
|
@@ -101,6 +102,7 @@ SELECT version FROM schema_version ORDER BY version DESC LIMIT 1;
|
|
|
101
102
|
export const INSERT_OR_GET_DUTY = `
|
|
102
103
|
WITH inserted AS (
|
|
103
104
|
INSERT INTO validator_duties (
|
|
105
|
+
rollup_address,
|
|
104
106
|
validator_address,
|
|
105
107
|
slot,
|
|
106
108
|
block_number,
|
|
@@ -111,9 +113,10 @@ WITH inserted AS (
|
|
|
111
113
|
node_id,
|
|
112
114
|
lock_token,
|
|
113
115
|
started_at
|
|
114
|
-
) VALUES ($1, $2, $3, $4, $5, 'signing', $
|
|
115
|
-
ON CONFLICT (validator_address, slot, duty_type, block_index_within_checkpoint) DO NOTHING
|
|
116
|
+
) VALUES ($1, $2, $3, $4, $5, $6, 'signing', $7, $8, $9, CURRENT_TIMESTAMP)
|
|
117
|
+
ON CONFLICT (rollup_address, validator_address, slot, duty_type, block_index_within_checkpoint) DO NOTHING
|
|
116
118
|
RETURNING
|
|
119
|
+
rollup_address,
|
|
117
120
|
validator_address,
|
|
118
121
|
slot,
|
|
119
122
|
block_number,
|
|
@@ -132,6 +135,7 @@ WITH inserted AS (
|
|
|
132
135
|
SELECT * FROM inserted
|
|
133
136
|
UNION ALL
|
|
134
137
|
SELECT
|
|
138
|
+
rollup_address,
|
|
135
139
|
validator_address,
|
|
136
140
|
slot,
|
|
137
141
|
block_number,
|
|
@@ -147,10 +151,11 @@ SELECT
|
|
|
147
151
|
error_message,
|
|
148
152
|
FALSE as is_new
|
|
149
153
|
FROM validator_duties
|
|
150
|
-
WHERE
|
|
151
|
-
AND
|
|
152
|
-
AND
|
|
153
|
-
AND
|
|
154
|
+
WHERE rollup_address = $1
|
|
155
|
+
AND validator_address = $2
|
|
156
|
+
AND slot = $3
|
|
157
|
+
AND duty_type = $6
|
|
158
|
+
AND block_index_within_checkpoint = $5
|
|
154
159
|
AND NOT EXISTS (SELECT 1 FROM inserted);
|
|
155
160
|
`;
|
|
156
161
|
|
|
@@ -162,12 +167,13 @@ UPDATE validator_duties
|
|
|
162
167
|
SET status = 'signed',
|
|
163
168
|
signature = $1,
|
|
164
169
|
completed_at = CURRENT_TIMESTAMP
|
|
165
|
-
WHERE
|
|
166
|
-
AND
|
|
167
|
-
AND
|
|
168
|
-
AND
|
|
170
|
+
WHERE rollup_address = $2
|
|
171
|
+
AND validator_address = $3
|
|
172
|
+
AND slot = $4
|
|
173
|
+
AND duty_type = $5
|
|
174
|
+
AND block_index_within_checkpoint = $6
|
|
169
175
|
AND status = 'signing'
|
|
170
|
-
AND lock_token = $
|
|
176
|
+
AND lock_token = $7;
|
|
171
177
|
`;
|
|
172
178
|
|
|
173
179
|
/**
|
|
@@ -176,12 +182,13 @@ WHERE validator_address = $2
|
|
|
176
182
|
*/
|
|
177
183
|
export const DELETE_DUTY = `
|
|
178
184
|
DELETE FROM validator_duties
|
|
179
|
-
WHERE
|
|
180
|
-
AND
|
|
181
|
-
AND
|
|
182
|
-
AND
|
|
185
|
+
WHERE rollup_address = $1
|
|
186
|
+
AND validator_address = $2
|
|
187
|
+
AND slot = $3
|
|
188
|
+
AND duty_type = $4
|
|
189
|
+
AND block_index_within_checkpoint = $5
|
|
183
190
|
AND status = 'signing'
|
|
184
|
-
AND lock_token = $
|
|
191
|
+
AND lock_token = $6;
|
|
185
192
|
`;
|
|
186
193
|
|
|
187
194
|
/**
|
|
@@ -196,23 +203,34 @@ WHERE status = 'signed'
|
|
|
196
203
|
|
|
197
204
|
/**
|
|
198
205
|
* Query to clean up old duties (for maintenance)
|
|
199
|
-
* Removes duties older than a specified
|
|
206
|
+
* Removes SIGNED duties older than a specified age (in milliseconds)
|
|
200
207
|
*/
|
|
201
208
|
export const CLEANUP_OLD_DUTIES = `
|
|
202
209
|
DELETE FROM validator_duties
|
|
203
|
-
WHERE status
|
|
204
|
-
AND started_at < $1;
|
|
210
|
+
WHERE status = 'signed'
|
|
211
|
+
AND started_at < CURRENT_TIMESTAMP - ($1 || ' milliseconds')::INTERVAL;
|
|
205
212
|
`;
|
|
206
213
|
|
|
207
214
|
/**
|
|
208
215
|
* Query to cleanup own stuck duties
|
|
209
216
|
* Removes duties in 'signing' status for a specific node that are older than maxAgeMs
|
|
217
|
+
* Uses DB's CURRENT_TIMESTAMP to avoid clock skew issues between nodes
|
|
210
218
|
*/
|
|
211
219
|
export const CLEANUP_OWN_STUCK_DUTIES = `
|
|
212
220
|
DELETE FROM validator_duties
|
|
213
221
|
WHERE node_id = $1
|
|
214
222
|
AND status = 'signing'
|
|
215
|
-
AND started_at < $2;
|
|
223
|
+
AND started_at < CURRENT_TIMESTAMP - ($2 || ' milliseconds')::INTERVAL;
|
|
224
|
+
`;
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Query to cleanup duties with outdated rollup address
|
|
228
|
+
* Removes all duties where the rollup address doesn't match the current one
|
|
229
|
+
* Used after a rollup upgrade to clean up duties for the old rollup
|
|
230
|
+
*/
|
|
231
|
+
export const CLEANUP_OUTDATED_ROLLUP_DUTIES = `
|
|
232
|
+
DELETE FROM validator_duties
|
|
233
|
+
WHERE rollup_address != $1;
|
|
216
234
|
`;
|
|
217
235
|
|
|
218
236
|
/**
|
|
@@ -231,6 +249,7 @@ export const DROP_SCHEMA_VERSION_TABLE = `DROP TABLE IF EXISTS schema_version;`;
|
|
|
231
249
|
*/
|
|
232
250
|
export const GET_STUCK_DUTIES = `
|
|
233
251
|
SELECT
|
|
252
|
+
rollup_address,
|
|
234
253
|
validator_address,
|
|
235
254
|
slot,
|
|
236
255
|
block_number,
|
package/src/db/types.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
import type { BlockNumber, CheckpointNumber, IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types';
|
|
2
2
|
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
3
3
|
import type { Signature } from '@aztec/foundation/eth-signature';
|
|
4
|
+
import { DutyType } from '@aztec/stdlib/ha-signing';
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* Row type from PostgreSQL query
|
|
7
8
|
*/
|
|
8
9
|
export interface DutyRow {
|
|
10
|
+
rollup_address: string;
|
|
9
11
|
validator_address: string;
|
|
10
12
|
slot: string;
|
|
11
13
|
block_number: string;
|
|
@@ -28,20 +30,6 @@ export interface InsertOrGetRow extends DutyRow {
|
|
|
28
30
|
is_new: boolean;
|
|
29
31
|
}
|
|
30
32
|
|
|
31
|
-
/**
|
|
32
|
-
* Type of validator duty being performed
|
|
33
|
-
*/
|
|
34
|
-
export enum DutyType {
|
|
35
|
-
BLOCK_PROPOSAL = 'BLOCK_PROPOSAL',
|
|
36
|
-
CHECKPOINT_PROPOSAL = 'CHECKPOINT_PROPOSAL',
|
|
37
|
-
ATTESTATION = 'ATTESTATION',
|
|
38
|
-
ATTESTATIONS_AND_SIGNERS = 'ATTESTATIONS_AND_SIGNERS',
|
|
39
|
-
GOVERNANCE_VOTE = 'GOVERNANCE_VOTE',
|
|
40
|
-
SLASHING_VOTE = 'SLASHING_VOTE',
|
|
41
|
-
AUTH_REQUEST = 'AUTH_REQUEST',
|
|
42
|
-
TXS = 'TXS',
|
|
43
|
-
}
|
|
44
|
-
|
|
45
33
|
/**
|
|
46
34
|
* Status of a duty in the database
|
|
47
35
|
*/
|
|
@@ -50,10 +38,15 @@ export enum DutyStatus {
|
|
|
50
38
|
SIGNED = 'signed',
|
|
51
39
|
}
|
|
52
40
|
|
|
41
|
+
// Re-export DutyType from stdlib
|
|
42
|
+
export { DutyType };
|
|
43
|
+
|
|
53
44
|
/**
|
|
54
45
|
* Record of a validator duty in the database
|
|
55
46
|
*/
|
|
56
47
|
export interface ValidatorDutyRecord {
|
|
48
|
+
/** Ethereum address of the rollup contract */
|
|
49
|
+
rollupAddress: EthAddress;
|
|
57
50
|
/** Ethereum address of the validator */
|
|
58
51
|
validatorAddress: EthAddress;
|
|
59
52
|
/** Slot number for this duty */
|
|
@@ -78,7 +71,7 @@ export interface ValidatorDutyRecord {
|
|
|
78
71
|
startedAt: Date;
|
|
79
72
|
/** When the duty signing was completed (success or failure) */
|
|
80
73
|
completedAt?: Date;
|
|
81
|
-
/** Error message
|
|
74
|
+
/** Error message (currently unused) */
|
|
82
75
|
errorMessage?: string;
|
|
83
76
|
}
|
|
84
77
|
|
|
@@ -87,6 +80,7 @@ export interface ValidatorDutyRecord {
|
|
|
87
80
|
* blockIndexWithinCheckpoint is REQUIRED and must be >= 0.
|
|
88
81
|
*/
|
|
89
82
|
export interface BlockProposalDutyIdentifier {
|
|
83
|
+
rollupAddress: EthAddress;
|
|
90
84
|
validatorAddress: EthAddress;
|
|
91
85
|
slot: SlotNumber;
|
|
92
86
|
/** Block index within checkpoint (0, 1, 2...). Required for block proposals. */
|
|
@@ -99,6 +93,7 @@ export interface BlockProposalDutyIdentifier {
|
|
|
99
93
|
* blockIndexWithinCheckpoint is not applicable (internally stored as -1).
|
|
100
94
|
*/
|
|
101
95
|
export interface OtherDutyIdentifier {
|
|
96
|
+
rollupAddress: EthAddress;
|
|
102
97
|
validatorAddress: EthAddress;
|
|
103
98
|
slot: SlotNumber;
|
|
104
99
|
dutyType:
|
package/src/factory.ts
CHANGED
|
@@ -1,10 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Factory functions for creating validator HA signers
|
|
3
3
|
*/
|
|
4
|
+
import { DateProvider } from '@aztec/foundation/timer';
|
|
5
|
+
import type { ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
|
|
6
|
+
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
7
|
+
|
|
4
8
|
import { Pool } from 'pg';
|
|
5
9
|
|
|
6
|
-
import type { ValidatorHASignerConfig } from './config.js';
|
|
7
10
|
import { PostgresSlashingProtectionDatabase } from './db/postgres.js';
|
|
11
|
+
import { HASignerMetrics } from './metrics.js';
|
|
8
12
|
import type { CreateHASignerDeps, SlashingProtectionDatabase } from './types.js';
|
|
9
13
|
import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
10
14
|
|
|
@@ -55,6 +59,10 @@ export async function createHASigner(
|
|
|
55
59
|
if (!databaseUrl) {
|
|
56
60
|
throw new Error('databaseUrl is required for createHASigner');
|
|
57
61
|
}
|
|
62
|
+
|
|
63
|
+
const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
|
|
64
|
+
const dateProvider = deps?.dateProvider ?? new DateProvider();
|
|
65
|
+
|
|
58
66
|
// Create connection pool (or use provided pool)
|
|
59
67
|
let pool: Pool;
|
|
60
68
|
if (!deps?.pool) {
|
|
@@ -75,8 +83,11 @@ export async function createHASigner(
|
|
|
75
83
|
// Verify database schema is initialized and version matches
|
|
76
84
|
await db.initialize();
|
|
77
85
|
|
|
86
|
+
// Create metrics
|
|
87
|
+
const metrics = new HASignerMetrics(telemetryClient, signerConfig.nodeId);
|
|
88
|
+
|
|
78
89
|
// Create signer
|
|
79
|
-
const signer = new ValidatorHASigner(db, { ...signerConfig, databaseUrl });
|
|
90
|
+
const signer = new ValidatorHASigner(db, { ...signerConfig, databaseUrl }, { metrics, dateProvider });
|
|
80
91
|
|
|
81
92
|
return { signer, db };
|
|
82
93
|
}
|
package/src/metrics.ts
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Attributes,
|
|
3
|
+
type Histogram,
|
|
4
|
+
Metrics,
|
|
5
|
+
type TelemetryClient,
|
|
6
|
+
type UpDownCounter,
|
|
7
|
+
createUpDownCounterWithDefault,
|
|
8
|
+
} from '@aztec/telemetry-client';
|
|
9
|
+
|
|
10
|
+
export type HACleanupType = 'stuck' | 'old' | 'outdated_rollup';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Metrics for HA signer tracking signing operations, lock acquisition, and cleanup.
|
|
14
|
+
*/
|
|
15
|
+
export class HASignerMetrics {
|
|
16
|
+
// Signing lifecycle metrics
|
|
17
|
+
private signingDuration: Histogram;
|
|
18
|
+
private signingSuccessCount: UpDownCounter;
|
|
19
|
+
private dutyAlreadySignedCount: UpDownCounter;
|
|
20
|
+
private slashingProtectionCount: UpDownCounter;
|
|
21
|
+
private signingErrorCount: UpDownCounter;
|
|
22
|
+
|
|
23
|
+
// Lock acquisition metrics
|
|
24
|
+
private lockAcquiredCount: UpDownCounter;
|
|
25
|
+
|
|
26
|
+
// Cleanup metrics
|
|
27
|
+
private cleanupStuckDutiesCount: UpDownCounter;
|
|
28
|
+
private cleanupOldDutiesCount: UpDownCounter;
|
|
29
|
+
private cleanupOutdatedRollupDutiesCount: UpDownCounter;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
client: TelemetryClient,
|
|
33
|
+
private nodeId: string,
|
|
34
|
+
name = 'HASignerMetrics',
|
|
35
|
+
) {
|
|
36
|
+
const meter = client.getMeter(name);
|
|
37
|
+
|
|
38
|
+
// Signing lifecycle
|
|
39
|
+
this.signingDuration = meter.createHistogram(Metrics.HA_SIGNER_SIGNING_DURATION);
|
|
40
|
+
this.signingSuccessCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_SUCCESS_COUNT);
|
|
41
|
+
this.dutyAlreadySignedCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_DUTY_ALREADY_SIGNED_COUNT);
|
|
42
|
+
this.slashingProtectionCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SLASHING_PROTECTION_COUNT);
|
|
43
|
+
this.signingErrorCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_ERROR_COUNT);
|
|
44
|
+
|
|
45
|
+
// Lock acquisition
|
|
46
|
+
this.lockAcquiredCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_LOCK_ACQUIRED_COUNT);
|
|
47
|
+
|
|
48
|
+
// Cleanup
|
|
49
|
+
this.cleanupStuckDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_STUCK_DUTIES_COUNT);
|
|
50
|
+
this.cleanupOldDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_OLD_DUTIES_COUNT);
|
|
51
|
+
this.cleanupOutdatedRollupDutiesCount = createUpDownCounterWithDefault(
|
|
52
|
+
meter,
|
|
53
|
+
Metrics.HA_SIGNER_CLEANUP_OUTDATED_ROLLUP_DUTIES_COUNT,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Record a successful signing operation.
|
|
59
|
+
* @param dutyType - The type of duty signed
|
|
60
|
+
* @param durationMs - Duration from start of signWithProtection to completion
|
|
61
|
+
*/
|
|
62
|
+
public recordSigningSuccess(dutyType: string, durationMs: number): void {
|
|
63
|
+
const attributes = {
|
|
64
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
65
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
66
|
+
};
|
|
67
|
+
this.signingSuccessCount.add(1, attributes);
|
|
68
|
+
this.signingDuration.record(durationMs, attributes);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Record a DutyAlreadySignedError (expected in HA; another node signed first).
|
|
73
|
+
* @param dutyType - The type of duty
|
|
74
|
+
*/
|
|
75
|
+
public recordDutyAlreadySigned(dutyType: string): void {
|
|
76
|
+
const attributes = {
|
|
77
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
78
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
79
|
+
};
|
|
80
|
+
this.dutyAlreadySignedCount.add(1, attributes);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Record a SlashingProtectionError (attempted to sign different data for same duty).
|
|
85
|
+
* @param dutyType - The type of duty
|
|
86
|
+
*/
|
|
87
|
+
public recordSlashingProtection(dutyType: string): void {
|
|
88
|
+
const attributes = {
|
|
89
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
90
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
91
|
+
};
|
|
92
|
+
this.slashingProtectionCount.add(1, attributes);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Record a signing function failure (lock will be deleted for retry).
|
|
97
|
+
* @param dutyType - The type of duty
|
|
98
|
+
*/
|
|
99
|
+
public recordSigningError(dutyType: string): void {
|
|
100
|
+
const attributes = {
|
|
101
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
102
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
103
|
+
};
|
|
104
|
+
this.signingErrorCount.add(1, attributes);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Record lock acquisition.
|
|
109
|
+
* @param acquired - Whether a new lock was acquired (true) or existing record found (false)
|
|
110
|
+
*/
|
|
111
|
+
public recordLockAcquire(acquired: boolean): void {
|
|
112
|
+
if (acquired) {
|
|
113
|
+
const attributes = {
|
|
114
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
115
|
+
};
|
|
116
|
+
this.lockAcquiredCount.add(1, attributes);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Record cleanup metrics.
|
|
122
|
+
* @param type - Type of cleanup
|
|
123
|
+
* @param count - Number of duties cleaned up
|
|
124
|
+
*/
|
|
125
|
+
public recordCleanup(type: HACleanupType, count: number): void {
|
|
126
|
+
const attributes = {
|
|
127
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
if (type === 'stuck') {
|
|
131
|
+
this.cleanupStuckDutiesCount.add(count, attributes);
|
|
132
|
+
} else if (type === 'old') {
|
|
133
|
+
this.cleanupOldDutiesCount.add(count, attributes);
|
|
134
|
+
} else if (type === 'outdated_rollup') {
|
|
135
|
+
this.cleanupOutdatedRollupDutiesCount.add(count, attributes);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|