@aztec/validator-ha-signer 5.0.0-rc.2 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dest/errors.d.ts +14 -1
- package/dest/errors.d.ts.map +1 -1
- package/dest/errors.js +15 -0
- package/dest/factory.d.ts +7 -9
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +28 -6
- package/dest/slashing_protection_service.d.ts +4 -2
- package/dest/slashing_protection_service.d.ts.map +1 -1
- package/dest/slashing_protection_service.js +10 -7
- package/dest/validator_ha_signer.d.ts +3 -2
- package/dest/validator_ha_signer.d.ts.map +1 -1
- package/dest/validator_ha_signer.js +29 -6
- package/package.json +6 -6
- package/src/errors.ts +21 -0
- package/src/factory.ts +34 -5
- package/src/slashing_protection_service.ts +12 -7
- package/src/validator_ha_signer.ts +43 -6
package/README.md
CHANGED
|
@@ -38,7 +38,7 @@ const { signer, db } = await createHASigner({
|
|
|
38
38
|
databaseUrl: process.env.DATABASE_URL,
|
|
39
39
|
nodeId: 'validator-node-1',
|
|
40
40
|
pollingIntervalMs: 100,
|
|
41
|
-
|
|
41
|
+
peerSigningTimeoutMs: 3000,
|
|
42
42
|
});
|
|
43
43
|
|
|
44
44
|
// Start background cleanup tasks
|
|
@@ -82,7 +82,7 @@ await db.initialize();
|
|
|
82
82
|
const signer = new ValidatorHASigner(db, {
|
|
83
83
|
nodeId: 'validator-node-1',
|
|
84
84
|
pollingIntervalMs: 100,
|
|
85
|
-
|
|
85
|
+
peerSigningTimeoutMs: 3000,
|
|
86
86
|
maxStuckDutiesAgeMs: 144000,
|
|
87
87
|
});
|
|
88
88
|
|
package/dest/errors.d.ts
CHANGED
|
@@ -15,6 +15,19 @@ export declare class DutyAlreadySignedError extends Error {
|
|
|
15
15
|
readonly signedByNode: string;
|
|
16
16
|
constructor(slot: SlotNumber, dutyType: DutyType, blockIndexWithinCheckpoint: number, signedByNode: string);
|
|
17
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* Thrown when the slashing-protection record for an in-flight signing operation can no longer be
|
|
20
|
+
* updated because it is no longer owned by this node - for example, the stuck-duty cleanup loop
|
|
21
|
+
* deleted the SIGNING row while the remote signer was slow. The produced signature must be
|
|
22
|
+
* discarded rather than broadcast: with no protection record in place, a later attempt for the
|
|
23
|
+
* same duty with different data would sign freely, which is slashable equivocation.
|
|
24
|
+
*/
|
|
25
|
+
export declare class SigningLockLostError extends Error {
|
|
26
|
+
readonly slot: SlotNumber;
|
|
27
|
+
readonly dutyType: DutyType;
|
|
28
|
+
readonly nodeId: string;
|
|
29
|
+
constructor(slot: SlotNumber, dutyType: DutyType, nodeId: string);
|
|
30
|
+
}
|
|
18
31
|
/**
|
|
19
32
|
* Thrown when attempting to sign data that conflicts with an already-signed duty.
|
|
20
33
|
* This means the same validator tried to sign DIFFERENT data for the same slot.
|
|
@@ -31,4 +44,4 @@ export declare class SlashingProtectionError extends Error {
|
|
|
31
44
|
readonly signedByNode: string;
|
|
32
45
|
constructor(slot: SlotNumber, dutyType: DutyType, blockIndexWithinCheckpoint: number, existingMessageHash: string, attemptedMessageHash: string, signedByNode: string);
|
|
33
46
|
}
|
|
34
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
47
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZXJyb3JzLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvZXJyb3JzLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsT0FBTyxLQUFLLEVBQUUsVUFBVSxFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFFbEUsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLE1BQU0sZUFBZSxDQUFDO0FBRTlDOzs7O0dBSUc7QUFDSCxxQkFBYSxzQkFBdUIsU0FBUSxLQUFLO2FBRTdCLElBQUksRUFBRSxVQUFVO2FBQ2hCLFFBQVEsRUFBRSxRQUFRO2FBQ2xCLDBCQUEwQixFQUFFLE1BQU07YUFDbEMsWUFBWSxFQUFFLE1BQU07SUFKdEMsWUFDa0IsSUFBSSxFQUFFLFVBQVUsRUFDaEIsUUFBUSxFQUFFLFFBQVEsRUFDbEIsMEJBQTBCLEVBQUUsTUFBTSxFQUNsQyxZQUFZLEVBQUUsTUFBTSxFQUlyQztDQUNGO0FBRUQ7Ozs7OztHQU1HO0FBQ0gscUJBQWEsb0JBQXFCLFNBQVEsS0FBSzthQUUzQixJQUFJLEVBQUUsVUFBVTthQUNoQixRQUFRLEVBQUUsUUFBUTthQUNsQixNQUFNLEVBQUUsTUFBTTtJQUhoQyxZQUNrQixJQUFJLEVBQUUsVUFBVSxFQUNoQixRQUFRLEVBQUUsUUFBUSxFQUNsQixNQUFNLEVBQUUsTUFBTSxFQU8vQjtDQUNGO0FBRUQ7Ozs7OztHQU1HO0FBQ0gscUJBQWEsdUJBQXdCLFNBQVEsS0FBSzthQUU5QixJQUFJLEVBQUUsVUFBVTthQUNoQixRQUFRLEVBQUUsUUFBUTthQUNsQiwwQkFBMEIsRUFBRSxNQUFNO2FBQ2xDLG1CQUFtQixFQUFFLE1BQU07YUFDM0Isb0JBQW9CLEVBQUUsTUFBTTthQUM1QixZQUFZLEVBQUUsTUFBTTtJQU50QyxZQUNrQixJQUFJLEVBQUUsVUFBVSxFQUNoQixRQUFRLEVBQUUsUUFBUSxFQUNsQiwwQkFBMEIsRUFBRSxNQUFNLEVBQ2xDLG1CQUFtQixFQUFFLE1BQU0sRUFDM0Isb0JBQW9CLEVBQUUsTUFBTSxFQUM1QixZQUFZLEVBQUUsTUFBTSxFQU9yQztDQUNGIn0=
|
package/dest/errors.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAElE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;aAE7B,IAAI,EAAE,UAAU;aAChB,QAAQ,EAAE,QAAQ;aAClB,0BAA0B,EAAE,MAAM;aAClC,YAAY,EAAE,MAAM;IAJtC,YACkB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,MAAM,EAClC,YAAY,EAAE,MAAM,EAIrC;CACF;AAED;;;;;;GAMG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;aAE9B,IAAI,EAAE,UAAU;aAChB,QAAQ,EAAE,QAAQ;aAClB,0BAA0B,EAAE,MAAM;aAClC,mBAAmB,EAAE,MAAM;aAC3B,oBAAoB,EAAE,MAAM;aAC5B,YAAY,EAAE,MAAM;IANtC,YACkB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,MAAM,EAClC,mBAAmB,EAAE,MAAM,EAC3B,oBAAoB,EAAE,MAAM,EAC5B,YAAY,EAAE,MAAM,EAOrC;CACF"}
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iCAAiC,CAAC;AAElE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C;;;;GAIG;AACH,qBAAa,sBAAuB,SAAQ,KAAK;aAE7B,IAAI,EAAE,UAAU;aAChB,QAAQ,EAAE,QAAQ;aAClB,0BAA0B,EAAE,MAAM;aAClC,YAAY,EAAE,MAAM;IAJtC,YACkB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,MAAM,EAClC,YAAY,EAAE,MAAM,EAIrC;CACF;AAED;;;;;;GAMG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAE3B,IAAI,EAAE,UAAU;aAChB,QAAQ,EAAE,QAAQ;aAClB,MAAM,EAAE,MAAM;IAHhC,YACkB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,QAAQ,EAClB,MAAM,EAAE,MAAM,EAO/B;CACF;AAED;;;;;;GAMG;AACH,qBAAa,uBAAwB,SAAQ,KAAK;aAE9B,IAAI,EAAE,UAAU;aAChB,QAAQ,EAAE,QAAQ;aAClB,0BAA0B,EAAE,MAAM;aAClC,mBAAmB,EAAE,MAAM;aAC3B,oBAAoB,EAAE,MAAM;aAC5B,YAAY,EAAE,MAAM;IANtC,YACkB,IAAI,EAAE,UAAU,EAChB,QAAQ,EAAE,QAAQ,EAClB,0BAA0B,EAAE,MAAM,EAClC,mBAAmB,EAAE,MAAM,EAC3B,oBAAoB,EAAE,MAAM,EAC5B,YAAY,EAAE,MAAM,EAOrC;CACF"}
|
package/dest/errors.js
CHANGED
|
@@ -14,6 +14,21 @@
|
|
|
14
14
|
this.name = 'DutyAlreadySignedError';
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
|
+
/**
|
|
18
|
+
* Thrown when the slashing-protection record for an in-flight signing operation can no longer be
|
|
19
|
+
* updated because it is no longer owned by this node - for example, the stuck-duty cleanup loop
|
|
20
|
+
* deleted the SIGNING row while the remote signer was slow. The produced signature must be
|
|
21
|
+
* discarded rather than broadcast: with no protection record in place, a later attempt for the
|
|
22
|
+
* same duty with different data would sign freely, which is slashable equivocation.
|
|
23
|
+
*/ export class SigningLockLostError extends Error {
|
|
24
|
+
slot;
|
|
25
|
+
dutyType;
|
|
26
|
+
nodeId;
|
|
27
|
+
constructor(slot, dutyType, nodeId){
|
|
28
|
+
super(`Slashing protection record for ${dutyType} at slot ${slot} was lost before signing completed ` + `(node ${nodeId}); discarding signature`), this.slot = slot, this.dutyType = dutyType, this.nodeId = nodeId;
|
|
29
|
+
this.name = 'SigningLockLostError';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
17
32
|
/**
|
|
18
33
|
* Thrown when attempting to sign data that conflicts with an already-signed duty.
|
|
19
34
|
* This means the same validator tried to sign DIFFERENT data for the same slot.
|
package/dest/factory.d.ts
CHANGED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Factory functions for creating validator HA signers
|
|
3
|
-
*/
|
|
4
1
|
import { DateProvider } from '@aztec/foundation/timer';
|
|
5
2
|
import type { LocalSignerConfig, ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
|
|
6
3
|
import type { CreateHASignerDeps, CreateLocalSignerWithProtectionDeps, SlashingProtectionDatabase } from './types.js';
|
|
@@ -22,7 +19,7 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
22
19
|
* databaseUrl: process.env.DATABASE_URL,
|
|
23
20
|
* nodeId: 'validator-node-1',
|
|
24
21
|
* pollingIntervalMs: 100,
|
|
25
|
-
*
|
|
22
|
+
* peerSigningTimeoutMs: 3000,
|
|
26
23
|
* });
|
|
27
24
|
* signer.start(); // Start background cleanup
|
|
28
25
|
*
|
|
@@ -49,9 +46,10 @@ export declare function createHASigner(config: ValidatorHASignerConfig, deps?: C
|
|
|
49
46
|
* high-availability (multi-node) setup. It prevents a proposer from sending two
|
|
50
47
|
* proposals for the same slot if the node crashes and restarts mid-proposal.
|
|
51
48
|
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
49
|
+
* `config.dataDirectory` is required so the protection database is persisted to disk and survives
|
|
50
|
+
* crashes/restarts. Booting without it throws, since an ephemeral store silently drops all
|
|
51
|
+
* double-signing protection across restarts. Set `config.allowEphemeralSigningProtection` to opt
|
|
52
|
+
* into the ephemeral store anyway (dev/test networks only) — a loud warning is logged in that case.
|
|
55
53
|
*
|
|
56
54
|
* @param config - Local signer config
|
|
57
55
|
* @param deps - Optional dependencies (telemetry, date provider).
|
|
@@ -70,8 +68,8 @@ export declare function createSharedSlashingProtectionDb(dateProvider?: DateProv
|
|
|
70
68
|
* Create a ValidatorHASigner backed by a pre-existing SlashingProtectionDatabase.
|
|
71
69
|
* Used for testing HA setups where multiple nodes share the same protection database.
|
|
72
70
|
*/
|
|
73
|
-
export declare function createSignerFromSharedDb(db: SlashingProtectionDatabase, config: Pick<ValidatorHASignerConfig, 'nodeId' | 'pollingIntervalMs' | '
|
|
71
|
+
export declare function createSignerFromSharedDb(db: SlashingProtectionDatabase, config: Pick<ValidatorHASignerConfig, 'nodeId' | 'pollingIntervalMs' | 'peerSigningTimeoutMs' | 'maxStuckDutiesAgeMs' | 'rollupAddress'>, deps?: CreateLocalSignerWithProtectionDeps): {
|
|
74
72
|
signer: ValidatorHASigner;
|
|
75
73
|
db: SlashingProtectionDatabase;
|
|
76
74
|
};
|
|
77
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
75
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiZmFjdG9yeS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL2ZhY3RvcnkudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBSUEsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBRXZELE9BQU8sS0FBSyxFQUFFLGlCQUFpQixFQUFFLHVCQUF1QixFQUFFLE1BQU0sMEJBQTBCLENBQUM7QUFRM0YsT0FBTyxLQUFLLEVBQUUsa0JBQWtCLEVBQUUsbUNBQW1DLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxZQUFZLENBQUM7QUFDdEgsT0FBTyxFQUFFLGlCQUFpQixFQUFFLE1BQU0sMEJBQTBCLENBQUM7QUFFN0Q7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBZ0NHO0FBQ0gsd0JBQXNCLGNBQWMsQ0FDbEMsTUFBTSxFQUFFLHVCQUF1QixFQUMvQixJQUFJLENBQUMsRUFBRSxrQkFBa0IsR0FDeEIsT0FBTyxDQUFDO0lBQ1QsTUFBTSxFQUFFLGlCQUFpQixDQUFDO0lBQzFCLEVBQUUsRUFBRSwwQkFBMEIsQ0FBQztDQUNoQyxDQUFDLENBaUREO0FBRUQ7Ozs7Ozs7Ozs7Ozs7OztHQWVHO0FBQ0gsd0JBQXNCLCtCQUErQixDQUNuRCxNQUFNLEVBQUUsaUJBQWlCLEVBQ3pCLElBQUksQ0FBQyxFQUFFLG1DQUFtQyxHQUN6QyxPQUFPLENBQUM7SUFDVCxNQUFNLEVBQUUsaUJBQWlCLENBQUM7SUFDMUIsRUFBRSxFQUFFLDBCQUEwQixDQUFDO0NBQ2hDLENBQUMsQ0FzREQ7QUFFRDs7O0dBR0c7QUFDSCx3QkFBc0IsZ0NBQWdDLENBQ3BELFlBQVksR0FBRSxZQUFpQyxHQUM5QyxPQUFPLENBQUMsMEJBQTBCLENBQUMsQ0FLckM7QUFFRDs7O0dBR0c7QUFDSCx3QkFBZ0Isd0JBQXdCLENBQ3RDLEVBQUUsRUFBRSwwQkFBMEIsRUFDOUIsTUFBTSxFQUFFLElBQUksQ0FDVix1QkFBdUIsRUFDdkIsUUFBUSxHQUFHLG1CQUFtQixHQUFHLHNCQUFzQixHQUFHLHFCQUFxQixHQUFHLGVBQWUsQ0FDbEcsRUFDRCxJQUFJLENBQUMsRUFBRSxtQ0FBbUMsR0FDekM7SUFBRSxNQUFNLEVBQUUsaUJBQWlCLENBQUM7SUFBQyxFQUFFLEVBQUUsMEJBQTBCLENBQUE7Q0FBRSxDQU0vRCJ9
|
package/dest/factory.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../src/factory.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../src/factory.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,0BAA0B,CAAC;AAQ3F,OAAO,KAAK,EAAE,kBAAkB,EAAE,mCAAmC,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAC;AACtH,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAsB,cAAc,CAClC,MAAM,EAAE,uBAAuB,EAC/B,IAAI,CAAC,EAAE,kBAAkB,GACxB,OAAO,CAAC;IACT,MAAM,EAAE,iBAAiB,CAAC;IAC1B,EAAE,EAAE,0BAA0B,CAAC;CAChC,CAAC,CAiDD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,+BAA+B,CACnD,MAAM,EAAE,iBAAiB,EACzB,IAAI,CAAC,EAAE,mCAAmC,GACzC,OAAO,CAAC;IACT,MAAM,EAAE,iBAAiB,CAAC;IAC1B,EAAE,EAAE,0BAA0B,CAAC;CAChC,CAAC,CAsDD;AAED;;;GAGG;AACH,wBAAsB,gCAAgC,CACpD,YAAY,GAAE,YAAiC,GAC9C,OAAO,CAAC,0BAA0B,CAAC,CAKrC;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,EAAE,EAAE,0BAA0B,EAC9B,MAAM,EAAE,IAAI,CACV,uBAAuB,EACvB,QAAQ,GAAG,mBAAmB,GAAG,sBAAsB,GAAG,qBAAqB,GAAG,eAAe,CAClG,EACD,IAAI,CAAC,EAAE,mCAAmC,GACzC;IAAE,MAAM,EAAE,iBAAiB,CAAC;IAAC,EAAE,EAAE,0BAA0B,CAAA;CAAE,CAM/D"}
|
package/dest/factory.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Factory functions for creating validator HA signers
|
|
3
|
-
*/ import {
|
|
3
|
+
*/ import { createLogger } from '@aztec/foundation/log';
|
|
4
|
+
import { DateProvider } from '@aztec/foundation/timer';
|
|
4
5
|
import { createStore } from '@aztec/kv-store/lmdb-v2';
|
|
5
6
|
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
6
7
|
import { Pool } from 'pg';
|
|
@@ -25,7 +26,7 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
25
26
|
* databaseUrl: process.env.DATABASE_URL,
|
|
26
27
|
* nodeId: 'validator-node-1',
|
|
27
28
|
* pollingIntervalMs: 100,
|
|
28
|
-
*
|
|
29
|
+
* peerSigningTimeoutMs: 3000,
|
|
29
30
|
* });
|
|
30
31
|
* signer.start(); // Start background cleanup
|
|
31
32
|
*
|
|
@@ -61,6 +62,18 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
61
62
|
} else {
|
|
62
63
|
pool = deps.pool;
|
|
63
64
|
}
|
|
65
|
+
// pg re-emits idle-client errors (e.g. a Postgres restart severing an idle connection) on the
|
|
66
|
+
// pool. Without an 'error' listener, Node escalates these to an uncaughtException and crashes the
|
|
67
|
+
// process - taking down every HA replica sharing the DB at once. pg destroys and replaces the
|
|
68
|
+
// errored client itself, so logging is the only action needed. Log just message/code, never the
|
|
69
|
+
// raw error object (it can carry connection metadata).
|
|
70
|
+
const log = createLogger('validator-ha-signer:factory');
|
|
71
|
+
pool.on('error', (err)=>{
|
|
72
|
+
log.warn('Postgres pool error on idle client', {
|
|
73
|
+
message: err.message,
|
|
74
|
+
code: err.code
|
|
75
|
+
});
|
|
76
|
+
});
|
|
64
77
|
// Create database instance
|
|
65
78
|
const db = new PostgresSlashingProtectionDatabase(pool);
|
|
66
79
|
// Verify database schema is initialized and version matches
|
|
@@ -84,9 +97,10 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
84
97
|
* high-availability (multi-node) setup. It prevents a proposer from sending two
|
|
85
98
|
* proposals for the same slot if the node crashes and restarts mid-proposal.
|
|
86
99
|
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
100
|
+
* `config.dataDirectory` is required so the protection database is persisted to disk and survives
|
|
101
|
+
* crashes/restarts. Booting without it throws, since an ephemeral store silently drops all
|
|
102
|
+
* double-signing protection across restarts. Set `config.allowEphemeralSigningProtection` to opt
|
|
103
|
+
* into the ephemeral store anyway (dev/test networks only) — a loud warning is logged in that case.
|
|
90
104
|
*
|
|
91
105
|
* @param config - Local signer config
|
|
92
106
|
* @param deps - Optional dependencies (telemetry, date provider).
|
|
@@ -94,13 +108,21 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
94
108
|
*/ export async function createLocalSignerWithProtection(config, deps) {
|
|
95
109
|
const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
|
|
96
110
|
const dateProvider = deps?.dateProvider ?? new DateProvider();
|
|
111
|
+
const log = createLogger('validator-ha-signer:factory');
|
|
112
|
+
if (!config.dataDirectory) {
|
|
113
|
+
if (!config.allowEphemeralSigningProtection) {
|
|
114
|
+
throw new Error('Local signing protection requires a persistent data directory, but none was configured. ' + 'Set DATA_DIRECTORY so double-signing protection survives restarts, or explicitly opt into an ' + 'ephemeral store (dev/test only) with VALIDATOR_ALLOW_EPHEMERAL_SIGNING_PROTECTION=true.');
|
|
115
|
+
}
|
|
116
|
+
log.warn('Local signing protection is running with an EPHEMERAL store: no data directory is configured. ' + 'Double-signing protection will NOT survive a restart. This is unsafe for production validators.');
|
|
117
|
+
}
|
|
97
118
|
const kvStore = await createStore('signing-protection', LmdbSlashingProtectionDatabase.SCHEMA_VERSION, {
|
|
98
119
|
dataDirectory: config.dataDirectory,
|
|
99
120
|
dataStoreMapSizeKb: config.signingProtectionMapSizeKb ?? config.dataStoreMapSizeKb,
|
|
100
121
|
rollupAddress: config.rollupAddress
|
|
101
122
|
}, undefined, {
|
|
102
123
|
onUpgrade: (dataDirectory, currentVersion, latestVersion)=>migrateLmdbSlashingProtectionDatabase(dataDirectory, currentVersion, latestVersion, config.signingProtectionMapSizeKb ?? config.dataStoreMapSizeKb),
|
|
103
|
-
schemaVersionMismatchPolicy: 'throw'
|
|
124
|
+
schemaVersionMismatchPolicy: 'throw',
|
|
125
|
+
versionFileReadFailurePolicy: 'throw'
|
|
104
126
|
});
|
|
105
127
|
const db = new LmdbSlashingProtectionDatabase(kvStore, dateProvider);
|
|
106
128
|
const signerConfig = {
|
|
@@ -7,6 +7,8 @@ export interface SlashingProtectionServiceDeps {
|
|
|
7
7
|
metrics: HASignerMetrics;
|
|
8
8
|
dateProvider: DateProvider;
|
|
9
9
|
}
|
|
10
|
+
/** Default max age (ms) of a stuck SIGNING duty before cleanup reclaims it: 2x the 72s Aztec slot duration. */
|
|
11
|
+
export declare const DEFAULT_MAX_STUCK_DUTIES_AGE_MS = 144000;
|
|
10
12
|
/**
|
|
11
13
|
* Slashing Protection Service
|
|
12
14
|
*
|
|
@@ -27,7 +29,7 @@ export declare class SlashingProtectionService {
|
|
|
27
29
|
private readonly config;
|
|
28
30
|
private readonly log;
|
|
29
31
|
private readonly pollingIntervalMs;
|
|
30
|
-
private readonly
|
|
32
|
+
private readonly peerSigningTimeoutMs;
|
|
31
33
|
private readonly maxStuckDutiesAgeMs;
|
|
32
34
|
private readonly metrics;
|
|
33
35
|
private readonly dateProvider;
|
|
@@ -90,4 +92,4 @@ export declare class SlashingProtectionService {
|
|
|
90
92
|
close(): Promise<void>;
|
|
91
93
|
private cleanup;
|
|
92
94
|
}
|
|
93
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
95
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2xhc2hpbmdfcHJvdGVjdGlvbl9zZXJ2aWNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvc2xhc2hpbmdfcHJvdGVjdGlvbl9zZXJ2aWNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQVNBLE9BQU8sS0FBSyxFQUFFLFlBQVksRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzVELE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sMEJBQTBCLENBQUM7QUFFakUsT0FBTyxFQUNMLEtBQUssb0JBQW9CLEVBQ3pCLEtBQUssZ0JBQWdCLEVBRXJCLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sZUFBZSxDQUFDO0FBRXZCLE9BQU8sS0FBSyxFQUFFLGVBQWUsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUNwRCxPQUFPLEtBQUssRUFBRSwwQkFBMEIsRUFBRSxNQUFNLFlBQVksQ0FBQztBQUU3RCxNQUFNLFdBQVcsNkJBQTZCO0lBQzVDLE9BQU8sRUFBRSxlQUFlLENBQUM7SUFDekIsWUFBWSxFQUFFLFlBQVksQ0FBQztDQUM1QjtBQUVELCtHQUErRztBQUMvRyxlQUFPLE1BQU0sK0JBQStCLFNBQVUsQ0FBQztBQUV2RDs7Ozs7Ozs7Ozs7Ozs7R0FjRztBQUNILHFCQUFhLHlCQUF5QjtJQWFsQyxPQUFPLENBQUMsUUFBUSxDQUFDLEVBQUU7SUFDbkIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxNQUFNO0lBYnpCLE9BQU8sQ0FBQyxRQUFRLENBQUMsR0FBRyxDQUFTO0lBQzdCLE9BQU8sQ0FBQyxRQUFRLENBQUMsaUJBQWlCLENBQVM7SUFDM0MsT0FBTyxDQUFDLFFBQVEsQ0FBQyxvQkFBb0IsQ0FBUztJQUM5QyxPQUFPLENBQUMsUUFBUSxDQUFDLG1CQUFtQixDQUFTO0lBRTdDLE9BQU8sQ0FBQyxRQUFRLENBQUMsT0FBTyxDQUFrQjtJQUMxQyxPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVksQ0FBZTtJQUU1QyxPQUFPLENBQUMscUJBQXFCLENBQWlCO0lBQzlDLE9BQU8sQ0FBQyx3QkFBd0IsQ0FBQyxDQUFTO0lBRTFDLFlBQ21CLEVBQUUsRUFBRSwwQkFBMEIsRUFDOUIsTUFBTSxFQUFFLGdCQUFnQixFQUN6QyxJQUFJLEVBQUUsNkJBQTZCLEVBVXBDO0lBRUQ7Ozs7Ozs7Ozs7Ozs7O09BY0c7SUFDRyxjQUFjLENBQUMsTUFBTSxFQUFFLG9CQUFvQixHQUFHLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FxRWxFO0lBRUQ7Ozs7OztPQU1HO0lBQ0csYUFBYSxDQUFDLE1BQU0sRUFBRSxtQkFBbUIsR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBMkJqRTtJQUVEOzs7Ozs7T0FNRztJQUNHLFVBQVUsQ0FBQyxNQUFNLEVBQUUsZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQXdCM0Q7SUFFRDs7T0FFRztJQUNILElBQUksTUFBTSxJQUFJLE1BQU0sQ0FFbkI7SUFFRDs7O09BR0c7SUFDSDs7O09BR0c7SUFDRyxLQUFLLGtCQVlWO0lBRUQ7O09BRUc7SUFDRyxJQUFJLGtCQUdUO0lBRUQ7OztPQUdHO0lBQ0csS0FBSyxrQkFHVjtZQU1hLE9BQU87Q0FrQ3RCIn0=
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"slashing_protection_service.d.ts","sourceRoot":"","sources":["../src/slashing_protection_service.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EAErB,KAAK,mBAAmB,EAEzB,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAC;AAE7D,MAAM,WAAW,6BAA6B;IAC5C,OAAO,EAAE,eAAe,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;CAC5B;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,yBAAyB;IAalC,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAbzB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,
|
|
1
|
+
{"version":3,"file":"slashing_protection_service.d.ts","sourceRoot":"","sources":["../src/slashing_protection_service.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EAErB,KAAK,mBAAmB,EAEzB,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAC;AAE7D,MAAM,WAAW,6BAA6B;IAC5C,OAAO,EAAE,eAAe,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;CAC5B;AAED,+GAA+G;AAC/G,eAAO,MAAM,+BAA+B,SAAU,CAAC;AAEvD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,yBAAyB;IAalC,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAbzB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAE7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAE5C,OAAO,CAAC,qBAAqB,CAAiB;IAC9C,OAAO,CAAC,wBAAwB,CAAC,CAAS;IAE1C,YACmB,EAAE,EAAE,0BAA0B,EAC9B,MAAM,EAAE,gBAAgB,EACzC,IAAI,EAAE,6BAA6B,EAUpC;IAED;;;;;;;;;;;;;;OAcG;IACG,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAqElE;IAED;;;;;;OAMG;IACG,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,CA2BjE;IAED;;;;;;OAMG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAwB3D;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;OAGG;IACH;;;OAGG;IACG,KAAK,kBAYV;IAED;;OAEG;IACG,IAAI,kBAGT;IAED;;;OAGG;IACG,KAAK,kBAGV;YAMa,OAAO;CAkCtB"}
|
|
@@ -8,6 +8,7 @@ import { RunningPromise } from '@aztec/foundation/promise';
|
|
|
8
8
|
import { sleep } from '@aztec/foundation/sleep';
|
|
9
9
|
import { DutyStatus, getBlockIndexFromDutyIdentifier } from './db/types.js';
|
|
10
10
|
import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
11
|
+
/** Default max age (ms) of a stuck SIGNING duty before cleanup reclaims it: 2x the 72s Aztec slot duration. */ export const DEFAULT_MAX_STUCK_DUTIES_AGE_MS = 144_000;
|
|
11
12
|
/**
|
|
12
13
|
* Slashing Protection Service
|
|
13
14
|
*
|
|
@@ -27,7 +28,7 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
27
28
|
config;
|
|
28
29
|
log;
|
|
29
30
|
pollingIntervalMs;
|
|
30
|
-
|
|
31
|
+
peerSigningTimeoutMs;
|
|
31
32
|
maxStuckDutiesAgeMs;
|
|
32
33
|
metrics;
|
|
33
34
|
dateProvider;
|
|
@@ -38,9 +39,8 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
38
39
|
this.config = config;
|
|
39
40
|
this.log = createLogger('slashing-protection');
|
|
40
41
|
this.pollingIntervalMs = config.pollingIntervalMs;
|
|
41
|
-
this.
|
|
42
|
-
|
|
43
|
-
this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
|
|
42
|
+
this.peerSigningTimeoutMs = config.peerSigningTimeoutMs;
|
|
43
|
+
this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? DEFAULT_MAX_STUCK_DUTIES_AGE_MS;
|
|
44
44
|
this.cleanupRunningPromise = new RunningPromise(this.cleanup.bind(this), this.log, this.maxStuckDutiesAgeMs);
|
|
45
45
|
this.metrics = deps.metrics;
|
|
46
46
|
this.dateProvider = deps.dateProvider;
|
|
@@ -96,10 +96,10 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
96
96
|
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
|
|
97
97
|
} else if (record.status === DutyStatus.SIGNING) {
|
|
98
98
|
// Another node is currently signing - check for timeout
|
|
99
|
-
if (this.dateProvider.now() - startTime > this.
|
|
99
|
+
if (this.dateProvider.now() - startTime > this.peerSigningTimeoutMs) {
|
|
100
100
|
this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
|
|
101
101
|
validatorAddress: validatorAddress.toString(),
|
|
102
|
-
timeoutMs: this.
|
|
102
|
+
timeoutMs: this.peerSigningTimeoutMs,
|
|
103
103
|
signingNodeId: record.nodeId
|
|
104
104
|
});
|
|
105
105
|
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
@@ -205,7 +205,10 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
205
205
|
* Periodic cleanup of stuck duties and optionally old signed duties.
|
|
206
206
|
* Runs in the background via RunningPromise.
|
|
207
207
|
*/ async cleanup() {
|
|
208
|
-
// 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status)
|
|
208
|
+
// 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status).
|
|
209
|
+
// This cannot race an in-flight signing: every signing operation is hard-bounded by a timeout
|
|
210
|
+
// clamped below maxStuckDutiesAgeMs / 2 (see ValidatorHASigner), so a live SIGNING row is
|
|
211
|
+
// always released long before it can be considered stuck.
|
|
209
212
|
const numStuckDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
|
|
210
213
|
if (numStuckDuties > 0) {
|
|
211
214
|
this.log.verbose(`Cleaned up ${numStuckDuties} stuck duties`, {
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
import type { Buffer32 } from '@aztec/foundation/buffer';
|
|
9
9
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
10
10
|
import type { Signature } from '@aztec/foundation/eth-signature';
|
|
11
|
-
import type
|
|
11
|
+
import { type DateProvider } from '@aztec/foundation/timer';
|
|
12
12
|
import { type BaseSignerConfig, type HAProtectedSigningContext } from '@aztec/stdlib/ha-signing';
|
|
13
13
|
import type { HASignerMetrics } from './metrics.js';
|
|
14
14
|
import type { SlashingProtectionDatabase } from './types.js';
|
|
@@ -42,6 +42,7 @@ export declare class ValidatorHASigner {
|
|
|
42
42
|
private readonly rollupAddress;
|
|
43
43
|
private readonly dateProvider;
|
|
44
44
|
private readonly metrics;
|
|
45
|
+
private readonly signerCallTimeoutMs;
|
|
45
46
|
constructor(db: SlashingProtectionDatabase, config: BaseSignerConfig, deps: ValidatorHASignerDeps);
|
|
46
47
|
/**
|
|
47
48
|
* Sign a message with slashing protection.
|
|
@@ -76,4 +77,4 @@ export declare class ValidatorHASigner {
|
|
|
76
77
|
*/
|
|
77
78
|
stop(): Promise<void>;
|
|
78
79
|
}
|
|
79
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
80
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidmFsaWRhdG9yX2hhX3NpZ25lci5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL3ZhbGlkYXRvcl9oYV9zaWduZXIudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUE7Ozs7OztHQU1HO0FBQ0gsT0FBTyxLQUFLLEVBQUUsUUFBUSxFQUFFLE1BQU0sMEJBQTBCLENBQUM7QUFDekQsT0FBTyxFQUFFLFVBQVUsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBQzNELE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRWpFLE9BQU8sRUFBRSxLQUFLLFlBQVksRUFBa0IsTUFBTSx5QkFBeUIsQ0FBQztBQUM1RSxPQUFPLEVBQ0wsS0FBSyxnQkFBZ0IsRUFFckIsS0FBSyx5QkFBeUIsRUFHL0IsTUFBTSwwQkFBMEIsQ0FBQztBQUlsQyxPQUFPLEtBQUssRUFBRSxlQUFlLEVBQUUsTUFBTSxjQUFjLENBQUM7QUFFcEQsT0FBTyxLQUFLLEVBQUUsMEJBQTBCLEVBQUUsTUFBTSxZQUFZLENBQUM7QUFFN0QsTUFBTSxXQUFXLHFCQUFxQjtJQUNwQyxPQUFPLEVBQUUsZUFBZSxDQUFDO0lBQ3pCLFlBQVksRUFBRSxZQUFZLENBQUM7Q0FDNUI7QUFLRDs7Ozs7Ozs7Ozs7Ozs7Ozs7O0dBa0JHO0FBQ0gscUJBQWEsaUJBQWlCO0lBVzFCLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQVZ6QixPQUFPLENBQUMsUUFBUSxDQUFDLEdBQUcsQ0FBUztJQUM3QixPQUFPLENBQUMsUUFBUSxDQUFDLGtCQUFrQixDQUE0QjtJQUMvRCxPQUFPLENBQUMsUUFBUSxDQUFDLGFBQWEsQ0FBYTtJQUUzQyxPQUFPLENBQUMsUUFBUSxDQUFDLFlBQVksQ0FBZTtJQUM1QyxPQUFPLENBQUMsUUFBUSxDQUFDLE9BQU8sQ0FBa0I7SUFDMUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxtQkFBbUIsQ0FBUztJQUU3QyxZQUNFLEVBQUUsRUFBRSwwQkFBMEIsRUFDYixNQUFNLEVBQUUsZ0JBQWdCLEVBQ3pDLElBQUksRUFBRSxxQkFBcUIsRUFpQzVCO0lBRUQ7Ozs7Ozs7Ozs7Ozs7Ozs7T0FnQkc7SUFDRyxrQkFBa0IsQ0FDdEIsZ0JBQWdCLEVBQUUsVUFBVSxFQUM1QixXQUFXLEVBQUUsUUFBUSxFQUNyQixPQUFPLEVBQUUseUJBQXlCLEVBQ2xDLE1BQU0sRUFBRSxDQUFDLFdBQVcsRUFBRSxRQUFRLEtBQUssT0FBTyxDQUFDLFNBQVMsQ0FBQyxHQUNwRCxPQUFPLENBQUMsU0FBUyxDQUFDLENBMkVwQjtJQUVEOztPQUVHO0lBQ0gsSUFBSSxNQUFNLElBQUksTUFBTSxDQUVuQjtJQUVEOzs7T0FHRztJQUNHLEtBQUssa0JBRVY7SUFFRDs7O09BR0c7SUFDRyxJQUFJLGtCQUdUO0NBQ0YifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validator_ha_signer.d.ts","sourceRoot":"","sources":["../src/validator_ha_signer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"validator_ha_signer.d.ts","sourceRoot":"","sources":["../src/validator_ha_signer.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAC3D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iCAAiC,CAAC;AAEjE,OAAO,EAAE,KAAK,YAAY,EAAkB,MAAM,yBAAyB,CAAC;AAC5E,OAAO,EACL,KAAK,gBAAgB,EAErB,KAAK,yBAAyB,EAG/B,MAAM,0BAA0B,CAAC;AAIlC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEpD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAC;AAE7D,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,eAAe,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;CAC5B;AAKD;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,iBAAiB;IAW1B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAVzB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA4B;IAC/D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAa;IAE3C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAE7C,YACE,EAAE,EAAE,0BAA0B,EACb,MAAM,EAAE,gBAAgB,EACzC,IAAI,EAAE,qBAAqB,EAiC5B;IAED;;;;;;;;;;;;;;;;OAgBG;IACG,kBAAkB,CACtB,gBAAgB,EAAE,UAAU,EAC5B,WAAW,EAAE,QAAQ,EACrB,OAAO,EAAE,yBAAyB,EAClC,MAAM,EAAE,CAAC,WAAW,EAAE,QAAQ,KAAK,OAAO,CAAC,SAAS,CAAC,GACpD,OAAO,CAAC,SAAS,CAAC,CA2EpB;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;OAGG;IACG,KAAK,kBAEV;IAED;;;OAGG;IACG,IAAI,kBAGT;CACF"}
|
|
@@ -5,8 +5,11 @@
|
|
|
5
5
|
* This ensures that even with multiple validator nodes running, only one
|
|
6
6
|
* node will sign for a given duty (slot + duty type).
|
|
7
7
|
*/ import { createLogger } from '@aztec/foundation/log';
|
|
8
|
+
import { executeTimeout } from '@aztec/foundation/timer';
|
|
8
9
|
import { DutyType, getBlockNumberFromSigningContext, getCheckpointNumberFromSigningContext } from '@aztec/stdlib/ha-signing';
|
|
9
|
-
import {
|
|
10
|
+
import { SigningLockLostError } from './errors.js';
|
|
11
|
+
import { DEFAULT_MAX_STUCK_DUTIES_AGE_MS, SlashingProtectionService } from './slashing_protection_service.js';
|
|
12
|
+
/** Default hard timeout (ms) for a single signer call when not configured. */ const DEFAULT_SIGNER_CALL_TIMEOUT_MS = 30_000;
|
|
10
13
|
/**
|
|
11
14
|
* Validator High Availability Signer
|
|
12
15
|
*
|
|
@@ -32,11 +35,20 @@ import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
|
32
35
|
rollupAddress;
|
|
33
36
|
dateProvider;
|
|
34
37
|
metrics;
|
|
38
|
+
signerCallTimeoutMs;
|
|
35
39
|
constructor(db, config, deps){
|
|
36
40
|
this.config = config;
|
|
37
41
|
this.log = createLogger('validator-ha-signer');
|
|
38
42
|
this.metrics = deps.metrics;
|
|
39
43
|
this.dateProvider = deps.dateProvider;
|
|
44
|
+
// Clamp the signer-call timeout below half the stuck-duty max age. This maintains the
|
|
45
|
+
// invariant that an in-flight signing always times out and releases its SIGNING row well before
|
|
46
|
+
// stuck-duty cleanup could consider it stuck, so cleanup can never delete a live duty (only
|
|
47
|
+
// signWithProtection writes SIGNING rows, and every path through it is bounded by this timeout).
|
|
48
|
+
// If timers misbehave anyway, the recordSuccess-returns-false throw is the backstop: the duty
|
|
49
|
+
// fails instead of broadcasting an unprotected signature.
|
|
50
|
+
const maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? DEFAULT_MAX_STUCK_DUTIES_AGE_MS;
|
|
51
|
+
this.signerCallTimeoutMs = Math.min(config.signerCallTimeoutMs ?? DEFAULT_SIGNER_CALL_TIMEOUT_MS, maxStuckDutiesAgeMs / 2);
|
|
40
52
|
if (!config.nodeId || config.nodeId === '') {
|
|
41
53
|
throw new Error('NODE_ID is required for high-availability setups');
|
|
42
54
|
}
|
|
@@ -47,7 +59,8 @@ import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
|
47
59
|
});
|
|
48
60
|
this.log.info('Validator HA Signer initialized with slashing protection', {
|
|
49
61
|
nodeId: config.nodeId,
|
|
50
|
-
rollupAddress: this.rollupAddress.toString()
|
|
62
|
+
rollupAddress: this.rollupAddress.toString(),
|
|
63
|
+
signerCallTimeoutMs: this.signerCallTimeoutMs
|
|
51
64
|
});
|
|
52
65
|
}
|
|
53
66
|
/**
|
|
@@ -97,10 +110,12 @@ import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
|
97
110
|
messageHash: messageHash.toString(),
|
|
98
111
|
nodeId: this.config.nodeId
|
|
99
112
|
});
|
|
100
|
-
// Perform signing
|
|
113
|
+
// Perform signing under a hard timeout. If the signer hangs, executeTimeout aborts and rejects;
|
|
114
|
+
// the orphaned signFn promise resolving later is discarded (never broadcast). A timeout takes the
|
|
115
|
+
// same failure path as any signing error: release the lock so the duty can be retried safely.
|
|
101
116
|
let signature;
|
|
102
117
|
try {
|
|
103
|
-
signature = await signFn(messageHash);
|
|
118
|
+
signature = await executeTimeout(()=>signFn(messageHash), this.signerCallTimeoutMs, ()=>new Error(`Signing operation for ${dutyType} at slot ${context.slot} timed out after ` + `${this.signerCallTimeoutMs}ms`));
|
|
104
119
|
} catch (error) {
|
|
105
120
|
// Delete duty to allow retry (only succeeds if we own the lock)
|
|
106
121
|
await this.slashingProtection.deleteDuty({
|
|
@@ -110,13 +125,21 @@ import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
|
110
125
|
this.metrics.recordSigningError(dutyType);
|
|
111
126
|
throw error;
|
|
112
127
|
}
|
|
113
|
-
// Record success (only succeeds if we own the lock)
|
|
114
|
-
|
|
128
|
+
// Record success (only succeeds if we still own the lock).
|
|
129
|
+
// A false result means our SIGNING row is gone or no longer ours (e.g. deleted by stuck-duty
|
|
130
|
+
// cleanup while signing was slow). We must not broadcast this signature: without a protection
|
|
131
|
+
// record, a later attempt for the same duty with different data would sign freely (slashable).
|
|
132
|
+
// Do not delete the duty here - we no longer own it, and another node may legitimately hold it.
|
|
133
|
+
const recorded = await this.slashingProtection.recordSuccess({
|
|
115
134
|
...dutyIdentifier,
|
|
116
135
|
signature,
|
|
117
136
|
nodeId: this.config.nodeId,
|
|
118
137
|
lockToken
|
|
119
138
|
});
|
|
139
|
+
if (!recorded) {
|
|
140
|
+
this.metrics.recordSigningError(dutyType);
|
|
141
|
+
throw new SigningLockLostError(context.slot, dutyType, this.config.nodeId);
|
|
142
|
+
}
|
|
120
143
|
const duration = this.dateProvider.now() - startTime;
|
|
121
144
|
this.metrics.recordSigningSuccess(dutyType, duration);
|
|
122
145
|
return signature;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aztec/validator-ha-signer",
|
|
3
|
-
"version": "5.0.
|
|
3
|
+
"version": "5.0.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
"./db": "./dest/db/index.js",
|
|
@@ -75,11 +75,11 @@
|
|
|
75
75
|
]
|
|
76
76
|
},
|
|
77
77
|
"dependencies": {
|
|
78
|
-
"@aztec/ethereum": "5.0.
|
|
79
|
-
"@aztec/foundation": "5.0.
|
|
80
|
-
"@aztec/kv-store": "5.0.
|
|
81
|
-
"@aztec/stdlib": "5.0.
|
|
82
|
-
"@aztec/telemetry-client": "5.0.
|
|
78
|
+
"@aztec/ethereum": "5.0.1",
|
|
79
|
+
"@aztec/foundation": "5.0.1",
|
|
80
|
+
"@aztec/kv-store": "5.0.1",
|
|
81
|
+
"@aztec/stdlib": "5.0.1",
|
|
82
|
+
"@aztec/telemetry-client": "5.0.1",
|
|
83
83
|
"node-pg-migrate": "^8.0.4",
|
|
84
84
|
"pg": "^8.11.3",
|
|
85
85
|
"tslib": "^2.4.0",
|
package/src/errors.ts
CHANGED
|
@@ -22,6 +22,27 @@ export class DutyAlreadySignedError extends Error {
|
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
/**
|
|
26
|
+
* Thrown when the slashing-protection record for an in-flight signing operation can no longer be
|
|
27
|
+
* updated because it is no longer owned by this node - for example, the stuck-duty cleanup loop
|
|
28
|
+
* deleted the SIGNING row while the remote signer was slow. The produced signature must be
|
|
29
|
+
* discarded rather than broadcast: with no protection record in place, a later attempt for the
|
|
30
|
+
* same duty with different data would sign freely, which is slashable equivocation.
|
|
31
|
+
*/
|
|
32
|
+
export class SigningLockLostError extends Error {
|
|
33
|
+
constructor(
|
|
34
|
+
public readonly slot: SlotNumber,
|
|
35
|
+
public readonly dutyType: DutyType,
|
|
36
|
+
public readonly nodeId: string,
|
|
37
|
+
) {
|
|
38
|
+
super(
|
|
39
|
+
`Slashing protection record for ${dutyType} at slot ${slot} was lost before signing completed ` +
|
|
40
|
+
`(node ${nodeId}); discarding signature`,
|
|
41
|
+
);
|
|
42
|
+
this.name = 'SigningLockLostError';
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
25
46
|
/**
|
|
26
47
|
* Thrown when attempting to sign data that conflicts with an already-signed duty.
|
|
27
48
|
* This means the same validator tried to sign DIFFERENT data for the same slot.
|
package/src/factory.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Factory functions for creating validator HA signers
|
|
3
3
|
*/
|
|
4
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
4
5
|
import { DateProvider } from '@aztec/foundation/timer';
|
|
5
6
|
import { createStore } from '@aztec/kv-store/lmdb-v2';
|
|
6
7
|
import type { LocalSignerConfig, ValidatorHASignerConfig } from '@aztec/stdlib/ha-signing';
|
|
@@ -31,7 +32,7 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
31
32
|
* databaseUrl: process.env.DATABASE_URL,
|
|
32
33
|
* nodeId: 'validator-node-1',
|
|
33
34
|
* pollingIntervalMs: 100,
|
|
34
|
-
*
|
|
35
|
+
* peerSigningTimeoutMs: 3000,
|
|
35
36
|
* });
|
|
36
37
|
* signer.start(); // Start background cleanup
|
|
37
38
|
*
|
|
@@ -79,6 +80,16 @@ export async function createHASigner(
|
|
|
79
80
|
pool = deps.pool;
|
|
80
81
|
}
|
|
81
82
|
|
|
83
|
+
// pg re-emits idle-client errors (e.g. a Postgres restart severing an idle connection) on the
|
|
84
|
+
// pool. Without an 'error' listener, Node escalates these to an uncaughtException and crashes the
|
|
85
|
+
// process - taking down every HA replica sharing the DB at once. pg destroys and replaces the
|
|
86
|
+
// errored client itself, so logging is the only action needed. Log just message/code, never the
|
|
87
|
+
// raw error object (it can carry connection metadata).
|
|
88
|
+
const log = createLogger('validator-ha-signer:factory');
|
|
89
|
+
pool.on('error', (err: NodeJS.ErrnoException) => {
|
|
90
|
+
log.warn('Postgres pool error on idle client', { message: err.message, code: err.code });
|
|
91
|
+
});
|
|
92
|
+
|
|
82
93
|
// Create database instance
|
|
83
94
|
const db = new PostgresSlashingProtectionDatabase(pool);
|
|
84
95
|
|
|
@@ -101,9 +112,10 @@ export async function createHASigner(
|
|
|
101
112
|
* high-availability (multi-node) setup. It prevents a proposer from sending two
|
|
102
113
|
* proposals for the same slot if the node crashes and restarts mid-proposal.
|
|
103
114
|
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
115
|
+
* `config.dataDirectory` is required so the protection database is persisted to disk and survives
|
|
116
|
+
* crashes/restarts. Booting without it throws, since an ephemeral store silently drops all
|
|
117
|
+
* double-signing protection across restarts. Set `config.allowEphemeralSigningProtection` to opt
|
|
118
|
+
* into the ephemeral store anyway (dev/test networks only) — a loud warning is logged in that case.
|
|
107
119
|
*
|
|
108
120
|
* @param config - Local signer config
|
|
109
121
|
* @param deps - Optional dependencies (telemetry, date provider).
|
|
@@ -119,6 +131,22 @@ export async function createLocalSignerWithProtection(
|
|
|
119
131
|
const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
|
|
120
132
|
const dateProvider = deps?.dateProvider ?? new DateProvider();
|
|
121
133
|
|
|
134
|
+
const log = createLogger('validator-ha-signer:factory');
|
|
135
|
+
|
|
136
|
+
if (!config.dataDirectory) {
|
|
137
|
+
if (!config.allowEphemeralSigningProtection) {
|
|
138
|
+
throw new Error(
|
|
139
|
+
'Local signing protection requires a persistent data directory, but none was configured. ' +
|
|
140
|
+
'Set DATA_DIRECTORY so double-signing protection survives restarts, or explicitly opt into an ' +
|
|
141
|
+
'ephemeral store (dev/test only) with VALIDATOR_ALLOW_EPHEMERAL_SIGNING_PROTECTION=true.',
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
log.warn(
|
|
145
|
+
'Local signing protection is running with an EPHEMERAL store: no data directory is configured. ' +
|
|
146
|
+
'Double-signing protection will NOT survive a restart. This is unsafe for production validators.',
|
|
147
|
+
);
|
|
148
|
+
}
|
|
149
|
+
|
|
122
150
|
const kvStore = await createStore(
|
|
123
151
|
'signing-protection',
|
|
124
152
|
LmdbSlashingProtectionDatabase.SCHEMA_VERSION,
|
|
@@ -137,6 +165,7 @@ export async function createLocalSignerWithProtection(
|
|
|
137
165
|
config.signingProtectionMapSizeKb ?? config.dataStoreMapSizeKb,
|
|
138
166
|
),
|
|
139
167
|
schemaVersionMismatchPolicy: 'throw',
|
|
168
|
+
versionFileReadFailurePolicy: 'throw',
|
|
140
169
|
},
|
|
141
170
|
);
|
|
142
171
|
|
|
@@ -175,7 +204,7 @@ export function createSignerFromSharedDb(
|
|
|
175
204
|
db: SlashingProtectionDatabase,
|
|
176
205
|
config: Pick<
|
|
177
206
|
ValidatorHASignerConfig,
|
|
178
|
-
'nodeId' | 'pollingIntervalMs' | '
|
|
207
|
+
'nodeId' | 'pollingIntervalMs' | 'peerSigningTimeoutMs' | 'maxStuckDutiesAgeMs' | 'rollupAddress'
|
|
179
208
|
>,
|
|
180
209
|
deps?: CreateLocalSignerWithProtectionDeps,
|
|
181
210
|
): { signer: ValidatorHASigner; db: SlashingProtectionDatabase } {
|
|
@@ -26,6 +26,9 @@ export interface SlashingProtectionServiceDeps {
|
|
|
26
26
|
dateProvider: DateProvider;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
/** Default max age (ms) of a stuck SIGNING duty before cleanup reclaims it: 2x the 72s Aztec slot duration. */
|
|
30
|
+
export const DEFAULT_MAX_STUCK_DUTIES_AGE_MS = 144_000;
|
|
31
|
+
|
|
29
32
|
/**
|
|
30
33
|
* Slashing Protection Service
|
|
31
34
|
*
|
|
@@ -44,7 +47,7 @@ export interface SlashingProtectionServiceDeps {
|
|
|
44
47
|
export class SlashingProtectionService {
|
|
45
48
|
private readonly log: Logger;
|
|
46
49
|
private readonly pollingIntervalMs: number;
|
|
47
|
-
private readonly
|
|
50
|
+
private readonly peerSigningTimeoutMs: number;
|
|
48
51
|
private readonly maxStuckDutiesAgeMs: number;
|
|
49
52
|
|
|
50
53
|
private readonly metrics: HASignerMetrics;
|
|
@@ -60,9 +63,8 @@ export class SlashingProtectionService {
|
|
|
60
63
|
) {
|
|
61
64
|
this.log = createLogger('slashing-protection');
|
|
62
65
|
this.pollingIntervalMs = config.pollingIntervalMs;
|
|
63
|
-
this.
|
|
64
|
-
|
|
65
|
-
this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
|
|
66
|
+
this.peerSigningTimeoutMs = config.peerSigningTimeoutMs;
|
|
67
|
+
this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? DEFAULT_MAX_STUCK_DUTIES_AGE_MS;
|
|
66
68
|
|
|
67
69
|
this.cleanupRunningPromise = new RunningPromise(this.cleanup.bind(this), this.log, this.maxStuckDutiesAgeMs);
|
|
68
70
|
this.metrics = deps.metrics;
|
|
@@ -132,10 +134,10 @@ export class SlashingProtectionService {
|
|
|
132
134
|
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
|
|
133
135
|
} else if (record.status === DutyStatus.SIGNING) {
|
|
134
136
|
// Another node is currently signing - check for timeout
|
|
135
|
-
if (this.dateProvider.now() - startTime > this.
|
|
137
|
+
if (this.dateProvider.now() - startTime > this.peerSigningTimeoutMs) {
|
|
136
138
|
this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
|
|
137
139
|
validatorAddress: validatorAddress.toString(),
|
|
138
|
-
timeoutMs: this.
|
|
140
|
+
timeoutMs: this.peerSigningTimeoutMs,
|
|
139
141
|
signingNodeId: record.nodeId,
|
|
140
142
|
});
|
|
141
143
|
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
@@ -275,7 +277,10 @@ export class SlashingProtectionService {
|
|
|
275
277
|
* Runs in the background via RunningPromise.
|
|
276
278
|
*/
|
|
277
279
|
private async cleanup() {
|
|
278
|
-
// 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status)
|
|
280
|
+
// 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status).
|
|
281
|
+
// This cannot race an in-flight signing: every signing operation is hard-bounded by a timeout
|
|
282
|
+
// clamped below maxStuckDutiesAgeMs / 2 (see ValidatorHASigner), so a live SIGNING row is
|
|
283
|
+
// always released long before it can be considered stuck.
|
|
279
284
|
const numStuckDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
|
|
280
285
|
if (numStuckDuties > 0) {
|
|
281
286
|
this.log.verbose(`Cleaned up ${numStuckDuties} stuck duties`, {
|
|
@@ -9,7 +9,7 @@ import type { Buffer32 } from '@aztec/foundation/buffer';
|
|
|
9
9
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
10
10
|
import type { Signature } from '@aztec/foundation/eth-signature';
|
|
11
11
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
12
|
-
import type
|
|
12
|
+
import { type DateProvider, executeTimeout } from '@aztec/foundation/timer';
|
|
13
13
|
import {
|
|
14
14
|
type BaseSignerConfig,
|
|
15
15
|
DutyType,
|
|
@@ -19,8 +19,9 @@ import {
|
|
|
19
19
|
} from '@aztec/stdlib/ha-signing';
|
|
20
20
|
|
|
21
21
|
import type { DutyIdentifier } from './db/types.js';
|
|
22
|
+
import { SigningLockLostError } from './errors.js';
|
|
22
23
|
import type { HASignerMetrics } from './metrics.js';
|
|
23
|
-
import { SlashingProtectionService } from './slashing_protection_service.js';
|
|
24
|
+
import { DEFAULT_MAX_STUCK_DUTIES_AGE_MS, SlashingProtectionService } from './slashing_protection_service.js';
|
|
24
25
|
import type { SlashingProtectionDatabase } from './types.js';
|
|
25
26
|
|
|
26
27
|
export interface ValidatorHASignerDeps {
|
|
@@ -28,6 +29,9 @@ export interface ValidatorHASignerDeps {
|
|
|
28
29
|
dateProvider: DateProvider;
|
|
29
30
|
}
|
|
30
31
|
|
|
32
|
+
/** Default hard timeout (ms) for a single signer call when not configured. */
|
|
33
|
+
const DEFAULT_SIGNER_CALL_TIMEOUT_MS = 30_000;
|
|
34
|
+
|
|
31
35
|
/**
|
|
32
36
|
* Validator High Availability Signer
|
|
33
37
|
*
|
|
@@ -54,6 +58,7 @@ export class ValidatorHASigner {
|
|
|
54
58
|
|
|
55
59
|
private readonly dateProvider: DateProvider;
|
|
56
60
|
private readonly metrics: HASignerMetrics;
|
|
61
|
+
private readonly signerCallTimeoutMs: number;
|
|
57
62
|
|
|
58
63
|
constructor(
|
|
59
64
|
db: SlashingProtectionDatabase,
|
|
@@ -65,6 +70,18 @@ export class ValidatorHASigner {
|
|
|
65
70
|
this.metrics = deps.metrics;
|
|
66
71
|
this.dateProvider = deps.dateProvider;
|
|
67
72
|
|
|
73
|
+
// Clamp the signer-call timeout below half the stuck-duty max age. This maintains the
|
|
74
|
+
// invariant that an in-flight signing always times out and releases its SIGNING row well before
|
|
75
|
+
// stuck-duty cleanup could consider it stuck, so cleanup can never delete a live duty (only
|
|
76
|
+
// signWithProtection writes SIGNING rows, and every path through it is bounded by this timeout).
|
|
77
|
+
// If timers misbehave anyway, the recordSuccess-returns-false throw is the backstop: the duty
|
|
78
|
+
// fails instead of broadcasting an unprotected signature.
|
|
79
|
+
const maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? DEFAULT_MAX_STUCK_DUTIES_AGE_MS;
|
|
80
|
+
this.signerCallTimeoutMs = Math.min(
|
|
81
|
+
config.signerCallTimeoutMs ?? DEFAULT_SIGNER_CALL_TIMEOUT_MS,
|
|
82
|
+
maxStuckDutiesAgeMs / 2,
|
|
83
|
+
);
|
|
84
|
+
|
|
68
85
|
if (!config.nodeId || config.nodeId === '') {
|
|
69
86
|
throw new Error('NODE_ID is required for high-availability setups');
|
|
70
87
|
}
|
|
@@ -73,9 +90,11 @@ export class ValidatorHASigner {
|
|
|
73
90
|
metrics: deps.metrics,
|
|
74
91
|
dateProvider: deps.dateProvider,
|
|
75
92
|
});
|
|
93
|
+
|
|
76
94
|
this.log.info('Validator HA Signer initialized with slashing protection', {
|
|
77
95
|
nodeId: config.nodeId,
|
|
78
96
|
rollupAddress: this.rollupAddress.toString(),
|
|
97
|
+
signerCallTimeoutMs: this.signerCallTimeoutMs,
|
|
79
98
|
});
|
|
80
99
|
}
|
|
81
100
|
|
|
@@ -135,10 +154,20 @@ export class ValidatorHASigner {
|
|
|
135
154
|
nodeId: this.config.nodeId,
|
|
136
155
|
});
|
|
137
156
|
|
|
138
|
-
// Perform signing
|
|
157
|
+
// Perform signing under a hard timeout. If the signer hangs, executeTimeout aborts and rejects;
|
|
158
|
+
// the orphaned signFn promise resolving later is discarded (never broadcast). A timeout takes the
|
|
159
|
+
// same failure path as any signing error: release the lock so the duty can be retried safely.
|
|
139
160
|
let signature: Signature;
|
|
140
161
|
try {
|
|
141
|
-
signature = await
|
|
162
|
+
signature = await executeTimeout(
|
|
163
|
+
() => signFn(messageHash),
|
|
164
|
+
this.signerCallTimeoutMs,
|
|
165
|
+
() =>
|
|
166
|
+
new Error(
|
|
167
|
+
`Signing operation for ${dutyType} at slot ${context.slot} timed out after ` +
|
|
168
|
+
`${this.signerCallTimeoutMs}ms`,
|
|
169
|
+
),
|
|
170
|
+
);
|
|
142
171
|
} catch (error: any) {
|
|
143
172
|
// Delete duty to allow retry (only succeeds if we own the lock)
|
|
144
173
|
await this.slashingProtection.deleteDuty({ ...dutyIdentifier, lockToken });
|
|
@@ -146,13 +175,21 @@ export class ValidatorHASigner {
|
|
|
146
175
|
throw error;
|
|
147
176
|
}
|
|
148
177
|
|
|
149
|
-
// Record success (only succeeds if we own the lock)
|
|
150
|
-
|
|
178
|
+
// Record success (only succeeds if we still own the lock).
|
|
179
|
+
// A false result means our SIGNING row is gone or no longer ours (e.g. deleted by stuck-duty
|
|
180
|
+
// cleanup while signing was slow). We must not broadcast this signature: without a protection
|
|
181
|
+
// record, a later attempt for the same duty with different data would sign freely (slashable).
|
|
182
|
+
// Do not delete the duty here - we no longer own it, and another node may legitimately hold it.
|
|
183
|
+
const recorded = await this.slashingProtection.recordSuccess({
|
|
151
184
|
...dutyIdentifier,
|
|
152
185
|
signature,
|
|
153
186
|
nodeId: this.config.nodeId,
|
|
154
187
|
lockToken,
|
|
155
188
|
});
|
|
189
|
+
if (!recorded) {
|
|
190
|
+
this.metrics.recordSigningError(dutyType);
|
|
191
|
+
throw new SigningLockLostError(context.slot, dutyType, this.config.nodeId);
|
|
192
|
+
}
|
|
156
193
|
|
|
157
194
|
const duration = this.dateProvider.now() - startTime;
|
|
158
195
|
this.metrics.recordSigningSuccess(dutyType, duration);
|