@aztec/validator-ha-signer 0.0.1-commit.d1f2d6c → 0.0.1-commit.d20b825a7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dest/db/index.d.ts +2 -1
- package/dest/db/index.d.ts.map +1 -1
- package/dest/db/index.js +1 -0
- package/dest/db/lmdb.d.ts +66 -0
- package/dest/db/lmdb.d.ts.map +1 -0
- package/dest/db/lmdb.js +189 -0
- package/dest/db/migrations/1_initial-schema.d.ts +4 -2
- package/dest/db/migrations/1_initial-schema.d.ts.map +1 -1
- package/dest/db/migrations/1_initial-schema.js +34 -4
- package/dest/db/migrations/2_add-checkpoint-number.d.ts +7 -0
- package/dest/db/migrations/2_add-checkpoint-number.d.ts.map +1 -0
- package/dest/db/migrations/2_add-checkpoint-number.js +17 -0
- package/dest/db/postgres.d.ts +20 -4
- package/dest/db/postgres.d.ts.map +1 -1
- package/dest/db/postgres.js +46 -17
- package/dest/db/schema.d.ts +18 -11
- package/dest/db/schema.d.ts.map +1 -1
- package/dest/db/schema.js +45 -23
- package/dest/db/types.d.ts +52 -22
- package/dest/db/types.d.ts.map +1 -1
- package/dest/db/types.js +31 -15
- package/dest/factory.d.ts +39 -4
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +78 -7
- package/dest/metrics.d.ts +51 -0
- package/dest/metrics.d.ts.map +1 -0
- package/dest/metrics.js +103 -0
- package/dest/slashing_protection_service.d.ts +19 -6
- package/dest/slashing_protection_service.d.ts.map +1 -1
- package/dest/slashing_protection_service.js +57 -17
- package/dest/types.d.ts +33 -72
- package/dest/types.d.ts.map +1 -1
- package/dest/types.js +4 -20
- package/dest/validator_ha_signer.d.ts +15 -6
- package/dest/validator_ha_signer.d.ts.map +1 -1
- package/dest/validator_ha_signer.js +26 -11
- package/package.json +10 -5
- package/src/db/index.ts +1 -0
- package/src/db/lmdb.ts +265 -0
- package/src/db/migrations/1_initial-schema.ts +35 -4
- package/src/db/migrations/2_add-checkpoint-number.ts +19 -0
- package/src/db/postgres.ts +47 -12
- package/src/db/schema.ts +47 -23
- package/src/db/types.ts +72 -20
- package/src/factory.ts +96 -6
- package/src/metrics.ts +138 -0
- package/src/slashing_protection_service.ts +79 -21
- package/src/types.ts +56 -103
- package/src/validator_ha_signer.ts +44 -15
- package/dest/config.d.ts +0 -79
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -73
- package/src/config.ts +0 -125
package/dest/factory.js
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Factory functions for creating validator HA signers
|
|
3
|
-
*/ import {
|
|
3
|
+
*/ import { DateProvider } from '@aztec/foundation/timer';
|
|
4
|
+
import { createStore } from '@aztec/kv-store/lmdb-v2';
|
|
5
|
+
import { getTelemetryClient } from '@aztec/telemetry-client';
|
|
6
|
+
import { Pool } from 'pg';
|
|
7
|
+
import { LmdbSlashingProtectionDatabase } from './db/lmdb.js';
|
|
4
8
|
import { PostgresSlashingProtectionDatabase } from './db/postgres.js';
|
|
9
|
+
import { HASignerMetrics } from './metrics.js';
|
|
5
10
|
import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
6
11
|
/**
|
|
7
12
|
* Create a validator HA signer with PostgreSQL backend
|
|
@@ -18,7 +23,6 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
18
23
|
* ```typescript
|
|
19
24
|
* const { signer, db } = await createHASigner({
|
|
20
25
|
* databaseUrl: process.env.DATABASE_URL,
|
|
21
|
-
* haSigningEnabled: true,
|
|
22
26
|
* nodeId: 'validator-node-1',
|
|
23
27
|
* pollingIntervalMs: 100,
|
|
24
28
|
* signingTimeoutMs: 3000,
|
|
@@ -38,14 +42,17 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
38
42
|
* @returns An object containing the signer and database instances
|
|
39
43
|
*/ export async function createHASigner(config, deps) {
|
|
40
44
|
const { databaseUrl, poolMaxCount, poolMinCount, poolIdleTimeoutMs, poolConnectionTimeoutMs, ...signerConfig } = config;
|
|
41
|
-
|
|
45
|
+
const databaseUrlValue = databaseUrl?.getValue();
|
|
46
|
+
if (!databaseUrlValue) {
|
|
42
47
|
throw new Error('databaseUrl is required for createHASigner');
|
|
43
48
|
}
|
|
49
|
+
const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
|
|
50
|
+
const dateProvider = deps?.dateProvider ?? new DateProvider();
|
|
44
51
|
// Create connection pool (or use provided pool)
|
|
45
52
|
let pool;
|
|
46
53
|
if (!deps?.pool) {
|
|
47
54
|
pool = new Pool({
|
|
48
|
-
connectionString:
|
|
55
|
+
connectionString: databaseUrlValue,
|
|
49
56
|
max: poolMaxCount ?? 10,
|
|
50
57
|
min: poolMinCount ?? 0,
|
|
51
58
|
idleTimeoutMillis: poolIdleTimeoutMs ?? 10_000,
|
|
@@ -58,10 +65,74 @@ import { ValidatorHASigner } from './validator_ha_signer.js';
|
|
|
58
65
|
const db = new PostgresSlashingProtectionDatabase(pool);
|
|
59
66
|
// Verify database schema is initialized and version matches
|
|
60
67
|
await db.initialize();
|
|
68
|
+
// Create metrics
|
|
69
|
+
const metrics = new HASignerMetrics(telemetryClient, signerConfig.nodeId);
|
|
61
70
|
// Create signer
|
|
62
|
-
const signer = new ValidatorHASigner(db, {
|
|
63
|
-
|
|
64
|
-
|
|
71
|
+
const signer = new ValidatorHASigner(db, signerConfig, {
|
|
72
|
+
metrics,
|
|
73
|
+
dateProvider
|
|
74
|
+
});
|
|
75
|
+
return {
|
|
76
|
+
signer,
|
|
77
|
+
db
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Create a local (single-node) signing protection signer backed by LMDB.
|
|
82
|
+
*
|
|
83
|
+
* This provides double-signing protection for nodes that are NOT running in a
|
|
84
|
+
* high-availability (multi-node) setup. It prevents a proposer from sending two
|
|
85
|
+
* proposals for the same slot if the node crashes and restarts mid-proposal.
|
|
86
|
+
*
|
|
87
|
+
* When `config.dataDirectory` is set, the protection database is persisted to disk
|
|
88
|
+
* and survives crashes/restarts. When unset, an ephemeral in-memory store is
|
|
89
|
+
* used which protects within a single run but not across restarts.
|
|
90
|
+
*
|
|
91
|
+
* @param config - Local signer config
|
|
92
|
+
* @param deps - Optional dependencies (telemetry, date provider).
|
|
93
|
+
* @returns An object containing the signer and database instances.
|
|
94
|
+
*/ export async function createLocalSignerWithProtection(config, deps) {
|
|
95
|
+
const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
|
|
96
|
+
const dateProvider = deps?.dateProvider ?? new DateProvider();
|
|
97
|
+
const kvStore = await createStore('signing-protection', LmdbSlashingProtectionDatabase.SCHEMA_VERSION, {
|
|
98
|
+
dataDirectory: config.dataDirectory,
|
|
99
|
+
dataStoreMapSizeKb: config.signingProtectionMapSizeKb ?? config.dataStoreMapSizeKb,
|
|
100
|
+
l1Contracts: config.l1Contracts
|
|
101
|
+
});
|
|
102
|
+
const db = new LmdbSlashingProtectionDatabase(kvStore, dateProvider);
|
|
103
|
+
const signerConfig = {
|
|
104
|
+
...config,
|
|
105
|
+
nodeId: config.nodeId || 'local'
|
|
106
|
+
};
|
|
107
|
+
const metrics = new HASignerMetrics(telemetryClient, signerConfig.nodeId, 'LocalSigningProtectionMetrics');
|
|
108
|
+
const signer = new ValidatorHASigner(db, signerConfig, {
|
|
109
|
+
metrics,
|
|
110
|
+
dateProvider
|
|
111
|
+
});
|
|
112
|
+
return {
|
|
113
|
+
signer,
|
|
114
|
+
db
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Create an in-memory LMDB-backed SlashingProtectionDatabase that can be shared across
|
|
119
|
+
* multiple validator nodes in the same process. Used for testing HA setups.
|
|
120
|
+
*/ export async function createSharedSlashingProtectionDb(dateProvider = new DateProvider()) {
|
|
121
|
+
const kvStore = await createStore('shared-signing-protection', LmdbSlashingProtectionDatabase.SCHEMA_VERSION, {
|
|
122
|
+
dataStoreMapSizeKb: 1024 * 1024
|
|
123
|
+
});
|
|
124
|
+
return new LmdbSlashingProtectionDatabase(kvStore, dateProvider);
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Create a ValidatorHASigner backed by a pre-existing SlashingProtectionDatabase.
|
|
128
|
+
* Used for testing HA setups where multiple nodes share the same protection database.
|
|
129
|
+
*/ export function createSignerFromSharedDb(db, config, deps) {
|
|
130
|
+
const telemetryClient = deps?.telemetryClient ?? getTelemetryClient();
|
|
131
|
+
const dateProvider = deps?.dateProvider ?? new DateProvider();
|
|
132
|
+
const metrics = new HASignerMetrics(telemetryClient, config.nodeId, 'SharedSigningProtectionMetrics');
|
|
133
|
+
const signer = new ValidatorHASigner(db, config, {
|
|
134
|
+
metrics,
|
|
135
|
+
dateProvider
|
|
65
136
|
});
|
|
66
137
|
return {
|
|
67
138
|
signer,
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { type TelemetryClient } from '@aztec/telemetry-client';
|
|
2
|
+
export type HACleanupType = 'stuck' | 'old' | 'outdated_rollup';
|
|
3
|
+
/**
|
|
4
|
+
* Metrics for HA signer tracking signing operations, lock acquisition, and cleanup.
|
|
5
|
+
*/
|
|
6
|
+
export declare class HASignerMetrics {
|
|
7
|
+
private nodeId;
|
|
8
|
+
private signingDuration;
|
|
9
|
+
private signingSuccessCount;
|
|
10
|
+
private dutyAlreadySignedCount;
|
|
11
|
+
private slashingProtectionCount;
|
|
12
|
+
private signingErrorCount;
|
|
13
|
+
private lockAcquiredCount;
|
|
14
|
+
private cleanupStuckDutiesCount;
|
|
15
|
+
private cleanupOldDutiesCount;
|
|
16
|
+
private cleanupOutdatedRollupDutiesCount;
|
|
17
|
+
constructor(client: TelemetryClient, nodeId: string, name?: string);
|
|
18
|
+
/**
|
|
19
|
+
* Record a successful signing operation.
|
|
20
|
+
* @param dutyType - The type of duty signed
|
|
21
|
+
* @param durationMs - Duration from start of signWithProtection to completion
|
|
22
|
+
*/
|
|
23
|
+
recordSigningSuccess(dutyType: string, durationMs: number): void;
|
|
24
|
+
/**
|
|
25
|
+
* Record a DutyAlreadySignedError (expected in HA; another node signed first).
|
|
26
|
+
* @param dutyType - The type of duty
|
|
27
|
+
*/
|
|
28
|
+
recordDutyAlreadySigned(dutyType: string): void;
|
|
29
|
+
/**
|
|
30
|
+
* Record a SlashingProtectionError (attempted to sign different data for same duty).
|
|
31
|
+
* @param dutyType - The type of duty
|
|
32
|
+
*/
|
|
33
|
+
recordSlashingProtection(dutyType: string): void;
|
|
34
|
+
/**
|
|
35
|
+
* Record a signing function failure (lock will be deleted for retry).
|
|
36
|
+
* @param dutyType - The type of duty
|
|
37
|
+
*/
|
|
38
|
+
recordSigningError(dutyType: string): void;
|
|
39
|
+
/**
|
|
40
|
+
* Record lock acquisition.
|
|
41
|
+
* @param acquired - Whether a new lock was acquired (true) or existing record found (false)
|
|
42
|
+
*/
|
|
43
|
+
recordLockAcquire(acquired: boolean): void;
|
|
44
|
+
/**
|
|
45
|
+
* Record cleanup metrics.
|
|
46
|
+
* @param type - Type of cleanup
|
|
47
|
+
* @param count - Number of duties cleaned up
|
|
48
|
+
*/
|
|
49
|
+
recordCleanup(type: HACleanupType, count: number): void;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWV0cmljcy5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL21ldHJpY3MudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUlMLEtBQUssZUFBZSxFQUdyQixNQUFNLHlCQUF5QixDQUFDO0FBRWpDLE1BQU0sTUFBTSxhQUFhLEdBQUcsT0FBTyxHQUFHLEtBQUssR0FBRyxpQkFBaUIsQ0FBQztBQUVoRTs7R0FFRztBQUNILHFCQUFhLGVBQWU7SUFrQnhCLE9BQU8sQ0FBQyxNQUFNO0lBaEJoQixPQUFPLENBQUMsZUFBZSxDQUFZO0lBQ25DLE9BQU8sQ0FBQyxtQkFBbUIsQ0FBZ0I7SUFDM0MsT0FBTyxDQUFDLHNCQUFzQixDQUFnQjtJQUM5QyxPQUFPLENBQUMsdUJBQXVCLENBQWdCO0lBQy9DLE9BQU8sQ0FBQyxpQkFBaUIsQ0FBZ0I7SUFHekMsT0FBTyxDQUFDLGlCQUFpQixDQUFnQjtJQUd6QyxPQUFPLENBQUMsdUJBQXVCLENBQWdCO0lBQy9DLE9BQU8sQ0FBQyxxQkFBcUIsQ0FBZ0I7SUFDN0MsT0FBTyxDQUFDLGdDQUFnQyxDQUFnQjtJQUV4RCxZQUNFLE1BQU0sRUFBRSxlQUFlLEVBQ2YsTUFBTSxFQUFFLE1BQU0sRUFDdEIsSUFBSSxTQUFvQixFQXFCekI7SUFFRDs7OztPQUlHO0lBQ0ksb0JBQW9CLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxVQUFVLEVBQUUsTUFBTSxHQUFHLElBQUksQ0FPdEU7SUFFRDs7O09BR0c7SUFDSSx1QkFBdUIsQ0FBQyxRQUFRLEVBQUUsTUFBTSxHQUFHLElBQUksQ0FNckQ7SUFFRDs7O09BR0c7SUFDSSx3QkFBd0IsQ0FBQyxRQUFRLEVBQUUsTUFBTSxHQUFHLElBQUksQ0FNdEQ7SUFFRDs7O09BR0c7SUFDSSxrQkFBa0IsQ0FBQyxRQUFRLEVBQUUsTUFBTSxHQUFHLElBQUksQ0FNaEQ7SUFFRDs7O09BR0c7SUFDSSxpQkFBaUIsQ0FBQyxRQUFRLEVBQUUsT0FBTyxHQUFHLElBQUksQ0FPaEQ7SUFFRDs7OztPQUlHO0lBQ0ksYUFBYSxDQUFDLElBQUksRUFBRSxhQUFhLEVBQUUsS0FBSyxFQUFFLE1BQU0sR0FBRyxJQUFJLENBWTdEO0NBQ0YifQ==
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,eAAe,EAGrB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,aAAa,GAAG,OAAO,GAAG,KAAK,GAAG,iBAAiB,CAAC;AAEhE;;GAEG;AACH,qBAAa,eAAe;IAkBxB,OAAO,CAAC,MAAM;IAhBhB,OAAO,CAAC,eAAe,CAAY;IACnC,OAAO,CAAC,mBAAmB,CAAgB;IAC3C,OAAO,CAAC,sBAAsB,CAAgB;IAC9C,OAAO,CAAC,uBAAuB,CAAgB;IAC/C,OAAO,CAAC,iBAAiB,CAAgB;IAGzC,OAAO,CAAC,iBAAiB,CAAgB;IAGzC,OAAO,CAAC,uBAAuB,CAAgB;IAC/C,OAAO,CAAC,qBAAqB,CAAgB;IAC7C,OAAO,CAAC,gCAAgC,CAAgB;IAExD,YACE,MAAM,EAAE,eAAe,EACf,MAAM,EAAE,MAAM,EACtB,IAAI,SAAoB,EAqBzB;IAED;;;;OAIG;IACI,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,CAOtE;IAED;;;OAGG;IACI,uBAAuB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAMrD;IAED;;;OAGG;IACI,wBAAwB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAMtD;IAED;;;OAGG;IACI,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAMhD;IAED;;;OAGG;IACI,iBAAiB,CAAC,QAAQ,EAAE,OAAO,GAAG,IAAI,CAOhD;IAED;;;;OAIG;IACI,aAAa,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAY7D;CACF"}
|
package/dest/metrics.js
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
import { Attributes, Metrics, createUpDownCounterWithDefault } from '@aztec/telemetry-client';
|
|
2
|
+
/**
|
|
3
|
+
* Metrics for HA signer tracking signing operations, lock acquisition, and cleanup.
|
|
4
|
+
*/ export class HASignerMetrics {
|
|
5
|
+
nodeId;
|
|
6
|
+
// Signing lifecycle metrics
|
|
7
|
+
signingDuration;
|
|
8
|
+
signingSuccessCount;
|
|
9
|
+
dutyAlreadySignedCount;
|
|
10
|
+
slashingProtectionCount;
|
|
11
|
+
signingErrorCount;
|
|
12
|
+
// Lock acquisition metrics
|
|
13
|
+
lockAcquiredCount;
|
|
14
|
+
// Cleanup metrics
|
|
15
|
+
cleanupStuckDutiesCount;
|
|
16
|
+
cleanupOldDutiesCount;
|
|
17
|
+
cleanupOutdatedRollupDutiesCount;
|
|
18
|
+
constructor(client, nodeId, name = 'HASignerMetrics'){
|
|
19
|
+
this.nodeId = nodeId;
|
|
20
|
+
const meter = client.getMeter(name);
|
|
21
|
+
// Signing lifecycle
|
|
22
|
+
this.signingDuration = meter.createHistogram(Metrics.HA_SIGNER_SIGNING_DURATION);
|
|
23
|
+
this.signingSuccessCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_SUCCESS_COUNT);
|
|
24
|
+
this.dutyAlreadySignedCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_DUTY_ALREADY_SIGNED_COUNT);
|
|
25
|
+
this.slashingProtectionCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SLASHING_PROTECTION_COUNT);
|
|
26
|
+
this.signingErrorCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_ERROR_COUNT);
|
|
27
|
+
// Lock acquisition
|
|
28
|
+
this.lockAcquiredCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_LOCK_ACQUIRED_COUNT);
|
|
29
|
+
// Cleanup
|
|
30
|
+
this.cleanupStuckDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_STUCK_DUTIES_COUNT);
|
|
31
|
+
this.cleanupOldDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_OLD_DUTIES_COUNT);
|
|
32
|
+
this.cleanupOutdatedRollupDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_OUTDATED_ROLLUP_DUTIES_COUNT);
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Record a successful signing operation.
|
|
36
|
+
* @param dutyType - The type of duty signed
|
|
37
|
+
* @param durationMs - Duration from start of signWithProtection to completion
|
|
38
|
+
*/ recordSigningSuccess(dutyType, durationMs) {
|
|
39
|
+
const attributes = {
|
|
40
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
41
|
+
[Attributes.HA_NODE_ID]: this.nodeId
|
|
42
|
+
};
|
|
43
|
+
this.signingSuccessCount.add(1, attributes);
|
|
44
|
+
this.signingDuration.record(durationMs, attributes);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Record a DutyAlreadySignedError (expected in HA; another node signed first).
|
|
48
|
+
* @param dutyType - The type of duty
|
|
49
|
+
*/ recordDutyAlreadySigned(dutyType) {
|
|
50
|
+
const attributes = {
|
|
51
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
52
|
+
[Attributes.HA_NODE_ID]: this.nodeId
|
|
53
|
+
};
|
|
54
|
+
this.dutyAlreadySignedCount.add(1, attributes);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Record a SlashingProtectionError (attempted to sign different data for same duty).
|
|
58
|
+
* @param dutyType - The type of duty
|
|
59
|
+
*/ recordSlashingProtection(dutyType) {
|
|
60
|
+
const attributes = {
|
|
61
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
62
|
+
[Attributes.HA_NODE_ID]: this.nodeId
|
|
63
|
+
};
|
|
64
|
+
this.slashingProtectionCount.add(1, attributes);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Record a signing function failure (lock will be deleted for retry).
|
|
68
|
+
* @param dutyType - The type of duty
|
|
69
|
+
*/ recordSigningError(dutyType) {
|
|
70
|
+
const attributes = {
|
|
71
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
72
|
+
[Attributes.HA_NODE_ID]: this.nodeId
|
|
73
|
+
};
|
|
74
|
+
this.signingErrorCount.add(1, attributes);
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Record lock acquisition.
|
|
78
|
+
* @param acquired - Whether a new lock was acquired (true) or existing record found (false)
|
|
79
|
+
*/ recordLockAcquire(acquired) {
|
|
80
|
+
if (acquired) {
|
|
81
|
+
const attributes = {
|
|
82
|
+
[Attributes.HA_NODE_ID]: this.nodeId
|
|
83
|
+
};
|
|
84
|
+
this.lockAcquiredCount.add(1, attributes);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Record cleanup metrics.
|
|
89
|
+
* @param type - Type of cleanup
|
|
90
|
+
* @param count - Number of duties cleaned up
|
|
91
|
+
*/ recordCleanup(type, count) {
|
|
92
|
+
const attributes = {
|
|
93
|
+
[Attributes.HA_NODE_ID]: this.nodeId
|
|
94
|
+
};
|
|
95
|
+
if (type === 'stuck') {
|
|
96
|
+
this.cleanupStuckDutiesCount.add(count, attributes);
|
|
97
|
+
} else if (type === 'old') {
|
|
98
|
+
this.cleanupOldDutiesCount.add(count, attributes);
|
|
99
|
+
} else if (type === 'outdated_rollup') {
|
|
100
|
+
this.cleanupOutdatedRollupDutiesCount.add(count, attributes);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -1,5 +1,12 @@
|
|
|
1
|
+
import type { DateProvider } from '@aztec/foundation/timer';
|
|
2
|
+
import type { BaseSignerConfig } from '@aztec/stdlib/ha-signing';
|
|
1
3
|
import { type CheckAndRecordParams, type DeleteDutyParams, type RecordSuccessParams } from './db/types.js';
|
|
2
|
-
import type {
|
|
4
|
+
import type { HASignerMetrics } from './metrics.js';
|
|
5
|
+
import type { SlashingProtectionDatabase } from './types.js';
|
|
6
|
+
export interface SlashingProtectionServiceDeps {
|
|
7
|
+
metrics: HASignerMetrics;
|
|
8
|
+
dateProvider: DateProvider;
|
|
9
|
+
}
|
|
3
10
|
/**
|
|
4
11
|
* Slashing Protection Service
|
|
5
12
|
*
|
|
@@ -22,8 +29,11 @@ export declare class SlashingProtectionService {
|
|
|
22
29
|
private readonly pollingIntervalMs;
|
|
23
30
|
private readonly signingTimeoutMs;
|
|
24
31
|
private readonly maxStuckDutiesAgeMs;
|
|
32
|
+
private readonly metrics;
|
|
33
|
+
private readonly dateProvider;
|
|
25
34
|
private cleanupRunningPromise;
|
|
26
|
-
|
|
35
|
+
private lastOldDutiesCleanupAtMs?;
|
|
36
|
+
constructor(db: SlashingProtectionDatabase, config: BaseSignerConfig, deps: SlashingProtectionServiceDeps);
|
|
27
37
|
/**
|
|
28
38
|
* Check if a duty can be performed and acquire the lock if so.
|
|
29
39
|
*
|
|
@@ -33,7 +43,6 @@ export declare class SlashingProtectionService {
|
|
|
33
43
|
* 2. If insert succeeds, we acquired the lock - return the lockToken
|
|
34
44
|
* 3. If a record exists, handle based on status:
|
|
35
45
|
* - SIGNED: Throw appropriate error (already signed or slashing protection)
|
|
36
|
-
* - FAILED: Delete the failed record
|
|
37
46
|
* - SIGNING: Wait and poll until status changes, then handle result
|
|
38
47
|
*
|
|
39
48
|
* @returns The lockToken that must be used for recordSuccess/deleteDuty
|
|
@@ -65,7 +74,11 @@ export declare class SlashingProtectionService {
|
|
|
65
74
|
* Start running tasks.
|
|
66
75
|
* Cleanup runs immediately on start to recover from any previous crashes.
|
|
67
76
|
*/
|
|
68
|
-
|
|
77
|
+
/**
|
|
78
|
+
* Start the background cleanup task.
|
|
79
|
+
* Also performs one-time cleanup of duties with outdated rollup addresses.
|
|
80
|
+
*/
|
|
81
|
+
start(): Promise<void>;
|
|
69
82
|
/**
|
|
70
83
|
* Stop the background cleanup task.
|
|
71
84
|
*/
|
|
@@ -75,6 +88,6 @@ export declare class SlashingProtectionService {
|
|
|
75
88
|
* Should be called after stop() during graceful shutdown.
|
|
76
89
|
*/
|
|
77
90
|
close(): Promise<void>;
|
|
78
|
-
private
|
|
91
|
+
private cleanup;
|
|
79
92
|
}
|
|
80
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
93
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2xhc2hpbmdfcHJvdGVjdGlvbl9zZXJ2aWNlLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvc2xhc2hpbmdfcHJvdGVjdGlvbl9zZXJ2aWNlLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQVNBLE9BQU8sS0FBSyxFQUFFLFlBQVksRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzVELE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sMEJBQTBCLENBQUM7QUFFakUsT0FBTyxFQUNMLEtBQUssb0JBQW9CLEVBQ3pCLEtBQUssZ0JBQWdCLEVBRXJCLEtBQUssbUJBQW1CLEVBRXpCLE1BQU0sZUFBZSxDQUFDO0FBRXZCLE9BQU8sS0FBSyxFQUFFLGVBQWUsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUNwRCxPQUFPLEtBQUssRUFBRSwwQkFBMEIsRUFBRSxNQUFNLFlBQVksQ0FBQztBQUU3RCxNQUFNLFdBQVcsNkJBQTZCO0lBQzVDLE9BQU8sRUFBRSxlQUFlLENBQUM7SUFDekIsWUFBWSxFQUFFLFlBQVksQ0FBQztDQUM1QjtBQUVEOzs7Ozs7Ozs7Ozs7OztHQWNHO0FBQ0gscUJBQWEseUJBQXlCO0lBYWxDLE9BQU8sQ0FBQyxRQUFRLENBQUMsRUFBRTtJQUNuQixPQUFPLENBQUMsUUFBUSxDQUFDLE1BQU07SUFiekIsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQVM7SUFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxpQkFBaUIsQ0FBUztJQUMzQyxPQUFPLENBQUMsUUFBUSxDQUFDLGdCQUFnQixDQUFTO0lBQzFDLE9BQU8sQ0FBQyxRQUFRLENBQUMsbUJBQW1CLENBQVM7SUFFN0MsT0FBTyxDQUFDLFFBQVEsQ0FBQyxPQUFPLENBQWtCO0lBQzFDLE9BQU8sQ0FBQyxRQUFRLENBQUMsWUFBWSxDQUFlO0lBRTVDLE9BQU8sQ0FBQyxxQkFBcUIsQ0FBaUI7SUFDOUMsT0FBTyxDQUFDLHdCQUF3QixDQUFDLENBQVM7SUFFMUMsWUFDbUIsRUFBRSxFQUFFLDBCQUEwQixFQUM5QixNQUFNLEVBQUUsZ0JBQWdCLEVBQ3pDLElBQUksRUFBRSw2QkFBNkIsRUFXcEM7SUFFRDs7Ozs7Ozs7Ozs7Ozs7T0FjRztJQUNHLGNBQWMsQ0FBQyxNQUFNLEVBQUUsb0JBQW9CLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxDQXFFbEU7SUFFRDs7Ozs7O09BTUc7SUFDRyxhQUFhLENBQUMsTUFBTSxFQUFFLG1CQUFtQixHQUFHLE9BQU8sQ0FBQyxPQUFPLENBQUMsQ0EyQmpFO0lBRUQ7Ozs7OztPQU1HO0lBQ0csVUFBVSxDQUFDLE1BQU0sRUFBRSxnQkFBZ0IsR0FBRyxPQUFPLENBQUMsT0FBTyxDQUFDLENBd0IzRDtJQUVEOztPQUVHO0lBQ0gsSUFBSSxNQUFNLElBQUksTUFBTSxDQUVuQjtJQUVEOzs7T0FHRztJQUNIOzs7T0FHRztJQUNHLEtBQUssa0JBWVY7SUFFRDs7T0FFRztJQUNHLElBQUksa0JBR1Q7SUFFRDs7O09BR0c7SUFDRyxLQUFLLGtCQUdWO1lBTWEsT0FBTztDQStCdEIifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"slashing_protection_service.d.ts","sourceRoot":"","sources":["../src/slashing_protection_service.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"slashing_protection_service.d.ts","sourceRoot":"","sources":["../src/slashing_protection_service.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAEjE,OAAO,EACL,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EAErB,KAAK,mBAAmB,EAEzB,MAAM,eAAe,CAAC;AAEvB,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAC;AAE7D,MAAM,WAAW,6BAA6B;IAC5C,OAAO,EAAE,eAAe,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;CAC5B;AAED;;;;;;;;;;;;;;GAcG;AACH,qBAAa,yBAAyB;IAalC,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAbzB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAS;IAC3C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAE7C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkB;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAE5C,OAAO,CAAC,qBAAqB,CAAiB;IAC9C,OAAO,CAAC,wBAAwB,CAAC,CAAS;IAE1C,YACmB,EAAE,EAAE,0BAA0B,EAC9B,MAAM,EAAE,gBAAgB,EACzC,IAAI,EAAE,6BAA6B,EAWpC;IAED;;;;;;;;;;;;;;OAcG;IACG,cAAc,CAAC,MAAM,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAqElE;IAED;;;;;;OAMG;IACG,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,CAAC,CA2BjE;IAED;;;;;;OAMG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAwB3D;IAED;;OAEG;IACH,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED;;;OAGG;IACH;;;OAGG;IACG,KAAK,kBAYV;IAED;;OAEG;IACG,IAAI,kBAGT;IAED;;;OAGG;IACG,KAAK,kBAGV;YAMa,OAAO;CA+BtB"}
|
|
@@ -29,8 +29,11 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
29
29
|
pollingIntervalMs;
|
|
30
30
|
signingTimeoutMs;
|
|
31
31
|
maxStuckDutiesAgeMs;
|
|
32
|
+
metrics;
|
|
33
|
+
dateProvider;
|
|
32
34
|
cleanupRunningPromise;
|
|
33
|
-
|
|
35
|
+
lastOldDutiesCleanupAtMs;
|
|
36
|
+
constructor(db, config, deps){
|
|
34
37
|
this.db = db;
|
|
35
38
|
this.config = config;
|
|
36
39
|
this.log = createLogger('slashing-protection');
|
|
@@ -38,7 +41,9 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
38
41
|
this.signingTimeoutMs = config.signingTimeoutMs;
|
|
39
42
|
// Default to 144s (2x 72s Aztec slot duration) if not explicitly configured
|
|
40
43
|
this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
|
|
41
|
-
this.cleanupRunningPromise = new RunningPromise(this.
|
|
44
|
+
this.cleanupRunningPromise = new RunningPromise(this.cleanup.bind(this), this.log, this.maxStuckDutiesAgeMs);
|
|
45
|
+
this.metrics = deps.metrics;
|
|
46
|
+
this.dateProvider = deps.dateProvider;
|
|
42
47
|
}
|
|
43
48
|
/**
|
|
44
49
|
* Check if a duty can be performed and acquire the lock if so.
|
|
@@ -49,7 +54,6 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
49
54
|
* 2. If insert succeeds, we acquired the lock - return the lockToken
|
|
50
55
|
* 3. If a record exists, handle based on status:
|
|
51
56
|
* - SIGNED: Throw appropriate error (already signed or slashing protection)
|
|
52
|
-
* - FAILED: Delete the failed record
|
|
53
57
|
* - SIGNING: Wait and poll until status changes, then handle result
|
|
54
58
|
*
|
|
55
59
|
* @returns The lockToken that must be used for recordSuccess/deleteDuty
|
|
@@ -57,7 +61,7 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
57
61
|
* @throws SlashingProtectionError if attempting to sign different data for same slot/duty
|
|
58
62
|
*/ async checkAndRecord(params) {
|
|
59
63
|
const { validatorAddress, slot, dutyType, messageHash, nodeId } = params;
|
|
60
|
-
const startTime =
|
|
64
|
+
const startTime = this.dateProvider.now();
|
|
61
65
|
this.log.debug(`Checking duty: ${dutyType} for slot ${slot}`, {
|
|
62
66
|
validatorAddress: validatorAddress.toString(),
|
|
63
67
|
nodeId
|
|
@@ -67,10 +71,11 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
67
71
|
const { isNew, record } = await this.db.tryInsertOrGetExisting(params);
|
|
68
72
|
if (isNew) {
|
|
69
73
|
// We successfully acquired the lock
|
|
70
|
-
this.log.
|
|
74
|
+
this.log.verbose(`Acquired lock for duty ${dutyType} at slot ${slot}`, {
|
|
71
75
|
validatorAddress: validatorAddress.toString(),
|
|
72
76
|
nodeId
|
|
73
77
|
});
|
|
78
|
+
this.metrics.recordLockAcquire(true);
|
|
74
79
|
return record.lockToken;
|
|
75
80
|
}
|
|
76
81
|
// Record already exists - handle based on status
|
|
@@ -84,17 +89,20 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
84
89
|
existingNodeId: record.nodeId,
|
|
85
90
|
attemptingNodeId: nodeId
|
|
86
91
|
});
|
|
92
|
+
this.metrics.recordSlashingProtection(dutyType);
|
|
87
93
|
throw new SlashingProtectionError(slot, dutyType, record.blockIndexWithinCheckpoint, record.messageHash, messageHash, record.nodeId);
|
|
88
94
|
}
|
|
95
|
+
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
89
96
|
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
|
|
90
97
|
} else if (record.status === DutyStatus.SIGNING) {
|
|
91
98
|
// Another node is currently signing - check for timeout
|
|
92
|
-
if (
|
|
99
|
+
if (this.dateProvider.now() - startTime > this.signingTimeoutMs) {
|
|
93
100
|
this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
|
|
94
101
|
validatorAddress: validatorAddress.toString(),
|
|
95
102
|
timeoutMs: this.signingTimeoutMs,
|
|
96
103
|
signingNodeId: record.nodeId
|
|
97
104
|
});
|
|
105
|
+
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
98
106
|
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, 'unknown (timeout)');
|
|
99
107
|
}
|
|
100
108
|
// Wait and poll
|
|
@@ -116,11 +124,11 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
116
124
|
*
|
|
117
125
|
* @returns true if the update succeeded, false if token didn't match
|
|
118
126
|
*/ async recordSuccess(params) {
|
|
119
|
-
const { validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
|
|
127
|
+
const { rollupAddress, validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
|
|
120
128
|
const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
|
|
121
|
-
const success = await this.db.updateDutySigned(validatorAddress, slot, dutyType, signature.toString(), lockToken, blockIndexWithinCheckpoint);
|
|
129
|
+
const success = await this.db.updateDutySigned(rollupAddress, validatorAddress, slot, dutyType, signature.toString(), lockToken, blockIndexWithinCheckpoint);
|
|
122
130
|
if (success) {
|
|
123
|
-
this.log.
|
|
131
|
+
this.log.verbose(`Recorded successful signing for duty ${dutyType} at slot ${slot}`, {
|
|
124
132
|
validatorAddress: validatorAddress.toString(),
|
|
125
133
|
nodeId
|
|
126
134
|
});
|
|
@@ -139,9 +147,9 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
139
147
|
*
|
|
140
148
|
* @returns true if the delete succeeded, false if token didn't match
|
|
141
149
|
*/ async deleteDuty(params) {
|
|
142
|
-
const { validatorAddress, slot, dutyType, lockToken } = params;
|
|
150
|
+
const { rollupAddress, validatorAddress, slot, dutyType, lockToken } = params;
|
|
143
151
|
const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
|
|
144
|
-
const success = await this.db.deleteDuty(validatorAddress, slot, dutyType, lockToken, blockIndexWithinCheckpoint);
|
|
152
|
+
const success = await this.db.deleteDuty(rollupAddress, validatorAddress, slot, dutyType, lockToken, blockIndexWithinCheckpoint);
|
|
145
153
|
if (success) {
|
|
146
154
|
this.log.info(`Deleted duty ${dutyType} at slot ${slot} to allow retry`, {
|
|
147
155
|
validatorAddress: validatorAddress.toString()
|
|
@@ -161,7 +169,18 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
161
169
|
/**
|
|
162
170
|
* Start running tasks.
|
|
163
171
|
* Cleanup runs immediately on start to recover from any previous crashes.
|
|
164
|
-
*/
|
|
172
|
+
*/ /**
|
|
173
|
+
* Start the background cleanup task.
|
|
174
|
+
* Also performs one-time cleanup of duties with outdated rollup addresses.
|
|
175
|
+
*/ async start() {
|
|
176
|
+
// One-time cleanup at startup: remove duties from previous rollup versions
|
|
177
|
+
const numOutdatedRollupDuties = await this.db.cleanupOutdatedRollupDuties(this.config.l1Contracts.rollupAddress);
|
|
178
|
+
if (numOutdatedRollupDuties > 0) {
|
|
179
|
+
this.log.info(`Cleaned up ${numOutdatedRollupDuties} duties with outdated rollup address at startup`, {
|
|
180
|
+
currentRollupAddress: this.config.l1Contracts.rollupAddress.toString()
|
|
181
|
+
});
|
|
182
|
+
this.metrics.recordCleanup('outdated_rollup', numOutdatedRollupDuties);
|
|
183
|
+
}
|
|
165
184
|
this.cleanupRunningPromise.start();
|
|
166
185
|
this.log.info('Slashing protection service started', {
|
|
167
186
|
nodeId: this.config.nodeId
|
|
@@ -183,14 +202,35 @@ import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
|
183
202
|
this.log.info('Slashing protection database connection closed');
|
|
184
203
|
}
|
|
185
204
|
/**
|
|
186
|
-
*
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
205
|
+
* Periodic cleanup of stuck duties and optionally old signed duties.
|
|
206
|
+
* Runs in the background via RunningPromise.
|
|
207
|
+
*/ async cleanup() {
|
|
208
|
+
// 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status)
|
|
209
|
+
const numStuckDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
|
|
210
|
+
if (numStuckDuties > 0) {
|
|
211
|
+
this.log.verbose(`Cleaned up ${numStuckDuties} stuck duties`, {
|
|
191
212
|
nodeId: this.config.nodeId,
|
|
192
213
|
maxStuckDutiesAgeMs: this.maxStuckDutiesAgeMs
|
|
193
214
|
});
|
|
215
|
+
this.metrics.recordCleanup('stuck', numStuckDuties);
|
|
216
|
+
}
|
|
217
|
+
// 2. Clean up old signed duties if configured
|
|
218
|
+
// we shouldn't run this as often as stuck duty cleanup.
|
|
219
|
+
if (this.config.cleanupOldDutiesAfterHours !== undefined) {
|
|
220
|
+
const maxAgeMs = this.config.cleanupOldDutiesAfterHours * 60 * 60 * 1000;
|
|
221
|
+
const nowMs = this.dateProvider.now();
|
|
222
|
+
const shouldRun = this.lastOldDutiesCleanupAtMs === undefined || nowMs - this.lastOldDutiesCleanupAtMs >= maxAgeMs;
|
|
223
|
+
if (shouldRun) {
|
|
224
|
+
const numOldDuties = await this.db.cleanupOldDuties(maxAgeMs);
|
|
225
|
+
this.lastOldDutiesCleanupAtMs = nowMs;
|
|
226
|
+
if (numOldDuties > 0) {
|
|
227
|
+
this.log.verbose(`Cleaned up ${numOldDuties} old signed duties`, {
|
|
228
|
+
cleanupOldDutiesAfterHours: this.config.cleanupOldDutiesAfterHours,
|
|
229
|
+
maxAgeMs
|
|
230
|
+
});
|
|
231
|
+
this.metrics.recordCleanup('old', numOldDuties);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
194
234
|
}
|
|
195
235
|
}
|
|
196
236
|
}
|