@aztec/validator-ha-signer 0.0.1-commit.1142ef1 → 0.0.1-commit.11bf3dd6e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -37
- package/dest/db/index.d.ts +2 -1
- package/dest/db/index.d.ts.map +1 -1
- package/dest/db/index.js +1 -0
- package/dest/db/lmdb.d.ts +66 -0
- package/dest/db/lmdb.d.ts.map +1 -0
- package/dest/db/lmdb.js +189 -0
- package/dest/db/migrations/1_initial-schema.d.ts +4 -2
- package/dest/db/migrations/1_initial-schema.d.ts.map +1 -1
- package/dest/db/migrations/1_initial-schema.js +34 -4
- package/dest/db/migrations/2_add-checkpoint-number.d.ts +7 -0
- package/dest/db/migrations/2_add-checkpoint-number.d.ts.map +1 -0
- package/dest/db/migrations/2_add-checkpoint-number.js +17 -0
- package/dest/db/postgres.d.ts +37 -6
- package/dest/db/postgres.d.ts.map +1 -1
- package/dest/db/postgres.js +88 -28
- package/dest/db/schema.d.ts +22 -11
- package/dest/db/schema.d.ts.map +1 -1
- package/dest/db/schema.js +55 -21
- package/dest/db/types.d.ts +116 -34
- package/dest/db/types.d.ts.map +1 -1
- package/dest/db/types.js +58 -8
- package/dest/errors.d.ts +9 -5
- package/dest/errors.d.ts.map +1 -1
- package/dest/errors.js +7 -4
- package/dest/factory.d.ts +42 -15
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +80 -15
- package/dest/metrics.d.ts +51 -0
- package/dest/metrics.d.ts.map +1 -0
- package/dest/metrics.js +103 -0
- package/dest/migrations.d.ts +1 -1
- package/dest/migrations.d.ts.map +1 -1
- package/dest/migrations.js +13 -2
- package/dest/slashing_protection_service.d.ts +25 -6
- package/dest/slashing_protection_service.d.ts.map +1 -1
- package/dest/slashing_protection_service.js +74 -22
- package/dest/test/pglite_pool.d.ts +92 -0
- package/dest/test/pglite_pool.d.ts.map +1 -0
- package/dest/test/pglite_pool.js +210 -0
- package/dest/types.d.ts +41 -16
- package/dest/types.d.ts.map +1 -1
- package/dest/types.js +5 -1
- package/dest/validator_ha_signer.d.ts +18 -13
- package/dest/validator_ha_signer.d.ts.map +1 -1
- package/dest/validator_ha_signer.js +47 -36
- package/package.json +15 -10
- package/src/db/index.ts +1 -0
- package/src/db/lmdb.ts +265 -0
- package/src/db/migrations/1_initial-schema.ts +35 -4
- package/src/db/migrations/2_add-checkpoint-number.ts +19 -0
- package/src/db/postgres.ts +111 -27
- package/src/db/schema.ts +57 -21
- package/src/db/types.ts +169 -33
- package/src/errors.ts +7 -2
- package/src/factory.ts +99 -15
- package/src/metrics.ts +138 -0
- package/src/migrations.ts +17 -1
- package/src/slashing_protection_service.ts +119 -27
- package/src/test/pglite_pool.ts +256 -0
- package/src/types.ts +71 -16
- package/src/validator_ha_signer.ts +67 -45
- package/dest/config.d.ts +0 -47
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -64
- package/src/config.ts +0 -116
package/src/metrics.ts
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import {
|
|
2
|
+
Attributes,
|
|
3
|
+
type Histogram,
|
|
4
|
+
Metrics,
|
|
5
|
+
type TelemetryClient,
|
|
6
|
+
type UpDownCounter,
|
|
7
|
+
createUpDownCounterWithDefault,
|
|
8
|
+
} from '@aztec/telemetry-client';
|
|
9
|
+
|
|
10
|
+
export type HACleanupType = 'stuck' | 'old' | 'outdated_rollup';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Metrics for HA signer tracking signing operations, lock acquisition, and cleanup.
|
|
14
|
+
*/
|
|
15
|
+
export class HASignerMetrics {
|
|
16
|
+
// Signing lifecycle metrics
|
|
17
|
+
private signingDuration: Histogram;
|
|
18
|
+
private signingSuccessCount: UpDownCounter;
|
|
19
|
+
private dutyAlreadySignedCount: UpDownCounter;
|
|
20
|
+
private slashingProtectionCount: UpDownCounter;
|
|
21
|
+
private signingErrorCount: UpDownCounter;
|
|
22
|
+
|
|
23
|
+
// Lock acquisition metrics
|
|
24
|
+
private lockAcquiredCount: UpDownCounter;
|
|
25
|
+
|
|
26
|
+
// Cleanup metrics
|
|
27
|
+
private cleanupStuckDutiesCount: UpDownCounter;
|
|
28
|
+
private cleanupOldDutiesCount: UpDownCounter;
|
|
29
|
+
private cleanupOutdatedRollupDutiesCount: UpDownCounter;
|
|
30
|
+
|
|
31
|
+
constructor(
|
|
32
|
+
client: TelemetryClient,
|
|
33
|
+
private nodeId: string,
|
|
34
|
+
name = 'HASignerMetrics',
|
|
35
|
+
) {
|
|
36
|
+
const meter = client.getMeter(name);
|
|
37
|
+
|
|
38
|
+
// Signing lifecycle
|
|
39
|
+
this.signingDuration = meter.createHistogram(Metrics.HA_SIGNER_SIGNING_DURATION);
|
|
40
|
+
this.signingSuccessCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_SUCCESS_COUNT);
|
|
41
|
+
this.dutyAlreadySignedCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_DUTY_ALREADY_SIGNED_COUNT);
|
|
42
|
+
this.slashingProtectionCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SLASHING_PROTECTION_COUNT);
|
|
43
|
+
this.signingErrorCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_SIGNING_ERROR_COUNT);
|
|
44
|
+
|
|
45
|
+
// Lock acquisition
|
|
46
|
+
this.lockAcquiredCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_LOCK_ACQUIRED_COUNT);
|
|
47
|
+
|
|
48
|
+
// Cleanup
|
|
49
|
+
this.cleanupStuckDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_STUCK_DUTIES_COUNT);
|
|
50
|
+
this.cleanupOldDutiesCount = createUpDownCounterWithDefault(meter, Metrics.HA_SIGNER_CLEANUP_OLD_DUTIES_COUNT);
|
|
51
|
+
this.cleanupOutdatedRollupDutiesCount = createUpDownCounterWithDefault(
|
|
52
|
+
meter,
|
|
53
|
+
Metrics.HA_SIGNER_CLEANUP_OUTDATED_ROLLUP_DUTIES_COUNT,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Record a successful signing operation.
|
|
59
|
+
* @param dutyType - The type of duty signed
|
|
60
|
+
* @param durationMs - Duration from start of signWithProtection to completion
|
|
61
|
+
*/
|
|
62
|
+
public recordSigningSuccess(dutyType: string, durationMs: number): void {
|
|
63
|
+
const attributes = {
|
|
64
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
65
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
66
|
+
};
|
|
67
|
+
this.signingSuccessCount.add(1, attributes);
|
|
68
|
+
this.signingDuration.record(durationMs, attributes);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Record a DutyAlreadySignedError (expected in HA; another node signed first).
|
|
73
|
+
* @param dutyType - The type of duty
|
|
74
|
+
*/
|
|
75
|
+
public recordDutyAlreadySigned(dutyType: string): void {
|
|
76
|
+
const attributes = {
|
|
77
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
78
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
79
|
+
};
|
|
80
|
+
this.dutyAlreadySignedCount.add(1, attributes);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Record a SlashingProtectionError (attempted to sign different data for same duty).
|
|
85
|
+
* @param dutyType - The type of duty
|
|
86
|
+
*/
|
|
87
|
+
public recordSlashingProtection(dutyType: string): void {
|
|
88
|
+
const attributes = {
|
|
89
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
90
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
91
|
+
};
|
|
92
|
+
this.slashingProtectionCount.add(1, attributes);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Record a signing function failure (lock will be deleted for retry).
|
|
97
|
+
* @param dutyType - The type of duty
|
|
98
|
+
*/
|
|
99
|
+
public recordSigningError(dutyType: string): void {
|
|
100
|
+
const attributes = {
|
|
101
|
+
[Attributes.HA_DUTY_TYPE]: dutyType,
|
|
102
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
103
|
+
};
|
|
104
|
+
this.signingErrorCount.add(1, attributes);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Record lock acquisition.
|
|
109
|
+
* @param acquired - Whether a new lock was acquired (true) or existing record found (false)
|
|
110
|
+
*/
|
|
111
|
+
public recordLockAcquire(acquired: boolean): void {
|
|
112
|
+
if (acquired) {
|
|
113
|
+
const attributes = {
|
|
114
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
115
|
+
};
|
|
116
|
+
this.lockAcquiredCount.add(1, attributes);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Record cleanup metrics.
|
|
122
|
+
* @param type - Type of cleanup
|
|
123
|
+
* @param count - Number of duties cleaned up
|
|
124
|
+
*/
|
|
125
|
+
public recordCleanup(type: HACleanupType, count: number): void {
|
|
126
|
+
const attributes = {
|
|
127
|
+
[Attributes.HA_NODE_ID]: this.nodeId,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
if (type === 'stuck') {
|
|
131
|
+
this.cleanupStuckDutiesCount.add(count, attributes);
|
|
132
|
+
} else if (type === 'old') {
|
|
133
|
+
this.cleanupOldDutiesCount.add(count, attributes);
|
|
134
|
+
} else if (type === 'outdated_rollup') {
|
|
135
|
+
this.cleanupOutdatedRollupDutiesCount.add(count, attributes);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
package/src/migrations.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { createLogger } from '@aztec/foundation/log';
|
|
5
5
|
|
|
6
|
+
import { readdirSync } from 'fs';
|
|
6
7
|
import { runner } from 'node-pg-migrate';
|
|
7
8
|
import { dirname, join } from 'path';
|
|
8
9
|
import { fileURLToPath } from 'url';
|
|
@@ -30,17 +31,32 @@ export async function runMigrations(databaseUrl: string, options: RunMigrationsO
|
|
|
30
31
|
|
|
31
32
|
const log = createLogger('validator-ha-signer:migrations');
|
|
32
33
|
|
|
34
|
+
const migrationsDir = join(__dirname, 'db', 'migrations');
|
|
35
|
+
|
|
33
36
|
try {
|
|
34
37
|
log.info(`Running migrations ${direction}...`);
|
|
35
38
|
|
|
39
|
+
// Filter out .d.ts and .d.ts.map files - node-pg-migrate only needs .js files
|
|
40
|
+
const migrationFiles = readdirSync(migrationsDir);
|
|
41
|
+
const jsMigrationFiles = migrationFiles.filter(
|
|
42
|
+
file => file.endsWith('.js') && !file.endsWith('.d.ts') && !file.endsWith('.d.ts.map'),
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
if (jsMigrationFiles.length === 0) {
|
|
46
|
+
log.info('No migration files found');
|
|
47
|
+
return [];
|
|
48
|
+
}
|
|
49
|
+
|
|
36
50
|
const appliedMigrations = await runner({
|
|
37
51
|
databaseUrl,
|
|
38
|
-
dir:
|
|
52
|
+
dir: migrationsDir,
|
|
39
53
|
direction,
|
|
40
54
|
migrationsTable: 'pgmigrations',
|
|
41
55
|
count: direction === 'down' ? 1 : Infinity,
|
|
42
56
|
verbose,
|
|
43
57
|
log: msg => (verbose ? log.info(msg) : log.debug(msg)),
|
|
58
|
+
// Ignore TypeScript declaration files - node-pg-migrate will try to import them otherwise
|
|
59
|
+
ignorePattern: '.*\\.d\\.(ts|js)$|.*\\.d\\.ts\\.map$',
|
|
44
60
|
});
|
|
45
61
|
|
|
46
62
|
if (appliedMigrations.length === 0) {
|
|
@@ -7,10 +7,24 @@
|
|
|
7
7
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
8
8
|
import { RunningPromise } from '@aztec/foundation/promise';
|
|
9
9
|
import { sleep } from '@aztec/foundation/sleep';
|
|
10
|
+
import type { DateProvider } from '@aztec/foundation/timer';
|
|
11
|
+
import type { BaseSignerConfig } from '@aztec/stdlib/ha-signing';
|
|
10
12
|
|
|
11
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
type CheckAndRecordParams,
|
|
15
|
+
type DeleteDutyParams,
|
|
16
|
+
DutyStatus,
|
|
17
|
+
type RecordSuccessParams,
|
|
18
|
+
getBlockIndexFromDutyIdentifier,
|
|
19
|
+
} from './db/types.js';
|
|
12
20
|
import { DutyAlreadySignedError, SlashingProtectionError } from './errors.js';
|
|
13
|
-
import type {
|
|
21
|
+
import type { HASignerMetrics } from './metrics.js';
|
|
22
|
+
import type { SlashingProtectionDatabase } from './types.js';
|
|
23
|
+
|
|
24
|
+
export interface SlashingProtectionServiceDeps {
|
|
25
|
+
metrics: HASignerMetrics;
|
|
26
|
+
dateProvider: DateProvider;
|
|
27
|
+
}
|
|
14
28
|
|
|
15
29
|
/**
|
|
16
30
|
* Slashing Protection Service
|
|
@@ -31,22 +45,28 @@ export class SlashingProtectionService {
|
|
|
31
45
|
private readonly log: Logger;
|
|
32
46
|
private readonly pollingIntervalMs: number;
|
|
33
47
|
private readonly signingTimeoutMs: number;
|
|
48
|
+
private readonly maxStuckDutiesAgeMs: number;
|
|
49
|
+
|
|
50
|
+
private readonly metrics: HASignerMetrics;
|
|
51
|
+
private readonly dateProvider: DateProvider;
|
|
34
52
|
|
|
35
53
|
private cleanupRunningPromise: RunningPromise;
|
|
54
|
+
private lastOldDutiesCleanupAtMs?: number;
|
|
36
55
|
|
|
37
56
|
constructor(
|
|
38
57
|
private readonly db: SlashingProtectionDatabase,
|
|
39
|
-
private readonly config:
|
|
58
|
+
private readonly config: BaseSignerConfig,
|
|
59
|
+
deps: SlashingProtectionServiceDeps,
|
|
40
60
|
) {
|
|
41
61
|
this.log = createLogger('slashing-protection');
|
|
42
62
|
this.pollingIntervalMs = config.pollingIntervalMs;
|
|
43
63
|
this.signingTimeoutMs = config.signingTimeoutMs;
|
|
64
|
+
// Default to 144s (2x 72s Aztec slot duration) if not explicitly configured
|
|
65
|
+
this.maxStuckDutiesAgeMs = config.maxStuckDutiesAgeMs ?? 144_000;
|
|
44
66
|
|
|
45
|
-
this.cleanupRunningPromise = new RunningPromise(
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
this.config.maxStuckDutiesAgeMs,
|
|
49
|
-
);
|
|
67
|
+
this.cleanupRunningPromise = new RunningPromise(this.cleanup.bind(this), this.log, this.maxStuckDutiesAgeMs);
|
|
68
|
+
this.metrics = deps.metrics;
|
|
69
|
+
this.dateProvider = deps.dateProvider;
|
|
50
70
|
}
|
|
51
71
|
|
|
52
72
|
/**
|
|
@@ -58,7 +78,6 @@ export class SlashingProtectionService {
|
|
|
58
78
|
* 2. If insert succeeds, we acquired the lock - return the lockToken
|
|
59
79
|
* 3. If a record exists, handle based on status:
|
|
60
80
|
* - SIGNED: Throw appropriate error (already signed or slashing protection)
|
|
61
|
-
* - FAILED: Delete the failed record
|
|
62
81
|
* - SIGNING: Wait and poll until status changes, then handle result
|
|
63
82
|
*
|
|
64
83
|
* @returns The lockToken that must be used for recordSuccess/deleteDuty
|
|
@@ -67,7 +86,7 @@ export class SlashingProtectionService {
|
|
|
67
86
|
*/
|
|
68
87
|
async checkAndRecord(params: CheckAndRecordParams): Promise<string> {
|
|
69
88
|
const { validatorAddress, slot, dutyType, messageHash, nodeId } = params;
|
|
70
|
-
const startTime =
|
|
89
|
+
const startTime = this.dateProvider.now();
|
|
71
90
|
|
|
72
91
|
this.log.debug(`Checking duty: ${dutyType} for slot ${slot}`, {
|
|
73
92
|
validatorAddress: validatorAddress.toString(),
|
|
@@ -80,10 +99,11 @@ export class SlashingProtectionService {
|
|
|
80
99
|
|
|
81
100
|
if (isNew) {
|
|
82
101
|
// We successfully acquired the lock
|
|
83
|
-
this.log.
|
|
102
|
+
this.log.verbose(`Acquired lock for duty ${dutyType} at slot ${slot}`, {
|
|
84
103
|
validatorAddress: validatorAddress.toString(),
|
|
85
104
|
nodeId,
|
|
86
105
|
});
|
|
106
|
+
this.metrics.recordLockAcquire(true);
|
|
87
107
|
return record.lockToken;
|
|
88
108
|
}
|
|
89
109
|
|
|
@@ -98,18 +118,28 @@ export class SlashingProtectionService {
|
|
|
98
118
|
existingNodeId: record.nodeId,
|
|
99
119
|
attemptingNodeId: nodeId,
|
|
100
120
|
});
|
|
101
|
-
|
|
121
|
+
this.metrics.recordSlashingProtection(dutyType);
|
|
122
|
+
throw new SlashingProtectionError(
|
|
123
|
+
slot,
|
|
124
|
+
dutyType,
|
|
125
|
+
record.blockIndexWithinCheckpoint,
|
|
126
|
+
record.messageHash,
|
|
127
|
+
messageHash,
|
|
128
|
+
record.nodeId,
|
|
129
|
+
);
|
|
102
130
|
}
|
|
103
|
-
|
|
131
|
+
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
132
|
+
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, record.nodeId);
|
|
104
133
|
} else if (record.status === DutyStatus.SIGNING) {
|
|
105
134
|
// Another node is currently signing - check for timeout
|
|
106
|
-
if (
|
|
135
|
+
if (this.dateProvider.now() - startTime > this.signingTimeoutMs) {
|
|
107
136
|
this.log.warn(`Timeout waiting for signing to complete for duty ${dutyType} at slot ${slot}`, {
|
|
108
137
|
validatorAddress: validatorAddress.toString(),
|
|
109
138
|
timeoutMs: this.signingTimeoutMs,
|
|
110
139
|
signingNodeId: record.nodeId,
|
|
111
140
|
});
|
|
112
|
-
|
|
141
|
+
this.metrics.recordDutyAlreadySigned(dutyType);
|
|
142
|
+
throw new DutyAlreadySignedError(slot, dutyType, record.blockIndexWithinCheckpoint, 'unknown (timeout)');
|
|
113
143
|
}
|
|
114
144
|
|
|
115
145
|
// Wait and poll
|
|
@@ -133,12 +163,21 @@ export class SlashingProtectionService {
|
|
|
133
163
|
* @returns true if the update succeeded, false if token didn't match
|
|
134
164
|
*/
|
|
135
165
|
async recordSuccess(params: RecordSuccessParams): Promise<boolean> {
|
|
136
|
-
const { validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
|
|
166
|
+
const { rollupAddress, validatorAddress, slot, dutyType, signature, nodeId, lockToken } = params;
|
|
167
|
+
const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
|
|
137
168
|
|
|
138
|
-
const success = await this.db.updateDutySigned(
|
|
169
|
+
const success = await this.db.updateDutySigned(
|
|
170
|
+
rollupAddress,
|
|
171
|
+
validatorAddress,
|
|
172
|
+
slot,
|
|
173
|
+
dutyType,
|
|
174
|
+
signature.toString(),
|
|
175
|
+
lockToken,
|
|
176
|
+
blockIndexWithinCheckpoint,
|
|
177
|
+
);
|
|
139
178
|
|
|
140
179
|
if (success) {
|
|
141
|
-
this.log.
|
|
180
|
+
this.log.verbose(`Recorded successful signing for duty ${dutyType} at slot ${slot}`, {
|
|
142
181
|
validatorAddress: validatorAddress.toString(),
|
|
143
182
|
nodeId,
|
|
144
183
|
});
|
|
@@ -160,9 +199,17 @@ export class SlashingProtectionService {
|
|
|
160
199
|
* @returns true if the delete succeeded, false if token didn't match
|
|
161
200
|
*/
|
|
162
201
|
async deleteDuty(params: DeleteDutyParams): Promise<boolean> {
|
|
163
|
-
const { validatorAddress, slot, dutyType, lockToken } = params;
|
|
202
|
+
const { rollupAddress, validatorAddress, slot, dutyType, lockToken } = params;
|
|
203
|
+
const blockIndexWithinCheckpoint = getBlockIndexFromDutyIdentifier(params);
|
|
164
204
|
|
|
165
|
-
const success = await this.db.deleteDuty(
|
|
205
|
+
const success = await this.db.deleteDuty(
|
|
206
|
+
rollupAddress,
|
|
207
|
+
validatorAddress,
|
|
208
|
+
slot,
|
|
209
|
+
dutyType,
|
|
210
|
+
lockToken,
|
|
211
|
+
blockIndexWithinCheckpoint,
|
|
212
|
+
);
|
|
166
213
|
|
|
167
214
|
if (success) {
|
|
168
215
|
this.log.info(`Deleted duty ${dutyType} at slot ${slot} to allow retry`, {
|
|
@@ -188,7 +235,20 @@ export class SlashingProtectionService {
|
|
|
188
235
|
* Start running tasks.
|
|
189
236
|
* Cleanup runs immediately on start to recover from any previous crashes.
|
|
190
237
|
*/
|
|
191
|
-
|
|
238
|
+
/**
|
|
239
|
+
* Start the background cleanup task.
|
|
240
|
+
* Also performs one-time cleanup of duties with outdated rollup addresses.
|
|
241
|
+
*/
|
|
242
|
+
async start() {
|
|
243
|
+
// One-time cleanup at startup: remove duties from previous rollup versions
|
|
244
|
+
const numOutdatedRollupDuties = await this.db.cleanupOutdatedRollupDuties(this.config.l1Contracts.rollupAddress);
|
|
245
|
+
if (numOutdatedRollupDuties > 0) {
|
|
246
|
+
this.log.info(`Cleaned up ${numOutdatedRollupDuties} duties with outdated rollup address at startup`, {
|
|
247
|
+
currentRollupAddress: this.config.l1Contracts.rollupAddress.toString(),
|
|
248
|
+
});
|
|
249
|
+
this.metrics.recordCleanup('outdated_rollup', numOutdatedRollupDuties);
|
|
250
|
+
}
|
|
251
|
+
|
|
192
252
|
this.cleanupRunningPromise.start();
|
|
193
253
|
this.log.info('Slashing protection service started', { nodeId: this.config.nodeId });
|
|
194
254
|
}
|
|
@@ -202,15 +262,47 @@ export class SlashingProtectionService {
|
|
|
202
262
|
}
|
|
203
263
|
|
|
204
264
|
/**
|
|
205
|
-
*
|
|
265
|
+
* Close the database connection.
|
|
266
|
+
* Should be called after stop() during graceful shutdown.
|
|
267
|
+
*/
|
|
268
|
+
async close() {
|
|
269
|
+
await this.db.close();
|
|
270
|
+
this.log.info('Slashing protection database connection closed');
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Periodic cleanup of stuck duties and optionally old signed duties.
|
|
275
|
+
* Runs in the background via RunningPromise.
|
|
206
276
|
*/
|
|
207
|
-
private async
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
277
|
+
private async cleanup() {
|
|
278
|
+
// 1. Clean up stuck duties (our own node's duties that got stuck in 'signing' status)
|
|
279
|
+
const numStuckDuties = await this.db.cleanupOwnStuckDuties(this.config.nodeId, this.maxStuckDutiesAgeMs);
|
|
280
|
+
if (numStuckDuties > 0) {
|
|
281
|
+
this.log.verbose(`Cleaned up ${numStuckDuties} stuck duties`, {
|
|
211
282
|
nodeId: this.config.nodeId,
|
|
212
|
-
maxStuckDutiesAgeMs: this.
|
|
283
|
+
maxStuckDutiesAgeMs: this.maxStuckDutiesAgeMs,
|
|
213
284
|
});
|
|
285
|
+
this.metrics.recordCleanup('stuck', numStuckDuties);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// 2. Clean up old signed duties if configured
|
|
289
|
+
// we shouldn't run this as often as stuck duty cleanup.
|
|
290
|
+
if (this.config.cleanupOldDutiesAfterHours !== undefined) {
|
|
291
|
+
const maxAgeMs = this.config.cleanupOldDutiesAfterHours * 60 * 60 * 1000;
|
|
292
|
+
const nowMs = this.dateProvider.now();
|
|
293
|
+
const shouldRun =
|
|
294
|
+
this.lastOldDutiesCleanupAtMs === undefined || nowMs - this.lastOldDutiesCleanupAtMs >= maxAgeMs;
|
|
295
|
+
if (shouldRun) {
|
|
296
|
+
const numOldDuties = await this.db.cleanupOldDuties(maxAgeMs);
|
|
297
|
+
this.lastOldDutiesCleanupAtMs = nowMs;
|
|
298
|
+
if (numOldDuties > 0) {
|
|
299
|
+
this.log.verbose(`Cleaned up ${numOldDuties} old signed duties`, {
|
|
300
|
+
cleanupOldDutiesAfterHours: this.config.cleanupOldDutiesAfterHours,
|
|
301
|
+
maxAgeMs,
|
|
302
|
+
});
|
|
303
|
+
this.metrics.recordCleanup('old', numOldDuties);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
214
306
|
}
|
|
215
307
|
}
|
|
216
308
|
}
|
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vendored pg-compatible Pool/Client wrapper for PGlite.
|
|
3
|
+
*
|
|
4
|
+
* Copied from @middle-management/pglite-pg-adapter v0.0.3
|
|
5
|
+
* https://www.npmjs.com/package/@middle-management/pglite-pg-adapter
|
|
6
|
+
*
|
|
7
|
+
* Modifications:
|
|
8
|
+
* - Converted to ESM and TypeScript
|
|
9
|
+
* - Uses PGliteInterface instead of PGlite class to avoid TypeScript
|
|
10
|
+
* type mismatches from ESM/CJS dual package resolution with private fields
|
|
11
|
+
* - Simplified rowCount calculation to handle CTEs properly
|
|
12
|
+
*/
|
|
13
|
+
import type { PGliteInterface } from '@electric-sql/pglite';
|
|
14
|
+
import { EventEmitter } from 'events';
|
|
15
|
+
import type { QueryResult, QueryResultRow } from 'pg';
|
|
16
|
+
import { Readable, Writable } from 'stream';
|
|
17
|
+
|
|
18
|
+
export interface PoolConfig {
|
|
19
|
+
pglite: PGliteInterface;
|
|
20
|
+
max?: number;
|
|
21
|
+
min?: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
interface ClientConfig {
|
|
25
|
+
pglite: PGliteInterface;
|
|
26
|
+
host?: string;
|
|
27
|
+
port?: number;
|
|
28
|
+
ssl?: boolean;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class Client extends EventEmitter {
|
|
32
|
+
protected pglite: PGliteInterface;
|
|
33
|
+
protected _connected = false;
|
|
34
|
+
readonly host: string;
|
|
35
|
+
readonly port: number;
|
|
36
|
+
readonly ssl: boolean;
|
|
37
|
+
readonly connection: object;
|
|
38
|
+
|
|
39
|
+
// Stub implementations for pg compatibility
|
|
40
|
+
readonly copyFrom = (): Writable => new Writable();
|
|
41
|
+
readonly copyTo = (): Readable => new Readable();
|
|
42
|
+
readonly pauseDrain = (): void => {};
|
|
43
|
+
readonly resumeDrain = (): void => {};
|
|
44
|
+
readonly escapeLiteral = (str: string): string => `'${str.replace(/'/g, "''")}'`;
|
|
45
|
+
readonly escapeIdentifier = (str: string): string => `"${str.replace(/"/g, '""')}"`;
|
|
46
|
+
readonly setTypeParser = (): void => {};
|
|
47
|
+
readonly getTypeParser = (): ((value: string) => unknown) => (value: string) => value;
|
|
48
|
+
|
|
49
|
+
constructor(config: ClientConfig) {
|
|
50
|
+
super();
|
|
51
|
+
this.pglite = config.pglite;
|
|
52
|
+
this.host = config.host || 'localhost';
|
|
53
|
+
this.port = config.port || 5432;
|
|
54
|
+
this.ssl = typeof config.ssl === 'boolean' ? config.ssl : !!config.ssl;
|
|
55
|
+
this.connection = {};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
connect(): Promise<void> {
|
|
59
|
+
if (this._connected) {
|
|
60
|
+
return Promise.resolve();
|
|
61
|
+
}
|
|
62
|
+
this._connected = true;
|
|
63
|
+
this.emit('connect');
|
|
64
|
+
return Promise.resolve();
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
end(): Promise<void> {
|
|
68
|
+
if (!this._connected) {
|
|
69
|
+
return Promise.resolve();
|
|
70
|
+
}
|
|
71
|
+
this._connected = false;
|
|
72
|
+
this.emit('end');
|
|
73
|
+
return Promise.resolve();
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async query<R extends QueryResultRow = any>(text: string, values?: any[]): Promise<QueryResult<R>> {
|
|
77
|
+
if (!this._connected) {
|
|
78
|
+
throw new Error('Client is not connected');
|
|
79
|
+
}
|
|
80
|
+
const result = await this.pglite.query<R>(text, values);
|
|
81
|
+
return this.convertPGliteResult(result);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
protected convertPGliteResult<R extends QueryResultRow>(result: {
|
|
85
|
+
rows: R[];
|
|
86
|
+
fields: Array<{ name: string; dataTypeID: number }>;
|
|
87
|
+
affectedRows?: number;
|
|
88
|
+
}): QueryResult<R> {
|
|
89
|
+
return {
|
|
90
|
+
command: '',
|
|
91
|
+
rowCount: 'affectedRows' in result ? (result.affectedRows ?? 0) : result.rows.length,
|
|
92
|
+
oid: 0,
|
|
93
|
+
fields: result.fields.map(field => ({
|
|
94
|
+
name: field.name,
|
|
95
|
+
tableID: 0,
|
|
96
|
+
columnID: 0,
|
|
97
|
+
dataTypeID: field.dataTypeID,
|
|
98
|
+
dataTypeSize: -1,
|
|
99
|
+
dataTypeModifier: -1,
|
|
100
|
+
format: 'text',
|
|
101
|
+
})),
|
|
102
|
+
rows: result.rows,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
get connected(): boolean {
|
|
107
|
+
return this._connected;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export class Pool extends EventEmitter {
|
|
112
|
+
private clients: PoolClient[] = [];
|
|
113
|
+
private availableClients: PoolClient[] = [];
|
|
114
|
+
private waitingQueue: Array<(client: PoolClient) => void> = [];
|
|
115
|
+
private _ended = false;
|
|
116
|
+
private pglite: PGliteInterface;
|
|
117
|
+
private _config: PoolConfig;
|
|
118
|
+
|
|
119
|
+
readonly expiredCount = 0;
|
|
120
|
+
readonly options: PoolConfig;
|
|
121
|
+
|
|
122
|
+
constructor(config: PoolConfig) {
|
|
123
|
+
super();
|
|
124
|
+
this._config = { max: 10, min: 0, ...config };
|
|
125
|
+
this.pglite = config.pglite;
|
|
126
|
+
this.options = config;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
get totalCount(): number {
|
|
130
|
+
return this.clients.length;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
get idleCount(): number {
|
|
134
|
+
return this.availableClients.length;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
get waitingCount(): number {
|
|
138
|
+
return this.waitingQueue.length;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
get ending(): boolean {
|
|
142
|
+
return this._ended;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
get ended(): boolean {
|
|
146
|
+
return this._ended;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
connect(): Promise<PoolClient> {
|
|
150
|
+
if (this._ended) {
|
|
151
|
+
return Promise.reject(new Error('Pool is ended'));
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
if (this.availableClients.length > 0) {
|
|
155
|
+
const client = this.availableClients.pop()!;
|
|
156
|
+
client._markInUse();
|
|
157
|
+
return Promise.resolve(client);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
if (this.clients.length < (this._config.max || 10)) {
|
|
161
|
+
const client = new PoolClient(this.pglite, this);
|
|
162
|
+
this.clients.push(client);
|
|
163
|
+
return Promise.resolve(client);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return new Promise(resolve => {
|
|
167
|
+
this.waitingQueue.push(resolve);
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
async query<R extends QueryResultRow = any>(text: string, values?: any[]): Promise<QueryResult<R>> {
|
|
172
|
+
const client = await this.connect();
|
|
173
|
+
try {
|
|
174
|
+
return await client.query<R>(text, values);
|
|
175
|
+
} finally {
|
|
176
|
+
client.release();
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
releaseClient(client: PoolClient): void {
|
|
181
|
+
const index = this.clients.indexOf(client);
|
|
182
|
+
if (index !== -1) {
|
|
183
|
+
client._markAvailable();
|
|
184
|
+
if (this.waitingQueue.length > 0) {
|
|
185
|
+
const resolve = this.waitingQueue.shift()!;
|
|
186
|
+
client._markInUse();
|
|
187
|
+
resolve(client);
|
|
188
|
+
} else {
|
|
189
|
+
this.availableClients.push(client);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
end(): Promise<void> {
|
|
195
|
+
this._ended = true;
|
|
196
|
+
this.clients.forEach(client => client._markReleased());
|
|
197
|
+
this.clients = [];
|
|
198
|
+
this.availableClients = [];
|
|
199
|
+
this.emit('end');
|
|
200
|
+
return Promise.resolve();
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export class PoolClient extends Client {
|
|
205
|
+
private pool: Pool;
|
|
206
|
+
private _released = false;
|
|
207
|
+
private _inUse = true;
|
|
208
|
+
private _userReleased = false;
|
|
209
|
+
|
|
210
|
+
constructor(pglite: PGliteInterface, pool: Pool) {
|
|
211
|
+
super({ pglite });
|
|
212
|
+
this.pool = pool;
|
|
213
|
+
this._connected = true;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
override async query<R extends QueryResultRow = any>(text: string, values?: any[]): Promise<QueryResult<R>> {
|
|
217
|
+
if (this._userReleased && !this._inUse) {
|
|
218
|
+
throw new Error('Client has been released back to the pool');
|
|
219
|
+
}
|
|
220
|
+
const result = await this.pglite.query<R>(text, values);
|
|
221
|
+
return this.convertPGliteResult(result);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
release(): void {
|
|
225
|
+
if (this._released || this._userReleased) {
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
this._userReleased = true;
|
|
229
|
+
this.pool.releaseClient(this);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
override end(): Promise<void> {
|
|
233
|
+
this.release();
|
|
234
|
+
return Promise.resolve();
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
_markInUse(): void {
|
|
238
|
+
this._inUse = true;
|
|
239
|
+
this._userReleased = false;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
_markAvailable(): void {
|
|
243
|
+
this._inUse = false;
|
|
244
|
+
this._userReleased = false;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
_markReleased(): void {
|
|
248
|
+
this._released = true;
|
|
249
|
+
this._inUse = false;
|
|
250
|
+
this._userReleased = true;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
override get connected(): boolean {
|
|
254
|
+
return this._connected && !this._released;
|
|
255
|
+
}
|
|
256
|
+
}
|