@aztec/prover-node 5.0.0-private.20260318 → 5.0.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +506 -0
- package/dest/actions/download-epoch-proving-job.js +1 -1
- package/dest/actions/rerun-epoch-proving-job.d.ts +4 -3
- package/dest/actions/rerun-epoch-proving-job.d.ts.map +1 -1
- package/dest/actions/rerun-epoch-proving-job.js +103 -21
- package/dest/bin/run-failed-epoch.js +1 -3
- package/dest/checkpoint-store.d.ts +83 -0
- package/dest/checkpoint-store.d.ts.map +1 -0
- package/dest/checkpoint-store.js +181 -0
- package/dest/config.d.ts +1 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +1 -1
- package/dest/factory.d.ts +1 -1
- package/dest/factory.d.ts.map +1 -1
- package/dest/factory.js +22 -8
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/job/checkpoint-prover.d.ts +134 -0
- package/dest/job/checkpoint-prover.d.ts.map +1 -0
- package/dest/job/checkpoint-prover.js +350 -0
- package/dest/job/epoch-session.d.ts +146 -0
- package/dest/job/epoch-session.d.ts.map +1 -0
- package/dest/job/epoch-session.js +709 -0
- package/dest/job/top-tree-job.d.ts +82 -0
- package/dest/job/top-tree-job.d.ts.map +1 -0
- package/dest/job/top-tree-job.js +152 -0
- package/dest/metrics.d.ts +29 -5
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +73 -9
- package/dest/monitors/epoch-monitor.js +6 -2
- package/dest/proof-publishing-service.d.ts +159 -0
- package/dest/proof-publishing-service.d.ts.map +1 -0
- package/dest/proof-publishing-service.js +334 -0
- package/dest/prover-node-publisher.d.ts +18 -11
- package/dest/prover-node-publisher.d.ts.map +1 -1
- package/dest/prover-node-publisher.js +195 -57
- package/dest/prover-node.d.ts +96 -68
- package/dest/prover-node.d.ts.map +1 -1
- package/dest/prover-node.js +382 -227
- package/dest/prover-publisher-factory.d.ts +2 -2
- package/dest/prover-publisher-factory.d.ts.map +1 -1
- package/dest/prover-publisher-factory.js +3 -3
- package/dest/session-manager.d.ts +158 -0
- package/dest/session-manager.d.ts.map +1 -0
- package/dest/session-manager.js +452 -0
- package/dest/test/index.d.ts +7 -6
- package/dest/test/index.d.ts.map +1 -1
- package/package.json +23 -23
- package/src/actions/download-epoch-proving-job.ts +1 -1
- package/src/actions/rerun-epoch-proving-job.ts +114 -28
- package/src/bin/run-failed-epoch.ts +1 -2
- package/src/checkpoint-store.ts +213 -0
- package/src/config.ts +2 -1
- package/src/factory.ts +18 -10
- package/src/index.ts +1 -0
- package/src/job/checkpoint-prover.ts +465 -0
- package/src/job/epoch-session.ts +424 -0
- package/src/job/top-tree-job.ts +227 -0
- package/src/metrics.ts +88 -12
- package/src/monitors/epoch-monitor.ts +2 -2
- package/src/proof-publishing-service.ts +424 -0
- package/src/prover-node-publisher.ts +220 -67
- package/src/prover-node.ts +439 -249
- package/src/prover-publisher-factory.ts +3 -3
- package/src/session-manager.ts +552 -0
- package/src/test/index.ts +6 -6
- package/dest/job/epoch-proving-job.d.ts +0 -63
- package/dest/job/epoch-proving-job.d.ts.map +0 -1
- package/dest/job/epoch-proving-job.js +0 -762
- package/src/job/epoch-proving-job.ts +0 -465
|
@@ -15,11 +15,11 @@ export declare class ProverPublisherFactory {
|
|
|
15
15
|
telemetry?: TelemetryClient;
|
|
16
16
|
}, bindings?: LoggerBindings | undefined);
|
|
17
17
|
start(): Promise<void>;
|
|
18
|
-
stop(): void
|
|
18
|
+
stop(): Promise<void>;
|
|
19
19
|
/**
|
|
20
20
|
* Creates a new Prover Publisher instance.
|
|
21
21
|
* @returns A new ProverNodePublisher instance.
|
|
22
22
|
*/
|
|
23
23
|
create(): Promise<ProverNodePublisher>;
|
|
24
24
|
}
|
|
25
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
25
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHJvdmVyLXB1Ymxpc2hlci1mYWN0b3J5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvcHJvdmVyLXB1Ymxpc2hlci1mYWN0b3J5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLDJCQUEyQixDQUFDO0FBQ2hFLE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQzdELE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sbUNBQW1DLENBQUM7QUFDMUUsT0FBTyxLQUFLLEVBQUUsY0FBYyxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDNUQsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUMzRixPQUFPLEtBQUssRUFBRSxlQUFlLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUUvRCxPQUFPLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUVqRSxxQkFBYSxzQkFBc0I7SUFFL0IsT0FBTyxDQUFDLE1BQU07SUFDZCxPQUFPLENBQUMsSUFBSTtJQUtaLE9BQU8sQ0FBQyxRQUFRLENBQUM7SUFQbkIsWUFDVSxNQUFNLEVBQUUsb0JBQW9CLEdBQUcscUJBQXFCLEVBQ3BELElBQUksRUFBRTtRQUNaLGNBQWMsRUFBRSxjQUFjLENBQUM7UUFDL0IsZ0JBQWdCLEVBQUUsZ0JBQWdCLENBQUMsU0FBUyxDQUFDLENBQUM7UUFDOUMsU0FBUyxDQUFDLEVBQUUsZUFBZSxDQUFDO0tBQzdCLEVBQ08sUUFBUSxDQUFDLDRCQUFnQixFQUMvQjtJQUVTLEtBQUssa0JBRWpCO0lBRVksSUFBSSxrQkFFaEI7SUFFRDs7O09BR0c7SUFDVSxNQUFNLElBQUksT0FBTyxDQUFDLG1CQUFtQixDQUFDLENBV2xEO0NBQ0YifQ==
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"prover-publisher-factory.d.ts","sourceRoot":"","sources":["../src/prover-publisher-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ,CAAC;IAPnB,YACU,MAAM,EAAE,oBAAoB,GAAG,qBAAqB,EACpD,IAAI,EAAE;QACZ,cAAc,EAAE,cAAc,CAAC;QAC/B,gBAAgB,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC9C,SAAS,CAAC,EAAE,eAAe,CAAC;KAC7B,EACO,QAAQ,CAAC,4BAAgB,EAC/B;IAES,KAAK,kBAEjB;
|
|
1
|
+
{"version":3,"file":"prover-publisher-factory.d.ts","sourceRoot":"","sources":["../src/prover-publisher-factory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAC3F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE/D,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,qBAAa,sBAAsB;IAE/B,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,IAAI;IAKZ,OAAO,CAAC,QAAQ,CAAC;IAPnB,YACU,MAAM,EAAE,oBAAoB,GAAG,qBAAqB,EACpD,IAAI,EAAE;QACZ,cAAc,EAAE,cAAc,CAAC;QAC/B,gBAAgB,EAAE,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAC9C,SAAS,CAAC,EAAE,eAAe,CAAC;KAC7B,EACO,QAAQ,CAAC,4BAAgB,EAC/B;IAES,KAAK,kBAEjB;IAEY,IAAI,kBAEhB;IAED;;;OAGG;IACU,MAAM,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAWlD;CACF"}
|
|
@@ -9,10 +9,10 @@ export class ProverPublisherFactory {
|
|
|
9
9
|
this.bindings = bindings;
|
|
10
10
|
}
|
|
11
11
|
async start() {
|
|
12
|
-
await this.deps.publisherManager.
|
|
12
|
+
await this.deps.publisherManager.start();
|
|
13
13
|
}
|
|
14
|
-
stop() {
|
|
15
|
-
this.deps.publisherManager.
|
|
14
|
+
async stop() {
|
|
15
|
+
await this.deps.publisherManager.stop();
|
|
16
16
|
}
|
|
17
17
|
/**
|
|
18
18
|
* Creates a new Prover Publisher instance.
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import { type EpochNumber } from '@aztec/foundation/branded-types';
|
|
2
|
+
import type { EthAddress } from '@aztec/foundation/eth-address';
|
|
3
|
+
import { type LoggerBindings } from '@aztec/foundation/log';
|
|
4
|
+
import type { DateProvider } from '@aztec/foundation/timer';
|
|
5
|
+
import type { EpochProverFactory } from '@aztec/prover-client';
|
|
6
|
+
import type { L2BlockSource } from '@aztec/stdlib/block';
|
|
7
|
+
import type { EpochProvingJobState } from '@aztec/stdlib/interfaces/server';
|
|
8
|
+
import type { CheckpointStore } from './checkpoint-store.js';
|
|
9
|
+
import { CheckpointProver } from './job/checkpoint-prover.js';
|
|
10
|
+
import type { EpochProvingJobData } from './job/epoch-proving-job-data.js';
|
|
11
|
+
import { EpochSession, type EpochSessionDeps, type EpochSessionHooks, type SessionSpec } from './job/epoch-session.js';
|
|
12
|
+
import type { ProverNodeJobMetrics } from './metrics.js';
|
|
13
|
+
import type { ProofPublishingService } from './proof-publishing-service.js';
|
|
14
|
+
/** Trigger payload for `reconcile`. */
|
|
15
|
+
export type ReconcileTrigger = {
|
|
16
|
+
kind: 'checkpoint';
|
|
17
|
+
epoch: EpochNumber;
|
|
18
|
+
} | {
|
|
19
|
+
kind: 'prune';
|
|
20
|
+
affectedEpochs: EpochNumber[];
|
|
21
|
+
} | {
|
|
22
|
+
kind: 'tick';
|
|
23
|
+
} | {
|
|
24
|
+
kind: 'start-proof';
|
|
25
|
+
spec: SessionSpec;
|
|
26
|
+
};
|
|
27
|
+
/** Config bag for session lifecycle decisions. */
|
|
28
|
+
export type SessionManagerConfig = {
|
|
29
|
+
/** Cap on the number of non-terminal sessions (full + partial). 0 disables. */
|
|
30
|
+
maxPendingJobs: number;
|
|
31
|
+
/** Interval at which the internal periodic tick fires `reconcile({ kind: 'tick' })`. */
|
|
32
|
+
tickIntervalMs: number;
|
|
33
|
+
/** Forwarded to every session: delay before top-tree proving, letting late reorgs settle. */
|
|
34
|
+
finalizationDelayMs: number | undefined;
|
|
35
|
+
};
|
|
36
|
+
export type SessionManagerDeps = {
|
|
37
|
+
checkpointStore: CheckpointStore;
|
|
38
|
+
l2BlockSource: Pick<L2BlockSource, 'isEpochComplete' | 'getCheckpoints' | 'getL1Constants' | 'getBlockNumber' | 'getBlockData'>;
|
|
39
|
+
proverFactory: EpochProverFactory;
|
|
40
|
+
proverId: EthAddress;
|
|
41
|
+
publishingService: ProofPublishingService;
|
|
42
|
+
metrics: ProverNodeJobMetrics;
|
|
43
|
+
dateProvider: DateProvider;
|
|
44
|
+
config: SessionManagerConfig;
|
|
45
|
+
/**
|
|
46
|
+
* Optional callback fired when a session terminates with `failed`. The session manager
|
|
47
|
+
* doesn't own the failure-upload action; it just notifies the owner.
|
|
48
|
+
*/
|
|
49
|
+
onSessionFailed?: (session: EpochSession) => Promise<void>;
|
|
50
|
+
bindings?: LoggerBindings;
|
|
51
|
+
};
|
|
52
|
+
/**
|
|
53
|
+
* Owns the lifecycle of every `EpochSession`. Each L2BlockStream event and periodic tick
|
|
54
|
+
* arrives via a dedicated entry point (`onCheckpointAdded`, `onPrune`, `onTick`, etc.) which
|
|
55
|
+
* schedules a `reconcile(trigger)` on a serial queue. Reconcile walks both session
|
|
56
|
+
* maps, cancels any session whose canonical content has shifted, re-creates it with
|
|
57
|
+
* the same spec but new content, and opens fresh full sessions for any epoch implicated
|
|
58
|
+
* by the trigger.
|
|
59
|
+
*/
|
|
60
|
+
export declare class SessionManager {
|
|
61
|
+
private readonly deps;
|
|
62
|
+
private readonly log;
|
|
63
|
+
private readonly fullSessions;
|
|
64
|
+
private readonly partialSessions;
|
|
65
|
+
/**
|
|
66
|
+
* Serialises every reconcile call. The trigger sources (L2BlockStream events, the
|
|
67
|
+
* periodic tick, JSON-RPC `startProof`) run independently, so without this queue two
|
|
68
|
+
* reconciles could interleave on the `await session.cancel(...)` step and orphan a
|
|
69
|
+
* freshly-constructed session.
|
|
70
|
+
*/
|
|
71
|
+
private readonly reconcileQueue;
|
|
72
|
+
/** Cached L1 constants, populated on first read. */
|
|
73
|
+
private cachedL1Constants;
|
|
74
|
+
/**
|
|
75
|
+
* Highest epoch for which the periodic tick has successfully created a full session.
|
|
76
|
+
* Monotonic high-water mark: once the tick observes a session for epoch X, it stops
|
|
77
|
+
* trying to open one — even if that session subsequently fails (only a new checkpoint
|
|
78
|
+
* event reopens it). Crucially, the mark only advances when a session actually exists
|
|
79
|
+
* post-open, so transient blockers (atMaxSessionLimit, archiver still indexing) leave
|
|
80
|
+
* the mark in place and the next tick retries.
|
|
81
|
+
*/
|
|
82
|
+
private lastTickEpoch;
|
|
83
|
+
/** Test-only hooks applied to every session this manager constructs. */
|
|
84
|
+
private sessionHooks;
|
|
85
|
+
/** Periodic tick that nudges reconcile to pick up newly-complete epochs. Started by `start()`. */
|
|
86
|
+
private epochTicker;
|
|
87
|
+
constructor(deps: SessionManagerDeps);
|
|
88
|
+
/**
|
|
89
|
+
* Starts the periodic tick. Separated from the constructor so tests can drive `onTick()`
|
|
90
|
+
* manually without the background ticker interleaving. Idempotent.
|
|
91
|
+
*/
|
|
92
|
+
start(): void;
|
|
93
|
+
/**
|
|
94
|
+
* Installs hooks applied to every session constructed from now on. Used by the e2e
|
|
95
|
+
* harness to interpose around top-tree proving (gate it, override it, observe it)
|
|
96
|
+
* without monkey-patching the orchestrator factory.
|
|
97
|
+
*/
|
|
98
|
+
setSessionHooks(hooks: EpochSessionHooks): void;
|
|
99
|
+
/** Every live (non-terminal) session. */
|
|
100
|
+
allSessions(): EpochSession[];
|
|
101
|
+
/** Returns the full session for `epoch`, if any. */
|
|
102
|
+
getFullSession(epoch: EpochNumber): EpochSession | undefined;
|
|
103
|
+
/** Returns the partial session for `spec`, if any. */
|
|
104
|
+
getPartialSession(spec: SessionSpec): EpochSession | undefined;
|
|
105
|
+
/** Observability summary used by the prover-node API. */
|
|
106
|
+
getJobs(): {
|
|
107
|
+
uuid: string;
|
|
108
|
+
status: EpochProvingJobState;
|
|
109
|
+
epochNumber: EpochNumber;
|
|
110
|
+
}[];
|
|
111
|
+
/** Called by ProverNode after a chain-checkpointed event has been added to the store. */
|
|
112
|
+
onCheckpointAdded(epoch: EpochNumber): Promise<void>;
|
|
113
|
+
/** Called by ProverNode after a chain-pruned event has flipped store provers to pruned. */
|
|
114
|
+
onPrune(affectedEpochs: EpochNumber[]): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Called periodically by ProverNode's ticker. Picks up epochs that have become complete
|
|
117
|
+
* by time without a fresh checkpoint event (e.g. the epoch's last slots are empty), and
|
|
118
|
+
* advances to the next epoch once the previous one is proven on L1.
|
|
119
|
+
*/
|
|
120
|
+
onTick(): Promise<void>;
|
|
121
|
+
/**
|
|
122
|
+
* Schedules a proof attempt for the supplied epoch and returns the job id without waiting for
|
|
123
|
+
* the proof to complete — proving can far outlast an HTTP request, so callers poll `getJobs()`
|
|
124
|
+
* for the outcome. Every session — full or partial — begins at the epoch's first slot; the
|
|
125
|
+
* partial's spec stops at the last canonical slot, while the full's stops at the epoch's last
|
|
126
|
+
* slot. Dedupes against any existing session covering the same range, returning its id.
|
|
127
|
+
*/
|
|
128
|
+
startProof(epoch: EpochNumber): Promise<string>;
|
|
129
|
+
/** Stops the tick, drains the reconcile queue, and cancels every live session. */
|
|
130
|
+
stop(): Promise<void>;
|
|
131
|
+
private scheduleReconcile;
|
|
132
|
+
private reconcile;
|
|
133
|
+
private recreateInvalidSessions;
|
|
134
|
+
private openFullSessionIfReady;
|
|
135
|
+
private openPartialSession;
|
|
136
|
+
protected constructSession(spec: SessionSpec, checkpoints: readonly CheckpointProver[]): EpochSession;
|
|
137
|
+
/** Extracted for test override. */
|
|
138
|
+
protected doConstructSession(spec: SessionSpec, checkpoints: readonly CheckpointProver[], sessionDeps: EpochSessionDeps, hooks?: EpochSessionHooks): EpochSession;
|
|
139
|
+
private buildSessionDeps;
|
|
140
|
+
private computeDeadline;
|
|
141
|
+
private runSession;
|
|
142
|
+
/**
|
|
143
|
+
* Builds the EpochProvingJobData snapshot for failure upload. Includes every checkpoint
|
|
144
|
+
* referenced by the session, regardless of whether sub-tree proving completed —
|
|
145
|
+
* partial state is still useful for post-mortem analysis.
|
|
146
|
+
*/
|
|
147
|
+
static buildSessionProvingData(session: EpochSession): EpochProvingJobData;
|
|
148
|
+
private atMaxSessionLimit;
|
|
149
|
+
private epochsForTrigger;
|
|
150
|
+
private nextUnprovenEpoch;
|
|
151
|
+
private canonicalCheckpointsForSpec;
|
|
152
|
+
private fireAndForgetCancel;
|
|
153
|
+
private checkpointsMatch;
|
|
154
|
+
private archiverFullyCovered;
|
|
155
|
+
private isProvenChainEncompassing;
|
|
156
|
+
private getL1Constants;
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic2Vzc2lvbi1tYW5hZ2VyLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvc2Vzc2lvbi1tYW5hZ2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBZSxLQUFLLFdBQVcsRUFBRSxNQUFNLGlDQUFpQyxDQUFDO0FBRWhGLE9BQU8sS0FBSyxFQUFFLFVBQVUsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBQ2hFLE9BQU8sRUFBZSxLQUFLLGNBQWMsRUFBZ0IsTUFBTSx1QkFBdUIsQ0FBQztBQUd2RixPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsTUFBTSx5QkFBeUIsQ0FBQztBQUM1RCxPQUFPLEtBQUssRUFBRSxrQkFBa0IsRUFBRSxNQUFNLHNCQUFzQixDQUFDO0FBQy9ELE9BQU8sS0FBSyxFQUFFLGFBQWEsRUFBRSxNQUFNLHFCQUFxQixDQUFDO0FBUXpELE9BQU8sS0FBSyxFQUFFLG9CQUFvQixFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFFNUUsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0sdUJBQXVCLENBQUM7QUFDN0QsT0FBTyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sNEJBQTRCLENBQUM7QUFDOUQsT0FBTyxLQUFLLEVBQUUsbUJBQW1CLEVBQUUsTUFBTSxpQ0FBaUMsQ0FBQztBQUMzRSxPQUFPLEVBQ0wsWUFBWSxFQUNaLEtBQUssZ0JBQWdCLEVBQ3JCLEtBQUssaUJBQWlCLEVBRXRCLEtBQUssV0FBVyxFQUVqQixNQUFNLHdCQUF3QixDQUFDO0FBQ2hDLE9BQU8sS0FBSyxFQUFFLG9CQUFvQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBQ3pELE9BQU8sS0FBSyxFQUFFLHNCQUFzQixFQUFFLE1BQU0sK0JBQStCLENBQUM7QUFFNUUsdUNBQXVDO0FBQ3ZDLE1BQU0sTUFBTSxnQkFBZ0IsR0FDeEI7SUFBRSxJQUFJLEVBQUUsWUFBWSxDQUFDO0lBQUMsS0FBSyxFQUFFLFdBQVcsQ0FBQTtDQUFFLEdBQzFDO0lBQUUsSUFBSSxFQUFFLE9BQU8sQ0FBQztJQUFDLGNBQWMsRUFBRSxXQUFXLEVBQUUsQ0FBQTtDQUFFLEdBQ2hEO0lBQUUsSUFBSSxFQUFFLE1BQU0sQ0FBQTtDQUFFLEdBQ2hCO0lBQUUsSUFBSSxFQUFFLGFBQWEsQ0FBQztJQUFDLElBQUksRUFBRSxXQUFXLENBQUE7Q0FBRSxDQUFDO0FBRS9DLGtEQUFrRDtBQUNsRCxNQUFNLE1BQU0sb0JBQW9CLEdBQUc7SUFDakMsK0VBQStFO0lBQy9FLGNBQWMsRUFBRSxNQUFNLENBQUM7SUFDdkIsd0ZBQXdGO0lBQ3hGLGNBQWMsRUFBRSxNQUFNLENBQUM7SUFDdkIsNkZBQTZGO0lBQzdGLG1CQUFtQixFQUFFLE1BQU0sR0FBRyxTQUFTLENBQUM7Q0FDekMsQ0FBQztBQUVGLE1BQU0sTUFBTSxrQkFBa0IsR0FBRztJQUMvQixlQUFlLEVBQUUsZUFBZSxDQUFDO0lBQ2pDLGFBQWEsRUFBRSxJQUFJLENBQ2pCLGFBQWEsRUFDYixpQkFBaUIsR0FBRyxnQkFBZ0IsR0FBRyxnQkFBZ0IsR0FBRyxnQkFBZ0IsR0FBRyxjQUFjLENBQzVGLENBQUM7SUFDRixhQUFhLEVBQUUsa0JBQWtCLENBQUM7SUFDbEMsUUFBUSxFQUFFLFVBQVUsQ0FBQztJQUNyQixpQkFBaUIsRUFBRSxzQkFBc0IsQ0FBQztJQUMxQyxPQUFPLEVBQUUsb0JBQW9CLENBQUM7SUFDOUIsWUFBWSxFQUFFLFlBQVksQ0FBQztJQUMzQixNQUFNLEVBQUUsb0JBQW9CLENBQUM7SUFDN0I7OztPQUdHO0lBQ0gsZUFBZSxDQUFDLEVBQUUsQ0FBQyxPQUFPLEVBQUUsWUFBWSxLQUFLLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUMzRCxRQUFRLENBQUMsRUFBRSxjQUFjLENBQUM7Q0FDM0IsQ0FBQztBQUVGOzs7Ozs7O0dBT0c7QUFDSCxxQkFBYSxjQUFjO0lBMkJiLE9BQU8sQ0FBQyxRQUFRLENBQUMsSUFBSTtJQTFCakMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQVM7SUFDN0IsT0FBTyxDQUFDLFFBQVEsQ0FBQyxZQUFZLENBQTZDO0lBQzFFLE9BQU8sQ0FBQyxRQUFRLENBQUMsZUFBZSxDQUF3QztJQUN4RTs7Ozs7T0FLRztJQUNILE9BQU8sQ0FBQyxRQUFRLENBQUMsY0FBYyxDQUFxQjtJQUNwRCxvREFBb0Q7SUFDcEQsT0FBTyxDQUFDLGlCQUFpQixDQUFnQztJQUN6RDs7Ozs7OztPQU9HO0lBQ0gsT0FBTyxDQUFDLGFBQWEsQ0FBMEI7SUFDL0Msd0VBQXdFO0lBQ3hFLE9BQU8sQ0FBQyxZQUFZLENBQWdDO0lBQ3BELGtHQUFrRztJQUNsRyxPQUFPLENBQUMsV0FBVyxDQUE2QjtJQUVoRCxZQUE2QixJQUFJLEVBQUUsa0JBQWtCLEVBR3BEO0lBRUQ7OztPQUdHO0lBQ0ksS0FBSyxJQUFJLElBQUksQ0FNbkI7SUFFRDs7OztPQUlHO0lBQ0ksZUFBZSxDQUFDLEtBQUssRUFBRSxpQkFBaUIsR0FBRyxJQUFJLENBRXJEO0lBSUQseUNBQXlDO0lBQ2xDLFdBQVcsSUFBSSxZQUFZLEVBQUUsQ0FFbkM7SUFFRCxvREFBb0Q7SUFDN0MsY0FBYyxDQUFDLEtBQUssRUFBRSxXQUFXLEdBQUcsWUFBWSxHQUFHLFNBQVMsQ0FFbEU7SUFFRCxzREFBc0Q7SUFDL0MsaUJBQWlCLENBQUMsSUFBSSxFQUFFLFdBQVcsR0FBRyxZQUFZLEdBQUcsU0FBUyxDQUVwRTtJQUVELHlEQUF5RDtJQUNsRCxPQUFPLElBQUk7UUFBRSxJQUFJLEVBQUUsTUFBTSxDQUFDO1FBQUMsTUFBTSxFQUFFLG9CQUFvQixDQUFDO1FBQUMsV0FBVyxFQUFFLFdBQVcsQ0FBQTtLQUFFLEVBQUUsQ0FNM0Y7SUFJRCx5RkFBeUY7SUFDbEYsaUJBQWlCLENBQUMsS0FBSyxFQUFFLFdBQVcsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBRTFEO0lBRUQsMkZBQTJGO0lBQ3BGLE9BQU8sQ0FBQyxjQUFjLEVBQUUsV0FBVyxFQUFFLEdBQUcsT0FBTyxDQUFDLElBQUksQ0FBQyxDQUUzRDtJQUVEOzs7O09BSUc7SUFDSSxNQUFNLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUU3QjtJQUlEOzs7Ozs7T0FNRztJQUNVLFVBQVUsQ0FBQyxLQUFLLEVBQUUsV0FBVyxHQUFHLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FvQzNEO0lBRUQsa0ZBQWtGO0lBQ3JFLElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBS2pDO0lBSUQsT0FBTyxDQUFDLGlCQUFpQjtZQUlYLFNBQVM7SUEwQnZCLE9BQU8sQ0FBQyx1QkFBdUI7WUFtQ2pCLHNCQUFzQjtJQStCcEMsT0FBTyxDQUFDLGtCQUFrQjtJQTJCMUIsU0FBUyxDQUFDLGdCQUFnQixDQUFDLElBQUksRUFBRSxXQUFXLEVBQUUsV0FBVyxFQUFFLFNBQVMsZ0JBQWdCLEVBQUUsR0FBRyxZQUFZLENBRXBHO0lBRUQsbUNBQW1DO0lBQ25DLFNBQVMsQ0FBQyxrQkFBa0IsQ0FDMUIsSUFBSSxFQUFFLFdBQVcsRUFDakIsV0FBVyxFQUFFLFNBQVMsZ0JBQWdCLEVBQUUsRUFDeEMsV0FBVyxFQUFFLGdCQUFnQixFQUM3QixLQUFLLENBQUMsRUFBRSxpQkFBaUIsR0FDeEIsWUFBWSxDQUVkO0lBRUQsT0FBTyxDQUFDLGdCQUFnQjtJQWdCeEIsT0FBTyxDQUFDLGVBQWU7WUFRVCxVQUFVO0lBa0J4Qjs7OztPQUlHO0lBQ0gsT0FBYyx1QkFBdUIsQ0FBQyxPQUFPLEVBQUUsWUFBWSxHQUFHLG1CQUFtQixDQWtCaEY7SUFJRCxPQUFPLENBQUMsaUJBQWlCO1lBU1gsZ0JBQWdCO1lBd0JoQixpQkFBaUI7SUFVL0IsT0FBTyxDQUFDLDJCQUEyQjtJQUluQyxPQUFPLENBQUMsbUJBQW1CO0lBSTNCLE9BQU8sQ0FBQyxnQkFBZ0I7SUFZeEIsT0FBTyxDQUFDLG9CQUFvQjtZQWtCZCx5QkFBeUI7WUFVekIsY0FBYztDQU03QiJ9
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-manager.d.ts","sourceRoot":"","sources":["../src/session-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAEhF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAChE,OAAO,EAAe,KAAK,cAAc,EAAgB,MAAM,uBAAuB,CAAC;AAGvF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAC5D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAQzD,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAE5E,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AAC3E,OAAO,EACL,YAAY,EACZ,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EAEtB,KAAK,WAAW,EAEjB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACzD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AAE5E,uCAAuC;AACvC,MAAM,MAAM,gBAAgB,GACxB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,WAAW,CAAA;CAAE,GAC1C;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,cAAc,EAAE,WAAW,EAAE,CAAA;CAAE,GAChD;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,IAAI,EAAE,WAAW,CAAA;CAAE,CAAC;AAE/C,kDAAkD;AAClD,MAAM,MAAM,oBAAoB,GAAG;IACjC,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB,wFAAwF;IACxF,cAAc,EAAE,MAAM,CAAC;IACvB,6FAA6F;IAC7F,mBAAmB,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,EAAE,eAAe,CAAC;IACjC,aAAa,EAAE,IAAI,CACjB,aAAa,EACb,iBAAiB,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,cAAc,CAC5F,CAAC;IACF,aAAa,EAAE,kBAAkB,CAAC;IAClC,QAAQ,EAAE,UAAU,CAAC;IACrB,iBAAiB,EAAE,sBAAsB,CAAC;IAC1C,OAAO,EAAE,oBAAoB,CAAC;IAC9B,YAAY,EAAE,YAAY,CAAC;IAC3B,MAAM,EAAE,oBAAoB,CAAC;IAC7B;;;OAGG;IACH,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3D,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC3B,CAAC;AAEF;;;;;;;GAOG;AACH,qBAAa,cAAc;IA2Bb,OAAO,CAAC,QAAQ,CAAC,IAAI;IA1BjC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA6C;IAC1E,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAwC;IACxE;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,oDAAoD;IACpD,OAAO,CAAC,iBAAiB,CAAgC;IACzD;;;;;;;OAOG;IACH,OAAO,CAAC,aAAa,CAA0B;IAC/C,wEAAwE;IACxE,OAAO,CAAC,YAAY,CAAgC;IACpD,kGAAkG;IAClG,OAAO,CAAC,WAAW,CAA6B;IAEhD,YAA6B,IAAI,EAAE,kBAAkB,EAGpD;IAED;;;OAGG;IACI,KAAK,IAAI,IAAI,CAMnB;IAED;;;;OAIG;IACI,eAAe,CAAC,KAAK,EAAE,iBAAiB,GAAG,IAAI,CAErD;IAID,yCAAyC;IAClC,WAAW,IAAI,YAAY,EAAE,CAEnC;IAED,oDAAoD;IAC7C,cAAc,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,GAAG,SAAS,CAElE;IAED,sDAAsD;IAC/C,iBAAiB,CAAC,IAAI,EAAE,WAAW,GAAG,YAAY,GAAG,SAAS,CAEpE;IAED,yDAAyD;IAClD,OAAO,IAAI;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAC;QAAC,WAAW,EAAE,WAAW,CAAA;KAAE,EAAE,CAM3F;IAID,yFAAyF;IAClF,iBAAiB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAE1D;IAED,2FAA2F;IACpF,OAAO,CAAC,cAAc,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAE3D;IAED;;;;OAIG;IACI,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7B;IAID;;;;;;OAMG;IACU,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAoC3D;IAED,kFAAkF;IACrE,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAKjC;IAID,OAAO,CAAC,iBAAiB;YAIX,SAAS;IA0BvB,OAAO,CAAC,uBAAuB;YAmCjB,sBAAsB;IA+BpC,OAAO,CAAC,kBAAkB;IA2B1B,SAAS,CAAC,gBAAgB,CAAC,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,SAAS,gBAAgB,EAAE,GAAG,YAAY,CAEpG;IAED,mCAAmC;IACnC,SAAS,CAAC,kBAAkB,CAC1B,IAAI,EAAE,WAAW,EACjB,WAAW,EAAE,SAAS,gBAAgB,EAAE,EACxC,WAAW,EAAE,gBAAgB,EAC7B,KAAK,CAAC,EAAE,iBAAiB,GACxB,YAAY,CAEd;IAED,OAAO,CAAC,gBAAgB;IAgBxB,OAAO,CAAC,eAAe;YAQT,UAAU;IAkBxB;;;;OAIG;IACH,OAAc,uBAAuB,CAAC,OAAO,EAAE,YAAY,GAAG,mBAAmB,CAkBhF;IAID,OAAO,CAAC,iBAAiB;YASX,gBAAgB;YAwBhB,iBAAiB;IAU/B,OAAO,CAAC,2BAA2B;IAInC,OAAO,CAAC,mBAAmB;IAI3B,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,oBAAoB;YAkBd,yBAAyB;YAUzB,cAAc;CAM7B"}
|
|
@@ -0,0 +1,452 @@
|
|
|
1
|
+
import { BlockNumber } from '@aztec/foundation/branded-types';
|
|
2
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
3
|
+
import { SerialQueue } from '@aztec/foundation/queue';
|
|
4
|
+
import { RunningPromise } from '@aztec/foundation/running-promise';
|
|
5
|
+
import { getEpochAtSlot, getProofSubmissionDeadlineTimestamp, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers';
|
|
6
|
+
import { CheckpointProver } from './job/checkpoint-prover.js';
|
|
7
|
+
import { EpochSession, specKey } from './job/epoch-session.js';
|
|
8
|
+
/**
|
|
9
|
+
* Owns the lifecycle of every `EpochSession`. Each L2BlockStream event and periodic tick
|
|
10
|
+
* arrives via a dedicated entry point (`onCheckpointAdded`, `onPrune`, `onTick`, etc.) which
|
|
11
|
+
* schedules a `reconcile(trigger)` on a serial queue. Reconcile walks both session
|
|
12
|
+
* maps, cancels any session whose canonical content has shifted, re-creates it with
|
|
13
|
+
* the same spec but new content, and opens fresh full sessions for any epoch implicated
|
|
14
|
+
* by the trigger.
|
|
15
|
+
*/ export class SessionManager {
|
|
16
|
+
deps;
|
|
17
|
+
log;
|
|
18
|
+
fullSessions;
|
|
19
|
+
partialSessions;
|
|
20
|
+
/**
|
|
21
|
+
* Serialises every reconcile call. The trigger sources (L2BlockStream events, the
|
|
22
|
+
* periodic tick, JSON-RPC `startProof`) run independently, so without this queue two
|
|
23
|
+
* reconciles could interleave on the `await session.cancel(...)` step and orphan a
|
|
24
|
+
* freshly-constructed session.
|
|
25
|
+
*/ reconcileQueue;
|
|
26
|
+
/** Cached L1 constants, populated on first read. */ cachedL1Constants;
|
|
27
|
+
/**
|
|
28
|
+
* Highest epoch for which the periodic tick has successfully created a full session.
|
|
29
|
+
* Monotonic high-water mark: once the tick observes a session for epoch X, it stops
|
|
30
|
+
* trying to open one — even if that session subsequently fails (only a new checkpoint
|
|
31
|
+
* event reopens it). Crucially, the mark only advances when a session actually exists
|
|
32
|
+
* post-open, so transient blockers (atMaxSessionLimit, archiver still indexing) leave
|
|
33
|
+
* the mark in place and the next tick retries.
|
|
34
|
+
*/ lastTickEpoch;
|
|
35
|
+
/** Test-only hooks applied to every session this manager constructs. */ sessionHooks;
|
|
36
|
+
/** Periodic tick that nudges reconcile to pick up newly-complete epochs. Started by `start()`. */ epochTicker;
|
|
37
|
+
constructor(deps){
|
|
38
|
+
this.deps = deps;
|
|
39
|
+
this.fullSessions = new Map();
|
|
40
|
+
this.partialSessions = new Map();
|
|
41
|
+
this.reconcileQueue = new SerialQueue();
|
|
42
|
+
this.log = createLogger('prover-node:session-manager', deps.bindings);
|
|
43
|
+
this.reconcileQueue.start();
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Starts the periodic tick. Separated from the constructor so tests can drive `onTick()`
|
|
47
|
+
* manually without the background ticker interleaving. Idempotent.
|
|
48
|
+
*/ start() {
|
|
49
|
+
if (this.epochTicker) {
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
this.epochTicker = new RunningPromise(()=>this.onTick(), this.log, this.deps.config.tickIntervalMs);
|
|
53
|
+
this.epochTicker.start();
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Installs hooks applied to every session constructed from now on. Used by the e2e
|
|
57
|
+
* harness to interpose around top-tree proving (gate it, override it, observe it)
|
|
58
|
+
* without monkey-patching the orchestrator factory.
|
|
59
|
+
*/ setSessionHooks(hooks) {
|
|
60
|
+
this.sessionHooks = hooks;
|
|
61
|
+
}
|
|
62
|
+
// ---------------- read-only views ----------------
|
|
63
|
+
/** Every live (non-terminal) session. */ allSessions() {
|
|
64
|
+
return [
|
|
65
|
+
...this.fullSessions.values(),
|
|
66
|
+
...this.partialSessions.values()
|
|
67
|
+
];
|
|
68
|
+
}
|
|
69
|
+
/** Returns the full session for `epoch`, if any. */ getFullSession(epoch) {
|
|
70
|
+
return this.fullSessions.get(epoch);
|
|
71
|
+
}
|
|
72
|
+
/** Returns the partial session for `spec`, if any. */ getPartialSession(spec) {
|
|
73
|
+
return this.partialSessions.get(specKey(spec));
|
|
74
|
+
}
|
|
75
|
+
/** Observability summary used by the prover-node API. */ getJobs() {
|
|
76
|
+
return this.allSessions().map((s)=>({
|
|
77
|
+
uuid: s.getId(),
|
|
78
|
+
status: s.getState(),
|
|
79
|
+
epochNumber: s.getEpochNumber()
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
// ---------------- event entry points ----------------
|
|
83
|
+
/** Called by ProverNode after a chain-checkpointed event has been added to the store. */ onCheckpointAdded(epoch) {
|
|
84
|
+
return this.scheduleReconcile({
|
|
85
|
+
kind: 'checkpoint',
|
|
86
|
+
epoch
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
/** Called by ProverNode after a chain-pruned event has flipped store provers to pruned. */ onPrune(affectedEpochs) {
|
|
90
|
+
return this.scheduleReconcile({
|
|
91
|
+
kind: 'prune',
|
|
92
|
+
affectedEpochs
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Called periodically by ProverNode's ticker. Picks up epochs that have become complete
|
|
97
|
+
* by time without a fresh checkpoint event (e.g. the epoch's last slots are empty), and
|
|
98
|
+
* advances to the next epoch once the previous one is proven on L1.
|
|
99
|
+
*/ onTick() {
|
|
100
|
+
return this.scheduleReconcile({
|
|
101
|
+
kind: 'tick'
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
// ---------------- public API ----------------
|
|
105
|
+
/**
|
|
106
|
+
* Schedules a proof attempt for the supplied epoch and returns the job id without waiting for
|
|
107
|
+
* the proof to complete — proving can far outlast an HTTP request, so callers poll `getJobs()`
|
|
108
|
+
* for the outcome. Every session — full or partial — begins at the epoch's first slot; the
|
|
109
|
+
* partial's spec stops at the last canonical slot, while the full's stops at the epoch's last
|
|
110
|
+
* slot. Dedupes against any existing session covering the same range, returning its id.
|
|
111
|
+
*/ async startProof(epoch) {
|
|
112
|
+
const canonical = await this.deps.checkpointStore.listCanonicalForEpoch(epoch);
|
|
113
|
+
if (canonical.length === 0) {
|
|
114
|
+
throw new EmptyEpochError(epoch);
|
|
115
|
+
}
|
|
116
|
+
// Don't re-prove an epoch the L1 proven chain already encompasses — it was already proven
|
|
117
|
+
// (possibly by another prover node), so a fresh proof would be wasted work.
|
|
118
|
+
if (await this.isProvenChainEncompassing(canonical)) {
|
|
119
|
+
throw new EpochAlreadyProvenError(epoch);
|
|
120
|
+
}
|
|
121
|
+
const l1Constants = await this.getL1Constants();
|
|
122
|
+
const [fromSlot] = getSlotRangeForEpoch(epoch, l1Constants);
|
|
123
|
+
const toSlot = canonical[canonical.length - 1].slotNumber;
|
|
124
|
+
const spec = {
|
|
125
|
+
kind: 'partial',
|
|
126
|
+
epochNumber: epoch,
|
|
127
|
+
fromSlot,
|
|
128
|
+
toSlot
|
|
129
|
+
};
|
|
130
|
+
// Reuse a session already covering this exact range rather than scheduling a duplicate.
|
|
131
|
+
const existingFull = this.getFullSession(epoch);
|
|
132
|
+
if (existingFull && !existingFull.isTerminal() && existingFull.getSpec().fromSlot === fromSlot && existingFull.getSpec().toSlot === toSlot) {
|
|
133
|
+
return existingFull.getId();
|
|
134
|
+
}
|
|
135
|
+
const existingPartial = this.getPartialSession(spec);
|
|
136
|
+
if (existingPartial && !existingPartial.isTerminal()) {
|
|
137
|
+
return existingPartial.getId();
|
|
138
|
+
}
|
|
139
|
+
await this.scheduleReconcile({
|
|
140
|
+
kind: 'start-proof',
|
|
141
|
+
spec
|
|
142
|
+
});
|
|
143
|
+
const created = this.getPartialSession(spec);
|
|
144
|
+
if (!created) {
|
|
145
|
+
throw new Error(`Failed to schedule partial proof for epoch ${epoch}`);
|
|
146
|
+
}
|
|
147
|
+
return created.getId();
|
|
148
|
+
}
|
|
149
|
+
/** Stops the tick, drains the reconcile queue, and cancels every live session. */ async stop() {
|
|
150
|
+
await this.epochTicker?.stop();
|
|
151
|
+
await this.reconcileQueue.cancel();
|
|
152
|
+
const sessions = this.allSessions();
|
|
153
|
+
await Promise.allSettled(sessions.map((s)=>s.cancel('prover-node stopping')));
|
|
154
|
+
}
|
|
155
|
+
// ---------------- reconcile ----------------
|
|
156
|
+
scheduleReconcile(trigger) {
|
|
157
|
+
return this.reconcileQueue.put(()=>this.reconcile(trigger));
|
|
158
|
+
}
|
|
159
|
+
async reconcile(trigger) {
|
|
160
|
+
this.log.debug(`Reconciling`, {
|
|
161
|
+
trigger
|
|
162
|
+
});
|
|
163
|
+
this.recreateInvalidSessions();
|
|
164
|
+
const implicatedEpochs = await this.epochsForTrigger(trigger);
|
|
165
|
+
for (const epoch of implicatedEpochs){
|
|
166
|
+
await this.openFullSessionIfReady(epoch);
|
|
167
|
+
}
|
|
168
|
+
// Advance the tick high-water mark only once a session actually exists for the epoch.
|
|
169
|
+
// `openFullSessionIfReady` can early-return without creating one (atMaxSessionLimit,
|
|
170
|
+
// archiver still indexing, etc.); in those cases we want the next tick to try again
|
|
171
|
+
// rather than skip the epoch forever.
|
|
172
|
+
if (trigger.kind === 'tick' && implicatedEpochs.length === 1) {
|
|
173
|
+
const epoch = implicatedEpochs[0];
|
|
174
|
+
if (this.fullSessions.has(epoch)) {
|
|
175
|
+
this.lastTickEpoch = epoch;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
if (trigger.kind === 'start-proof') {
|
|
179
|
+
this.openPartialSession(trigger.spec);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
recreateInvalidSessions() {
|
|
183
|
+
for (const [key, session] of Array.from(this.fullSessions.entries())){
|
|
184
|
+
if (session.isTerminal()) {
|
|
185
|
+
this.fullSessions.delete(key);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const canonical = this.canonicalCheckpointsForSpec(session.getSpec());
|
|
189
|
+
if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
|
|
190
|
+
this.fireAndForgetCancel(session, 'canonical content changed');
|
|
191
|
+
this.fullSessions.delete(key);
|
|
192
|
+
if (canonical.length > 0) {
|
|
193
|
+
const newSession = this.constructSession(session.getSpec(), canonical);
|
|
194
|
+
this.fullSessions.set(key, newSession);
|
|
195
|
+
void this.runSession(newSession);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
for (const [key, session] of Array.from(this.partialSessions.entries())){
|
|
200
|
+
if (session.isTerminal()) {
|
|
201
|
+
this.partialSessions.delete(key);
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
const canonical = this.canonicalCheckpointsForSpec(session.getSpec());
|
|
205
|
+
if (!this.checkpointsMatch(session.getCheckpoints(), canonical)) {
|
|
206
|
+
this.fireAndForgetCancel(session, 'canonical content changed');
|
|
207
|
+
this.partialSessions.delete(key);
|
|
208
|
+
if (canonical.length > 0) {
|
|
209
|
+
const newSession = this.constructSession(session.getSpec(), canonical);
|
|
210
|
+
this.partialSessions.set(key, newSession);
|
|
211
|
+
void this.runSession(newSession);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
async openFullSessionIfReady(epoch) {
|
|
217
|
+
if (this.fullSessions.has(epoch)) {
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
if (this.atMaxSessionLimit()) {
|
|
221
|
+
this.log.debug(`Skipping full-session open for epoch ${epoch}: max pending jobs reached`);
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
224
|
+
if (!await this.deps.l2BlockSource.isEpochComplete(epoch)) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
const l1Constants = await this.getL1Constants();
|
|
228
|
+
const archiverCps = await this.deps.l2BlockSource.getCheckpoints({
|
|
229
|
+
epoch
|
|
230
|
+
});
|
|
231
|
+
if (archiverCps.length === 0) {
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
const [fromSlot, toSlot] = getSlotRangeForEpoch(epoch, l1Constants);
|
|
235
|
+
const canonical = this.deps.checkpointStore.listCanonicalInSlotRange(fromSlot, toSlot);
|
|
236
|
+
if (!this.archiverFullyCovered(archiverCps, canonical)) {
|
|
237
|
+
this.log.debug(`Skipping full-session open for epoch ${epoch}: archiver checkpoints not all in store`, {
|
|
238
|
+
archiverCount: archiverCps.length,
|
|
239
|
+
storeCount: canonical.length
|
|
240
|
+
});
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
const spec = {
|
|
244
|
+
kind: 'full',
|
|
245
|
+
epochNumber: epoch,
|
|
246
|
+
fromSlot,
|
|
247
|
+
toSlot
|
|
248
|
+
};
|
|
249
|
+
const session = this.constructSession(spec, canonical);
|
|
250
|
+
this.fullSessions.set(epoch, session);
|
|
251
|
+
void this.runSession(session);
|
|
252
|
+
}
|
|
253
|
+
openPartialSession(spec) {
|
|
254
|
+
const canonical = this.deps.checkpointStore.listCanonicalInSlotRange(spec.fromSlot, spec.toSlot);
|
|
255
|
+
if (canonical.length === 0) {
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
// Reuse a live partial session for this epoch whose checkpoint set already matches the
|
|
259
|
+
// canonical content — e.g. a repeated `startProof` with no new checkpoints mined since the
|
|
260
|
+
// last one. Reconstructing would re-prove identical content and burn a pending-job slot.
|
|
261
|
+
const existing = Array.from(this.partialSessions.values()).find((s)=>s.getSpec().epochNumber === spec.epochNumber && !s.isTerminal() && this.checkpointsMatch(s.getCheckpoints(), canonical));
|
|
262
|
+
if (existing) {
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
265
|
+
if (this.atMaxSessionLimit()) {
|
|
266
|
+
throw new Error(`Maximum pending proving jobs ${this.deps.config.maxPendingJobs} reached.`);
|
|
267
|
+
}
|
|
268
|
+
const session = this.constructSession(spec, canonical);
|
|
269
|
+
this.partialSessions.set(specKey(spec), session);
|
|
270
|
+
void this.runSession(session);
|
|
271
|
+
}
|
|
272
|
+
// ---------------- session construction ----------------
|
|
273
|
+
constructSession(spec, checkpoints) {
|
|
274
|
+
return this.doConstructSession(spec, checkpoints, this.buildSessionDeps(spec.epochNumber), this.sessionHooks);
|
|
275
|
+
}
|
|
276
|
+
/** Extracted for test override. */ doConstructSession(spec, checkpoints, sessionDeps, hooks) {
|
|
277
|
+
return new EpochSession(spec, checkpoints, {
|
|
278
|
+
...sessionDeps,
|
|
279
|
+
hooks
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
buildSessionDeps(epochNumber) {
|
|
283
|
+
const config = {
|
|
284
|
+
finalizationDelayMs: this.deps.config.finalizationDelayMs
|
|
285
|
+
};
|
|
286
|
+
return {
|
|
287
|
+
proverFactory: this.deps.proverFactory,
|
|
288
|
+
proverId: this.deps.proverId,
|
|
289
|
+
publishingService: this.deps.publishingService,
|
|
290
|
+
metrics: this.deps.metrics,
|
|
291
|
+
dateProvider: this.deps.dateProvider,
|
|
292
|
+
deadline: this.computeDeadline(epochNumber),
|
|
293
|
+
config,
|
|
294
|
+
bindings: this.deps.bindings
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
computeDeadline(epochNumber) {
|
|
298
|
+
if (!this.cachedL1Constants) {
|
|
299
|
+
return undefined;
|
|
300
|
+
}
|
|
301
|
+
const ts = getProofSubmissionDeadlineTimestamp(epochNumber, this.cachedL1Constants);
|
|
302
|
+
return new Date(Number(ts) * 1000);
|
|
303
|
+
}
|
|
304
|
+
async runSession(session) {
|
|
305
|
+
// A reconcile may have cancelled this session before it starts (content-change
|
|
306
|
+
// recreation). Don't proceed — start() would build a TopTreeJob that should never run.
|
|
307
|
+
if (session.isTerminal()) {
|
|
308
|
+
this.log.debug(`Skipping start for ${session.getId()}: already terminal (${session.getState()})`);
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
const state = await session.start();
|
|
312
|
+
this.log.info(`Session ${session.getId()} exited with state ${state}`);
|
|
313
|
+
if (state === 'failed' && this.deps.onSessionFailed) {
|
|
314
|
+
try {
|
|
315
|
+
await this.deps.onSessionFailed(session);
|
|
316
|
+
} catch (err) {
|
|
317
|
+
this.log.error(`Error in onSessionFailed callback for ${session.getSpec().epochNumber}`, err);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
/**
|
|
322
|
+
* Builds the EpochProvingJobData snapshot for failure upload. Includes every checkpoint
|
|
323
|
+
* referenced by the session, regardless of whether sub-tree proving completed —
|
|
324
|
+
* partial state is still useful for post-mortem analysis.
|
|
325
|
+
*/ static buildSessionProvingData(session) {
|
|
326
|
+
const checkpoints = session.getCheckpoints();
|
|
327
|
+
const txs = new Map();
|
|
328
|
+
const l1ToL2Messages = {};
|
|
329
|
+
for (const c of checkpoints){
|
|
330
|
+
for (const [hash, tx] of c.txs){
|
|
331
|
+
txs.set(hash, tx);
|
|
332
|
+
}
|
|
333
|
+
l1ToL2Messages[c.checkpoint.number] = c.l1ToL2Messages;
|
|
334
|
+
}
|
|
335
|
+
return {
|
|
336
|
+
epochNumber: session.getSpec().epochNumber,
|
|
337
|
+
checkpoints: checkpoints.map((c)=>c.checkpoint),
|
|
338
|
+
txs,
|
|
339
|
+
l1ToL2Messages,
|
|
340
|
+
previousBlockHeader: checkpoints[0].previousBlockHeader,
|
|
341
|
+
attestations: []
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
// ---------------- reconcile helpers ----------------
|
|
345
|
+
atMaxSessionLimit() {
|
|
346
|
+
const { maxPendingJobs: max } = this.deps.config;
|
|
347
|
+
if (!max || max <= 0) {
|
|
348
|
+
return false;
|
|
349
|
+
}
|
|
350
|
+
const live = this.allSessions().filter((s)=>!s.isTerminal()).length;
|
|
351
|
+
return live >= max;
|
|
352
|
+
}
|
|
353
|
+
async epochsForTrigger(trigger) {
|
|
354
|
+
switch(trigger.kind){
|
|
355
|
+
case 'checkpoint':
|
|
356
|
+
return [
|
|
357
|
+
trigger.epoch
|
|
358
|
+
];
|
|
359
|
+
case 'prune':
|
|
360
|
+
return trigger.affectedEpochs;
|
|
361
|
+
case 'tick':
|
|
362
|
+
{
|
|
363
|
+
const epoch = await this.nextUnprovenEpoch();
|
|
364
|
+
if (epoch === undefined || this.lastTickEpoch !== undefined && epoch <= this.lastTickEpoch) {
|
|
365
|
+
return [];
|
|
366
|
+
}
|
|
367
|
+
return [
|
|
368
|
+
epoch
|
|
369
|
+
];
|
|
370
|
+
}
|
|
371
|
+
case 'start-proof':
|
|
372
|
+
return [];
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* The next epoch to prove: the epoch containing the first block after the proven tip.
|
|
377
|
+
* Returns undefined when that block has not been mined yet (e.g. nothing new to prove).
|
|
378
|
+
* Subsequent ticks advance only once the chain's proven height moves forward, so epochs
|
|
379
|
+
* are proven in order rather than all at once.
|
|
380
|
+
*/ async nextUnprovenEpoch() {
|
|
381
|
+
const lastProven = await this.deps.l2BlockSource.getBlockNumber({
|
|
382
|
+
tag: 'proven'
|
|
383
|
+
}) ?? BlockNumber.ZERO;
|
|
384
|
+
const firstToProve = BlockNumber(lastProven + 1);
|
|
385
|
+
const header = (await this.deps.l2BlockSource.getBlockData({
|
|
386
|
+
number: firstToProve
|
|
387
|
+
}))?.header;
|
|
388
|
+
if (!header) {
|
|
389
|
+
return undefined;
|
|
390
|
+
}
|
|
391
|
+
return getEpochAtSlot(header.getSlot(), await this.getL1Constants());
|
|
392
|
+
}
|
|
393
|
+
canonicalCheckpointsForSpec(spec) {
|
|
394
|
+
return this.deps.checkpointStore.listCanonicalInSlotRange(spec.fromSlot, spec.toSlot);
|
|
395
|
+
}
|
|
396
|
+
fireAndForgetCancel(session, reason) {
|
|
397
|
+
void session.cancel(reason).catch((err)=>this.log.warn(`Error cancelling session ${session.getId()}`, err));
|
|
398
|
+
}
|
|
399
|
+
checkpointsMatch(a, b) {
|
|
400
|
+
if (a.length !== b.length) {
|
|
401
|
+
return false;
|
|
402
|
+
}
|
|
403
|
+
for(let i = 0; i < a.length; i++){
|
|
404
|
+
if (a[i].id !== b[i].id || a[i].isCancelled()) {
|
|
405
|
+
return false;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
return true;
|
|
409
|
+
}
|
|
410
|
+
archiverFullyCovered(archiverCps, storeCps) {
|
|
411
|
+
if (storeCps.length < archiverCps.length) {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
// Compare by content-addressed id (number, slot, archive root) rather than checkpoint number:
|
|
415
|
+
// a reorg can keep the number while changing the checkpoint's post-state archive root.
|
|
416
|
+
const storeIds = new Set(storeCps.map((p)=>p.id));
|
|
417
|
+
return archiverCps.every((cp)=>storeIds.has(CheckpointProver.idFor(cp.checkpoint)));
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Returns true if the L1 proven tip already covers every canonical checkpoint in the set — i.e.
|
|
421
|
+
* the epoch has already been fully proven, so there is no point starting a new proof for it.
|
|
422
|
+
* Conservatively returns false when nothing is proven yet.
|
|
423
|
+
*/ async isProvenChainEncompassing(canonical) {
|
|
424
|
+
const provenBlock = await this.deps.l2BlockSource.getBlockNumber({
|
|
425
|
+
tag: 'proven'
|
|
426
|
+
});
|
|
427
|
+
if (!provenBlock || provenBlock <= 0) {
|
|
428
|
+
return false;
|
|
429
|
+
}
|
|
430
|
+
const lastCheckpoint = canonical[canonical.length - 1].checkpoint;
|
|
431
|
+
const lastBlock = lastCheckpoint.blocks[lastCheckpoint.blocks.length - 1].number;
|
|
432
|
+
return provenBlock >= lastBlock;
|
|
433
|
+
}
|
|
434
|
+
async getL1Constants() {
|
|
435
|
+
if (!this.cachedL1Constants) {
|
|
436
|
+
this.cachedL1Constants = await this.deps.l2BlockSource.getL1Constants();
|
|
437
|
+
}
|
|
438
|
+
return this.cachedL1Constants;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
class EmptyEpochError extends Error {
|
|
442
|
+
constructor(epochNumber){
|
|
443
|
+
super(`No blocks found for epoch ${epochNumber}`);
|
|
444
|
+
this.name = 'EmptyEpochError';
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
class EpochAlreadyProvenError extends Error {
|
|
448
|
+
constructor(epochNumber){
|
|
449
|
+
super(`Epoch ${epochNumber} is already proven on L1`);
|
|
450
|
+
this.name = 'EpochAlreadyProvenError';
|
|
451
|
+
}
|
|
452
|
+
}
|