@peerbit/program 6.0.34 → 6.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/handler.d.ts +146 -1
- package/dist/src/handler.d.ts.map +1 -1
- package/dist/src/handler.js +1726 -78
- package/dist/src/handler.js.map +1 -1
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/program.d.ts +40 -1
- package/dist/src/program.d.ts.map +1 -1
- package/dist/src/program.js +521 -50
- package/dist/src/program.js.map +1 -1
- package/package.json +14 -14
- package/src/handler.ts +2481 -82
- package/src/index.ts +1 -0
- package/src/program.ts +697 -59
package/src/handler.ts
CHANGED
|
@@ -7,8 +7,128 @@ import type { Address } from "./address.js";
|
|
|
7
7
|
export const logger = loggerFn("peerbit:program:handler");
|
|
8
8
|
|
|
9
9
|
type ProgramMergeStrategy = "replace" | "reject" | "reuse";
|
|
10
|
+
type HandlerLifecycleState = "running" | "stopping" | "stopped" | "failed";
|
|
11
|
+
export type TerminalOperation = "close" | "drop";
|
|
12
|
+
export type TerminalBaseCommit = {
|
|
13
|
+
epoch: number;
|
|
14
|
+
version: number;
|
|
15
|
+
type: TerminalOperation;
|
|
16
|
+
from?: Manageable<any>;
|
|
17
|
+
result: boolean;
|
|
18
|
+
releasedParentReferences: number;
|
|
19
|
+
};
|
|
20
|
+
export const TERMINAL_BASE_CHECKPOINT = Symbol.for(
|
|
21
|
+
"@peerbit/program/terminal-base-checkpoint",
|
|
22
|
+
);
|
|
23
|
+
export const TERMINAL_BASE_COMMIT = Symbol.for(
|
|
24
|
+
"@peerbit/program/terminal-base-commit",
|
|
25
|
+
);
|
|
26
|
+
export const TERMINAL_BASE_RETRY = Symbol.for(
|
|
27
|
+
"@peerbit/program/terminal-base-retry",
|
|
28
|
+
);
|
|
29
|
+
export const TERMINAL_OUTER_CLEANUP_RETAIN = Symbol.for(
|
|
30
|
+
"@peerbit/program/terminal-outer-cleanup-retain",
|
|
31
|
+
);
|
|
32
|
+
export const TERMINAL_OUTER_CLEANUP_RELEASE = Symbol.for(
|
|
33
|
+
"@peerbit/program/terminal-outer-cleanup-release",
|
|
34
|
+
);
|
|
35
|
+
const TERMINAL_OPERATION_NOT_STARTED = Symbol.for(
|
|
36
|
+
"@peerbit/program/terminal-operation-not-started-error",
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* A terminal-operation precondition rejected the call before any lifecycle or
|
|
41
|
+
* resource mutation began. Handler leaves the program retryable instead of
|
|
42
|
+
* retaining this as partially completed cleanup.
|
|
43
|
+
*/
|
|
44
|
+
export class TerminalOperationNotStartedError extends Error {
|
|
45
|
+
readonly [TERMINAL_OPERATION_NOT_STARTED] = true;
|
|
46
|
+
|
|
47
|
+
constructor(message: string) {
|
|
48
|
+
super(message);
|
|
49
|
+
this.name = "TerminalOperationNotStartedError";
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
static [Symbol.hasInstance](instance: unknown): boolean {
|
|
53
|
+
return Boolean(
|
|
54
|
+
instance &&
|
|
55
|
+
typeof instance === "object" &&
|
|
56
|
+
(instance as Record<symbol, unknown>)[
|
|
57
|
+
TERMINAL_OPERATION_NOT_STARTED
|
|
58
|
+
] === true,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
10
62
|
export type ExtractArgs<T> = T extends CanOpen<infer Args> ? Args : never;
|
|
11
63
|
|
|
64
|
+
type FailedTerminalCall = {
|
|
65
|
+
type: TerminalOperation;
|
|
66
|
+
args: any[];
|
|
67
|
+
commit?: TerminalBaseCommit;
|
|
68
|
+
releasedParentReferences?: number;
|
|
69
|
+
};
|
|
70
|
+
|
|
71
|
+
type OuterTerminalCall = {
|
|
72
|
+
type: TerminalOperation;
|
|
73
|
+
from?: Manageable<any>;
|
|
74
|
+
terminal: boolean;
|
|
75
|
+
claimedOwnerReference: boolean;
|
|
76
|
+
invokedWrapper: (...args: any[]) => Promise<boolean>;
|
|
77
|
+
ownerReferencesBeforeInvoke?: number;
|
|
78
|
+
ownerReferencesAfterInvoke?: number;
|
|
79
|
+
promise: Promise<boolean>;
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
type TerminalOperationState<T extends Manageable<any>> = {
|
|
83
|
+
program: T;
|
|
84
|
+
address: Address;
|
|
85
|
+
activeCalls: number;
|
|
86
|
+
pending: Set<Promise<boolean>>;
|
|
87
|
+
failed: boolean;
|
|
88
|
+
failedOperation?: TerminalOperation;
|
|
89
|
+
failedCall?: FailedTerminalCall;
|
|
90
|
+
terminalCompleted: boolean;
|
|
91
|
+
outerTail: Promise<void>;
|
|
92
|
+
outerCalls: Set<OuterTerminalCall>;
|
|
93
|
+
closeWrapper?: (...args: any[]) => Promise<boolean>;
|
|
94
|
+
dropWrapper?: (...args: any[]) => Promise<boolean>;
|
|
95
|
+
closeOperation?: (...args: any[]) => Promise<boolean>;
|
|
96
|
+
dropOperation?: (...args: any[]) => Promise<boolean>;
|
|
97
|
+
synchronouslyInvokingWrapper?: {
|
|
98
|
+
type: TerminalOperation;
|
|
99
|
+
wrapper: (...args: any[]) => Promise<boolean>;
|
|
100
|
+
args: any[];
|
|
101
|
+
};
|
|
102
|
+
cleanupLease?: object;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
type CleanupResidual = {
|
|
106
|
+
program: Manageable<any>;
|
|
107
|
+
failures: FailedTerminalCall[];
|
|
108
|
+
terminalState?: TerminalOperationState<any>;
|
|
109
|
+
cleanupLease?: object;
|
|
110
|
+
activeReservations: number;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
type OpeningReservationGroup = {
|
|
114
|
+
participantReferences: Map<Manageable<any>, number>;
|
|
115
|
+
reservations: Set<OpeningReservation<any>>;
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
type OpeningReservation<T extends Manageable<any>> = {
|
|
119
|
+
promise: Promise<T>;
|
|
120
|
+
phase: "opening" | "rollback";
|
|
121
|
+
observedReservations: Set<OpeningReservation<T>>;
|
|
122
|
+
adoptedParents: Set<Manageable<any>>;
|
|
123
|
+
contributedParticipants: Set<Manageable<any>>;
|
|
124
|
+
wakeWaiters: Set<() => void>;
|
|
125
|
+
group: OpeningReservationGroup;
|
|
126
|
+
rootProgram?: Manageable<any>;
|
|
127
|
+
programsByAddress: Map<string, Set<Manageable<any>>>;
|
|
128
|
+
programs: Set<Manageable<any>>;
|
|
129
|
+
provisionalTerminalStates: Set<TerminalOperationState<T>>;
|
|
130
|
+
};
|
|
131
|
+
|
|
12
132
|
export type EventOptions = {
|
|
13
133
|
onBeforeOpen?: (program: Manageable<any>) => Promise<void> | void;
|
|
14
134
|
onOpen?: (program: Manageable<any>) => Promise<void> | void;
|
|
@@ -36,7 +156,7 @@ export interface Saveable {
|
|
|
36
156
|
format?: string;
|
|
37
157
|
timeout?: number;
|
|
38
158
|
skipOnAddress?: boolean;
|
|
39
|
-
|
|
159
|
+
save?: (address: string) => boolean | Promise<boolean>;
|
|
40
160
|
},
|
|
41
161
|
): Promise<Address>;
|
|
42
162
|
|
|
@@ -57,6 +177,27 @@ export type Manageable<Args> = Closeable &
|
|
|
57
177
|
CanEmit & {
|
|
58
178
|
children: Manageable<any>[];
|
|
59
179
|
parents: (Manageable<any> | undefined)[];
|
|
180
|
+
/**
|
|
181
|
+
* Programs with terminal work that must happen before `Program.end()` can
|
|
182
|
+
* fence parent reuse while that work is in progress.
|
|
183
|
+
*/
|
|
184
|
+
readonly acceptsParentAttachments?: boolean;
|
|
185
|
+
/** Base terminal tail/deletion that must be resumed with this operation. */
|
|
186
|
+
readonly pendingTerminalOperation?: TerminalOperation;
|
|
187
|
+
/** A lifecycle callback must not recursively wait for Handler.stop(). */
|
|
188
|
+
readonly terminalLifecycleCallbackRunning?: boolean;
|
|
189
|
+
[TERMINAL_BASE_CHECKPOINT]?: () => number;
|
|
190
|
+
[TERMINAL_BASE_COMMIT]?: (
|
|
191
|
+
afterVersion: number,
|
|
192
|
+
type: TerminalOperation,
|
|
193
|
+
from?: Manageable<any>,
|
|
194
|
+
) => TerminalBaseCommit | undefined;
|
|
195
|
+
[TERMINAL_BASE_RETRY]?: (
|
|
196
|
+
commit: TerminalBaseCommit,
|
|
197
|
+
operation: () => Promise<boolean>,
|
|
198
|
+
) => Promise<boolean>;
|
|
199
|
+
[TERMINAL_OUTER_CLEANUP_RETAIN]?: () => object;
|
|
200
|
+
[TERMINAL_OUTER_CLEANUP_RELEASE]?: (lease: object) => void;
|
|
60
201
|
} & WithNode;
|
|
61
202
|
|
|
62
203
|
export type ProgramInitializationOptions<Args, T extends Manageable<Args>> = {
|
|
@@ -66,7 +207,14 @@ export type ProgramInitializationOptions<Args, T extends Manageable<Args>> = {
|
|
|
66
207
|
WithParent<T> &
|
|
67
208
|
EventOptions;
|
|
68
209
|
|
|
210
|
+
const assertCanAttachParent = (child: Manageable<any>) => {
|
|
211
|
+
if (child.acceptsParentAttachments === false) {
|
|
212
|
+
throw new Error("Program is terminating and cannot accept another parent");
|
|
213
|
+
}
|
|
214
|
+
};
|
|
215
|
+
|
|
69
216
|
export const addParent = (child: Manageable<any>, parent?: Manageable<any>) => {
|
|
217
|
+
assertCanAttachParent(child);
|
|
70
218
|
if (child.parents && child.parents.includes(parent) && parent == null) {
|
|
71
219
|
return; // prevent root parents to exist multiple times. This will allow use to close a program onces even if it is reused multiple times
|
|
72
220
|
}
|
|
@@ -77,10 +225,40 @@ export const addParent = (child: Manageable<any>, parent?: Manageable<any>) => {
|
|
|
77
225
|
}
|
|
78
226
|
};
|
|
79
227
|
|
|
228
|
+
/**
|
|
229
|
+
* Owns lifecycle state for one client. Handler instances do not share address
|
|
230
|
+
* reservations, so a client must use a single Handler for a program graph.
|
|
231
|
+
*/
|
|
80
232
|
export class Handler<T extends Manageable<any>> {
|
|
81
233
|
items: Map<string, T>;
|
|
82
234
|
private _openQueue: Map<string, PQueue>;
|
|
83
235
|
private _openingPromises: Map<string, Promise<T>>;
|
|
236
|
+
private _openingReservations: Set<OpeningReservation<T>>;
|
|
237
|
+
private _openingReservationsByPromise: WeakMap<
|
|
238
|
+
Promise<T>,
|
|
239
|
+
OpeningReservation<T>
|
|
240
|
+
>;
|
|
241
|
+
private _openingReservationsByAddress: Map<
|
|
242
|
+
string,
|
|
243
|
+
Set<OpeningReservation<T>>
|
|
244
|
+
>;
|
|
245
|
+
private _parentAttachmentReservations: Map<Manageable<any>, number>;
|
|
246
|
+
private _openAdmissions: Set<Promise<void>>;
|
|
247
|
+
private _openLifecycleCallbacks: number;
|
|
248
|
+
private _terminalLifecycleCallbacks: number;
|
|
249
|
+
private _lifecycleMethodInvocations: number;
|
|
250
|
+
private _acceptingOpens: boolean;
|
|
251
|
+
private _stopPromise?: Promise<void>;
|
|
252
|
+
private _lifecycleState: HandlerLifecycleState;
|
|
253
|
+
private _failedInitializations: Map<string, T>;
|
|
254
|
+
private _cleanupResiduals: Map<Manageable<any>, CleanupResidual>;
|
|
255
|
+
private _terminalOperationsByProgram: WeakMap<
|
|
256
|
+
Manageable<any>,
|
|
257
|
+
TerminalOperationState<T>
|
|
258
|
+
>;
|
|
259
|
+
private _terminalOperationsByAddress: Map<Address, TerminalOperationState<T>>;
|
|
260
|
+
private _initializationRollbacks: WeakSet<Manageable<any>>;
|
|
261
|
+
private _replacementClosures: WeakSet<Manageable<any>>;
|
|
84
262
|
|
|
85
263
|
constructor(
|
|
86
264
|
readonly properties: {
|
|
@@ -92,19 +270,1129 @@ export class Handler<T extends Manageable<any>> {
|
|
|
92
270
|
) => Promise<T | undefined>;
|
|
93
271
|
shouldMonitor: (thing: any) => boolean;
|
|
94
272
|
identity: Identity;
|
|
273
|
+
getDependencies?: (program: T) => Manageable<any>[];
|
|
95
274
|
},
|
|
96
275
|
) {
|
|
97
276
|
this._openQueue = new Map();
|
|
98
277
|
this._openingPromises = new Map();
|
|
278
|
+
this._openingReservations = new Set();
|
|
279
|
+
this._openingReservationsByPromise = new WeakMap();
|
|
280
|
+
this._openingReservationsByAddress = new Map();
|
|
281
|
+
this._parentAttachmentReservations = new Map();
|
|
282
|
+
this._openAdmissions = new Set();
|
|
283
|
+
this._openLifecycleCallbacks = 0;
|
|
284
|
+
this._terminalLifecycleCallbacks = 0;
|
|
285
|
+
this._lifecycleMethodInvocations = 0;
|
|
286
|
+
this._acceptingOpens = true;
|
|
287
|
+
this._lifecycleState = "running";
|
|
288
|
+
this._failedInitializations = new Map();
|
|
289
|
+
this._cleanupResiduals = new Map();
|
|
290
|
+
this._terminalOperationsByProgram = new WeakMap();
|
|
291
|
+
this._terminalOperationsByAddress = new Map();
|
|
292
|
+
this._initializationRollbacks = new WeakSet();
|
|
293
|
+
this._replacementClosures = new WeakSet();
|
|
99
294
|
this.items = new Map();
|
|
100
295
|
}
|
|
101
296
|
|
|
102
|
-
|
|
297
|
+
/**
|
|
298
|
+
* Re-open admissions after the owning client has finished starting all of the
|
|
299
|
+
* services programs depend on. This is an internal client lifecycle hook, not
|
|
300
|
+
* a substitute for starting the client itself.
|
|
301
|
+
*
|
|
302
|
+
* This is called by the owning client after its services have started.
|
|
303
|
+
*/
|
|
304
|
+
start(): void {
|
|
305
|
+
if (this._lifecycleState === "running") {
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
this.assertCanStart();
|
|
309
|
+
this._lifecycleState = "running";
|
|
310
|
+
this._acceptingOpens = true;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
/** Validate restart eligibility without opening admissions. */
|
|
314
|
+
assertCanStart(): void {
|
|
315
|
+
if (this._lifecycleState === "running") {
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (
|
|
319
|
+
this._lifecycleState !== "stopped" ||
|
|
320
|
+
this._stopPromise ||
|
|
321
|
+
this.items.size > 0 ||
|
|
322
|
+
this._openingPromises.size > 0 ||
|
|
323
|
+
this._openingReservations.size > 0 ||
|
|
324
|
+
this._openingReservationsByAddress.size > 0 ||
|
|
325
|
+
this._parentAttachmentReservations.size > 0 ||
|
|
326
|
+
this._openAdmissions.size > 0 ||
|
|
327
|
+
this._failedInitializations.size > 0 ||
|
|
328
|
+
this._cleanupResiduals.size > 0 ||
|
|
329
|
+
this._terminalOperationsByAddress.size > 0
|
|
330
|
+
) {
|
|
331
|
+
throw new Error(
|
|
332
|
+
"Program handler cannot start before a successful stop has fully drained",
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* Stop the handler after admitted lifecycle work drains.
|
|
339
|
+
*
|
|
340
|
+
* Lifecycle overrides and callbacks must not await their own handler/client
|
|
341
|
+
* stop. Immediate reentry is rejected. After user code has yielded, portable
|
|
342
|
+
* JavaScript runtimes provide no reliable async-call provenance with which to
|
|
343
|
+
* distinguish self-reentry from an unrelated external stop; schedule stop from
|
|
344
|
+
* the lifecycle owner instead. A conservatively rejected external stop remains
|
|
345
|
+
* retryable after the active callback finishes.
|
|
346
|
+
*/
|
|
347
|
+
stop(): Promise<void> {
|
|
348
|
+
if (
|
|
349
|
+
this._openLifecycleCallbacks > 0 ||
|
|
350
|
+
this._terminalLifecycleCallbacks > 0 ||
|
|
351
|
+
this._lifecycleMethodInvocations > 0 ||
|
|
352
|
+
[...this.items.values()].some(
|
|
353
|
+
(program) => program.terminalLifecycleCallbackRunning === true,
|
|
354
|
+
)
|
|
355
|
+
) {
|
|
356
|
+
return Promise.reject(
|
|
357
|
+
new TerminalOperationNotStartedError(
|
|
358
|
+
"Program lifecycle callbacks cannot wait for their own handler to stop",
|
|
359
|
+
),
|
|
360
|
+
);
|
|
361
|
+
}
|
|
362
|
+
if (this._stopPromise) {
|
|
363
|
+
return this._stopPromise;
|
|
364
|
+
}
|
|
365
|
+
if (this._lifecycleState === "stopped") {
|
|
366
|
+
return Promise.resolve();
|
|
367
|
+
}
|
|
368
|
+
// Close admissions synchronously. Every open that passed this fence has a
|
|
369
|
+
// completion promise in `_openAdmissions`, so stop has a stable set to drain.
|
|
370
|
+
this._acceptingOpens = false;
|
|
371
|
+
this._lifecycleState = "stopping";
|
|
372
|
+
this._stopPromise = this.stopOnce().then(
|
|
373
|
+
() => {
|
|
374
|
+
// The client still has storage, index, and transport teardown to perform.
|
|
375
|
+
// Only its completed start() transition may re-open admissions.
|
|
376
|
+
this._lifecycleState = "stopped";
|
|
377
|
+
this._stopPromise = undefined;
|
|
378
|
+
},
|
|
379
|
+
(error) => {
|
|
380
|
+
// Keep admissions closed after a partial stop. Clearing only the promise
|
|
381
|
+
// makes stop() retryable without exposing the live residual items to opens.
|
|
382
|
+
this._lifecycleState = "failed";
|
|
383
|
+
this._stopPromise = undefined;
|
|
384
|
+
throw error;
|
|
385
|
+
},
|
|
386
|
+
);
|
|
387
|
+
return this._stopPromise;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Async methods execute synchronously until their first await. Fence that
|
|
392
|
+
* invocation window so a lifecycle override cannot make stop wait on the very
|
|
393
|
+
* operation that is awaiting stop. Awaited user callbacks have their own wider
|
|
394
|
+
* callback fences below.
|
|
395
|
+
*/
|
|
396
|
+
private invokeLifecycleMethod<R>(
|
|
397
|
+
operation: () => R | PromiseLike<R>,
|
|
398
|
+
): Promise<R> {
|
|
399
|
+
this._lifecycleMethodInvocations += 1;
|
|
400
|
+
try {
|
|
401
|
+
return Promise.resolve(operation());
|
|
402
|
+
} finally {
|
|
403
|
+
this._lifecycleMethodInvocations -= 1;
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Program.end() can only observe the base implementation's lifetime. A
|
|
409
|
+
* subclass is still free to do asynchronous cleanup after `super.close()` or
|
|
410
|
+
* `super.drop()` has returned. Wrap the public, dynamically-dispatched method
|
|
411
|
+
* on every managed instance so Handler state covers that full outer promise.
|
|
412
|
+
*/
|
|
413
|
+
private monitorTerminalOperations(
|
|
414
|
+
address: Address,
|
|
415
|
+
program: T,
|
|
416
|
+
): { state: TerminalOperationState<T>; newLifecycle: boolean } {
|
|
417
|
+
let state = this._terminalOperationsByProgram.get(program);
|
|
418
|
+
const newLifecycle =
|
|
419
|
+
!state || this._terminalOperationsByAddress.get(address) !== state;
|
|
420
|
+
if (!state) {
|
|
421
|
+
state = {
|
|
422
|
+
program,
|
|
423
|
+
address,
|
|
424
|
+
activeCalls: 0,
|
|
425
|
+
pending: new Set(),
|
|
426
|
+
failed: false,
|
|
427
|
+
terminalCompleted: false,
|
|
428
|
+
outerTail: Promise.resolve(),
|
|
429
|
+
outerCalls: new Set(),
|
|
430
|
+
};
|
|
431
|
+
this._terminalOperationsByProgram.set(program, state);
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
state.address = address;
|
|
435
|
+
if (newLifecycle) {
|
|
436
|
+
state.failed = false;
|
|
437
|
+
state.failedOperation = undefined;
|
|
438
|
+
state.failedCall = undefined;
|
|
439
|
+
state.terminalCompleted = false;
|
|
440
|
+
}
|
|
441
|
+
this._terminalOperationsByAddress.set(address, state);
|
|
442
|
+
this.refreshTerminalOperationWrappers(state);
|
|
443
|
+
return { state, newLifecycle };
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
private refreshTerminalOperationWrappers(
|
|
447
|
+
state: TerminalOperationState<T>,
|
|
448
|
+
): void {
|
|
449
|
+
const program = state.program;
|
|
450
|
+
if (program.close !== state.closeWrapper) {
|
|
451
|
+
const close = program.close;
|
|
452
|
+
state.closeOperation = close;
|
|
453
|
+
const closeWrapper = (...args: any[]) =>
|
|
454
|
+
this.runTerminalOperation(state, "close", close, args, closeWrapper);
|
|
455
|
+
state.closeWrapper = closeWrapper;
|
|
456
|
+
program.close = closeWrapper;
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
const droppable = program as T & {
|
|
460
|
+
drop?: (...args: any[]) => Promise<boolean>;
|
|
461
|
+
};
|
|
462
|
+
if (
|
|
463
|
+
typeof droppable.drop === "function" &&
|
|
464
|
+
droppable.drop !== state.dropWrapper
|
|
465
|
+
) {
|
|
466
|
+
const drop = droppable.drop;
|
|
467
|
+
state.dropOperation = drop;
|
|
468
|
+
const dropWrapper = (...args: any[]) =>
|
|
469
|
+
this.runTerminalOperation(state, "drop", drop, args, dropWrapper);
|
|
470
|
+
state.dropWrapper = dropWrapper;
|
|
471
|
+
droppable.drop = dropWrapper;
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
private runTerminalOperation(
|
|
476
|
+
state: TerminalOperationState<T>,
|
|
477
|
+
type: TerminalOperation,
|
|
478
|
+
operation: (...args: any[]) => Promise<boolean>,
|
|
479
|
+
args: any[],
|
|
480
|
+
invokedWrapper: (...args: any[]) => Promise<boolean>,
|
|
481
|
+
): Promise<boolean> {
|
|
482
|
+
const from = args[0] as Manageable<any> | undefined;
|
|
483
|
+
const synchronousInvocation = state.synchronouslyInvokingWrapper;
|
|
484
|
+
if (synchronousInvocation) {
|
|
485
|
+
if (type !== synchronousInvocation.type) {
|
|
486
|
+
return Promise.reject(
|
|
487
|
+
new TerminalOperationNotStartedError(
|
|
488
|
+
"Program terminal methods cannot synchronously invoke another terminal operation",
|
|
489
|
+
),
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
if (!this.sameTerminalArgs(args, synchronousInvocation.args)) {
|
|
493
|
+
return Promise.reject(
|
|
494
|
+
new TerminalOperationNotStartedError(
|
|
495
|
+
"A captured terminal wrapper cannot change the active operation owner",
|
|
496
|
+
),
|
|
497
|
+
);
|
|
498
|
+
}
|
|
499
|
+
if (invokedWrapper !== synchronousInvocation.wrapper) {
|
|
500
|
+
// A post-open replacement may delegate to the Handler wrapper it
|
|
501
|
+
// captured before being re-wrapped. Peel that stale wrapper back to its
|
|
502
|
+
// raw operation; the current outer wrapper still monitors the full promise.
|
|
503
|
+
try {
|
|
504
|
+
return Promise.resolve(operation.apply(state.program, args));
|
|
505
|
+
} catch (error) {
|
|
506
|
+
return Promise.reject(error);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
return Promise.reject(
|
|
510
|
+
new TerminalOperationNotStartedError(
|
|
511
|
+
"Program terminal methods cannot wait for their own active operation",
|
|
512
|
+
),
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
if (
|
|
516
|
+
state.program.terminalLifecycleCallbackRunning ||
|
|
517
|
+
(this._terminalLifecycleCallbacks > 0 && state.activeCalls > 0)
|
|
518
|
+
) {
|
|
519
|
+
return Promise.reject(
|
|
520
|
+
new TerminalOperationNotStartedError(
|
|
521
|
+
"Program lifecycle callbacks cannot wait for their own terminal operation",
|
|
522
|
+
),
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
const parentIndex =
|
|
526
|
+
state.program.parents?.findIndex((parent) => parent === from) ?? -1;
|
|
527
|
+
const terminal =
|
|
528
|
+
state.program.closed ||
|
|
529
|
+
parentIndex === -1 ||
|
|
530
|
+
(state.program.parents?.length ?? 0) === 1;
|
|
531
|
+
const currentWrapper =
|
|
532
|
+
type === "close" ? state.closeWrapper : state.dropWrapper;
|
|
533
|
+
if (
|
|
534
|
+
currentWrapper &&
|
|
535
|
+
invokedWrapper !== currentWrapper &&
|
|
536
|
+
[...state.outerCalls].some(
|
|
537
|
+
(call) => call.type === type && call.invokedWrapper === currentWrapper,
|
|
538
|
+
)
|
|
539
|
+
) {
|
|
540
|
+
// Once replacement code has yielded there is no portable async-call
|
|
541
|
+
// context that can distinguish its captured stale wrapper from an
|
|
542
|
+
// unrelated external stale call. Never return the active outer promise to
|
|
543
|
+
// code that may itself be awaiting this call: reject before base progress
|
|
544
|
+
// instead of forming a self-cycle that wedges Handler.stop().
|
|
545
|
+
return Promise.reject(
|
|
546
|
+
new TerminalOperationNotStartedError(
|
|
547
|
+
"A stale terminal wrapper cannot join its active replacement after yielding",
|
|
548
|
+
),
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
const matching = [...state.outerCalls].find(
|
|
552
|
+
(call) =>
|
|
553
|
+
call.terminal &&
|
|
554
|
+
call.from === from &&
|
|
555
|
+
(call.type === type || (type === "close" && call.type === "drop")),
|
|
556
|
+
);
|
|
557
|
+
if (matching) {
|
|
558
|
+
return matching.promise;
|
|
559
|
+
}
|
|
560
|
+
try {
|
|
561
|
+
this.assertTerminalGraphCanStart(state.program);
|
|
562
|
+
} catch (error) {
|
|
563
|
+
return Promise.reject(error);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
// The outer wrapper serializes subclass methods before Program sees them. A
|
|
567
|
+
// call admitted while another owner is still queued can therefore look
|
|
568
|
+
// non-terminal now even though it will be the final release when it runs.
|
|
569
|
+
// Count already-admitted compatible calls against the owner's current
|
|
570
|
+
// references so a duplicate of that future terminal call shares its exact
|
|
571
|
+
// promise. Keep one distinct call per real duplicate ownership reference.
|
|
572
|
+
const compatibleCalls = [...state.outerCalls].filter(
|
|
573
|
+
(call) =>
|
|
574
|
+
call.from === from &&
|
|
575
|
+
(call.type === type || (type === "close" && call.type === "drop")),
|
|
576
|
+
);
|
|
577
|
+
const ownerReferences =
|
|
578
|
+
state.program.parents?.filter((parent) => parent === from).length ?? 0;
|
|
579
|
+
const reservedOwnerReferences = compatibleCalls.filter(
|
|
580
|
+
(call) => call.claimedOwnerReference,
|
|
581
|
+
).length;
|
|
582
|
+
const synchronouslyReleasedOwnerReferences = compatibleCalls.reduce(
|
|
583
|
+
(released, call) => {
|
|
584
|
+
const before = call.ownerReferencesBeforeInvoke;
|
|
585
|
+
const after = call.ownerReferencesAfterInvoke;
|
|
586
|
+
return (
|
|
587
|
+
released +
|
|
588
|
+
(before == null || after == null
|
|
589
|
+
? 0
|
|
590
|
+
: Math.min(1, Math.max(0, before - after)))
|
|
591
|
+
);
|
|
592
|
+
},
|
|
593
|
+
0,
|
|
594
|
+
);
|
|
595
|
+
const unclaimedOwnerReferences =
|
|
596
|
+
ownerReferences -
|
|
597
|
+
(reservedOwnerReferences - synchronouslyReleasedOwnerReferences);
|
|
598
|
+
if (compatibleCalls.length > 0 && unclaimedOwnerReferences <= 0) {
|
|
599
|
+
return compatibleCalls.at(-1)!.promise;
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
let resolve!: (value: boolean) => void;
|
|
603
|
+
let reject!: (reason?: unknown) => void;
|
|
604
|
+
const tracked = new Promise<boolean>((promiseResolve, promiseReject) => {
|
|
605
|
+
resolve = promiseResolve;
|
|
606
|
+
reject = promiseReject;
|
|
607
|
+
});
|
|
608
|
+
const outerCall: OuterTerminalCall = {
|
|
609
|
+
type,
|
|
610
|
+
from,
|
|
611
|
+
terminal,
|
|
612
|
+
claimedOwnerReference: ownerReferences > 0,
|
|
613
|
+
invokedWrapper,
|
|
614
|
+
promise: tracked,
|
|
615
|
+
};
|
|
616
|
+
const hasPredecessor = state.outerCalls.size > 0;
|
|
617
|
+
state.outerCalls.add(outerCall);
|
|
618
|
+
state.activeCalls += 1;
|
|
619
|
+
state.terminalCompleted = false;
|
|
620
|
+
this._terminalOperationsByAddress.set(state.address, state);
|
|
621
|
+
state.pending.add(tracked);
|
|
622
|
+
|
|
623
|
+
const predecessor = state.outerTail;
|
|
624
|
+
const execute = async (): Promise<boolean> => {
|
|
625
|
+
const recovering = state.failed;
|
|
626
|
+
const failedCall = state.failedCall;
|
|
627
|
+
if (
|
|
628
|
+
recovering &&
|
|
629
|
+
(failedCall?.type !== type ||
|
|
630
|
+
!this.sameTerminalArgs(failedCall.args, args))
|
|
631
|
+
) {
|
|
632
|
+
throw new Error(
|
|
633
|
+
`Program at ${state.address} has failed terminal cleanup that must be retried first`,
|
|
634
|
+
);
|
|
635
|
+
}
|
|
636
|
+
|
|
637
|
+
const startedClosed = state.program.closed;
|
|
638
|
+
const startedAcceptingParents = state.program.acceptsParentAttachments;
|
|
639
|
+
const startedParents = [...(state.program.parents ?? [])];
|
|
640
|
+
const checkpoint = this.terminalCheckpoint(state.program);
|
|
641
|
+
const existingCleanupLease = state.cleanupLease;
|
|
642
|
+
if (!state.cleanupLease) {
|
|
643
|
+
state.cleanupLease = state.program[TERMINAL_OUTER_CLEANUP_RETAIN]?.();
|
|
644
|
+
}
|
|
645
|
+
const releaseProvisionalCleanupLease = () => {
|
|
646
|
+
if (!existingCleanupLease && state.cleanupLease) {
|
|
647
|
+
state.program[TERMINAL_OUTER_CLEANUP_RELEASE]?.(state.cleanupLease);
|
|
648
|
+
state.cleanupLease = undefined;
|
|
649
|
+
}
|
|
650
|
+
};
|
|
651
|
+
try {
|
|
652
|
+
const invoke = () => {
|
|
653
|
+
outerCall.ownerReferencesBeforeInvoke =
|
|
654
|
+
state.program.parents?.filter((parent) => parent === from).length ??
|
|
655
|
+
0;
|
|
656
|
+
let operationResult: Promise<boolean>;
|
|
657
|
+
const previousSynchronouslyInvokingWrapper =
|
|
658
|
+
state.synchronouslyInvokingWrapper;
|
|
659
|
+
state.synchronouslyInvokingWrapper = {
|
|
660
|
+
type,
|
|
661
|
+
wrapper: invokedWrapper,
|
|
662
|
+
args,
|
|
663
|
+
};
|
|
664
|
+
try {
|
|
665
|
+
operationResult = this.invokeLifecycleMethod(() =>
|
|
666
|
+
operation.apply(state.program, args),
|
|
667
|
+
);
|
|
668
|
+
} finally {
|
|
669
|
+
state.synchronouslyInvokingWrapper =
|
|
670
|
+
previousSynchronouslyInvokingWrapper;
|
|
671
|
+
outerCall.ownerReferencesAfterInvoke =
|
|
672
|
+
state.program.parents?.filter((parent) => parent === from)
|
|
673
|
+
.length ?? 0;
|
|
674
|
+
}
|
|
675
|
+
return operationResult!;
|
|
676
|
+
};
|
|
677
|
+
const result =
|
|
678
|
+
recovering && failedCall?.commit
|
|
679
|
+
? await this.retryCommittedTerminalCall(
|
|
680
|
+
state.program,
|
|
681
|
+
failedCall.commit,
|
|
682
|
+
invoke,
|
|
683
|
+
)
|
|
684
|
+
: await invoke();
|
|
685
|
+
if (recovering) {
|
|
686
|
+
const recoveryOwner = failedCall?.args[0] as
|
|
687
|
+
| Manageable<any>
|
|
688
|
+
| undefined;
|
|
689
|
+
const releasedDuringRetry = Math.max(
|
|
690
|
+
0,
|
|
691
|
+
startedParents.filter((parent) => parent === recoveryOwner).length -
|
|
692
|
+
(state.program.parents?.filter(
|
|
693
|
+
(parent) => parent === recoveryOwner,
|
|
694
|
+
).length ?? 0),
|
|
695
|
+
);
|
|
696
|
+
this.removeReleasedParentChildReferences(
|
|
697
|
+
state.program,
|
|
698
|
+
recoveryOwner,
|
|
699
|
+
Math.max(
|
|
700
|
+
failedCall?.commit?.releasedParentReferences ??
|
|
701
|
+
failedCall?.releasedParentReferences ??
|
|
702
|
+
0,
|
|
703
|
+
releasedDuringRetry,
|
|
704
|
+
),
|
|
705
|
+
);
|
|
706
|
+
state.failed = false;
|
|
707
|
+
state.failedOperation = undefined;
|
|
708
|
+
state.failedCall = undefined;
|
|
709
|
+
this.releaseRetainedTerminalIdentity(state);
|
|
710
|
+
} else {
|
|
711
|
+
const commit = this.terminalCommit(
|
|
712
|
+
state.program,
|
|
713
|
+
checkpoint,
|
|
714
|
+
type,
|
|
715
|
+
from,
|
|
716
|
+
);
|
|
717
|
+
this.removeReleasedParentChildReferences(
|
|
718
|
+
state.program,
|
|
719
|
+
from,
|
|
720
|
+
commit?.releasedParentReferences ?? 0,
|
|
721
|
+
);
|
|
722
|
+
}
|
|
723
|
+
if (result && state.program.closed && !state.failed) {
|
|
724
|
+
state.terminalCompleted = true;
|
|
725
|
+
}
|
|
726
|
+
if (state.cleanupLease) {
|
|
727
|
+
state.program[TERMINAL_OUTER_CLEANUP_RELEASE]?.(state.cleanupLease);
|
|
728
|
+
state.cleanupLease = undefined;
|
|
729
|
+
}
|
|
730
|
+
return result;
|
|
731
|
+
} catch (error) {
|
|
732
|
+
const observedCommit =
|
|
733
|
+
failedCall?.commit ||
|
|
734
|
+
this.terminalCommit(state.program, checkpoint, type, from);
|
|
735
|
+
if (
|
|
736
|
+
!recovering &&
|
|
737
|
+
error instanceof TerminalOperationNotStartedError &&
|
|
738
|
+
!observedCommit
|
|
739
|
+
) {
|
|
740
|
+
// Exclude this call's provisional lease while validating that the
|
|
741
|
+
// explicit precondition error made no lifecycle mutation.
|
|
742
|
+
releaseProvisionalCleanupLease();
|
|
743
|
+
if (
|
|
744
|
+
state.program.closed === startedClosed &&
|
|
745
|
+
state.program.acceptsParentAttachments ===
|
|
746
|
+
startedAcceptingParents &&
|
|
747
|
+
this.sameParents(state.program.parents ?? [], startedParents)
|
|
748
|
+
) {
|
|
749
|
+
state.terminalCompleted = startedClosed;
|
|
750
|
+
throw error;
|
|
751
|
+
}
|
|
752
|
+
state.cleanupLease = state.program[TERMINAL_OUTER_CLEANUP_RETAIN]?.();
|
|
753
|
+
}
|
|
754
|
+
if (startedClosed && !recovering) {
|
|
755
|
+
// A fresh drop of an already-cleanly-closed program, or a drop queued
|
|
756
|
+
// behind a winning close, is an API rejection rather than failed cleanup.
|
|
757
|
+
state.terminalCompleted = true;
|
|
758
|
+
releaseProvisionalCleanupLease();
|
|
759
|
+
throw error;
|
|
760
|
+
}
|
|
761
|
+
const commit = observedCommit;
|
|
762
|
+
const baseProgressed =
|
|
763
|
+
commit != null ||
|
|
764
|
+
state.program.closed !== startedClosed ||
|
|
765
|
+
!this.sameParents(state.program.parents ?? [], startedParents);
|
|
766
|
+
const recoveryType: TerminalOperation =
|
|
767
|
+
type === "drop" && !baseProgressed
|
|
768
|
+
? "drop"
|
|
769
|
+
: commit || state.program.closed
|
|
770
|
+
? type
|
|
771
|
+
: "close";
|
|
772
|
+
const releasedParentReferences = Math.max(
|
|
773
|
+
failedCall?.releasedParentReferences ?? 0,
|
|
774
|
+
startedParents.filter((parent) => parent === from).length -
|
|
775
|
+
(state.program.parents?.filter((parent) => parent === from)
|
|
776
|
+
.length ?? 0),
|
|
777
|
+
);
|
|
778
|
+
state.failed = true;
|
|
779
|
+
state.failedOperation = recoveryType;
|
|
780
|
+
state.failedCall = {
|
|
781
|
+
type: recoveryType,
|
|
782
|
+
args: [...args],
|
|
783
|
+
commit: recoveryType === type ? commit : undefined,
|
|
784
|
+
releasedParentReferences,
|
|
785
|
+
};
|
|
786
|
+
state.terminalCompleted = false;
|
|
787
|
+
const monitored = this.items.get(state.address);
|
|
788
|
+
if (!monitored || monitored === state.program) {
|
|
789
|
+
this.items.set(state.address, state.program);
|
|
790
|
+
} else {
|
|
791
|
+
// `items` is address-keyed, but embedded opening graphs may contain a
|
|
792
|
+
// distinct instance with the same address as a monitored root. Never
|
|
793
|
+
// orphan that root by replacing it with this failed cleanup identity.
|
|
794
|
+
const residual = this.reserveCleanupResidual(state.program);
|
|
795
|
+
residual.terminalState = state;
|
|
796
|
+
}
|
|
797
|
+
this._terminalOperationsByAddress.set(state.address, state);
|
|
798
|
+
throw error;
|
|
799
|
+
}
|
|
800
|
+
};
|
|
801
|
+
|
|
802
|
+
// Invoke the first outer method immediately. Program.end() and subclasses
|
|
803
|
+
// establish their mutation/attachment fences synchronously before their
|
|
804
|
+
// first await; deferring through Promise.then would admit same-turn writes
|
|
805
|
+
// after close/drop had already been requested. Only true followers use the
|
|
806
|
+
// serialized promise tail.
|
|
807
|
+
const execution = hasPredecessor
|
|
808
|
+
? predecessor.then(execute, execute)
|
|
809
|
+
: execute();
|
|
810
|
+
state.outerTail = tracked.then(
|
|
811
|
+
(): void => undefined,
|
|
812
|
+
(): void => undefined,
|
|
813
|
+
);
|
|
814
|
+
void execution.then(resolve, reject);
|
|
815
|
+
void tracked.then(
|
|
816
|
+
() => this.finishOuterTerminalCall(state, outerCall, tracked),
|
|
817
|
+
() => this.finishOuterTerminalCall(state, outerCall, tracked),
|
|
818
|
+
);
|
|
819
|
+
logger.trace(
|
|
820
|
+
`Tracking outer ${type} operation for program at ${state.address}`,
|
|
821
|
+
);
|
|
822
|
+
return tracked;
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
private finishOuterTerminalCall(
|
|
826
|
+
state: TerminalOperationState<T>,
|
|
827
|
+
call: OuterTerminalCall,
|
|
828
|
+
promise: Promise<boolean>,
|
|
829
|
+
): void {
|
|
830
|
+
state.activeCalls -= 1;
|
|
831
|
+
state.outerCalls.delete(call);
|
|
832
|
+
state.pending.delete(promise);
|
|
833
|
+
this.finishTerminalOperation(state);
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
private sameTerminalArgs(left: any[], right: any[]): boolean {
|
|
837
|
+
return (
|
|
838
|
+
left.length === right.length &&
|
|
839
|
+
left.every((value, index) => value === right[index])
|
|
840
|
+
);
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
private sameParents(
|
|
844
|
+
left: (Manageable<any> | undefined)[],
|
|
845
|
+
right: (Manageable<any> | undefined)[],
|
|
846
|
+
): boolean {
|
|
847
|
+
return (
|
|
848
|
+
left.length === right.length &&
|
|
849
|
+
left.every((parent, index) => parent === right[index])
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
private terminalCheckpoint(program: Manageable<any>): number {
|
|
854
|
+
return program[TERMINAL_BASE_CHECKPOINT]?.() ?? 0;
|
|
855
|
+
}
|
|
856
|
+
|
|
857
|
+
private terminalCommit(
|
|
858
|
+
program: Manageable<any>,
|
|
859
|
+
checkpoint: number,
|
|
860
|
+
type: TerminalOperation,
|
|
861
|
+
from?: Manageable<any>,
|
|
862
|
+
): TerminalBaseCommit | undefined {
|
|
863
|
+
return program[TERMINAL_BASE_COMMIT]?.(checkpoint, type, from);
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
private retryCommittedTerminalCall(
|
|
867
|
+
program: Manageable<any>,
|
|
868
|
+
commit: TerminalBaseCommit,
|
|
869
|
+
operation: () => Promise<boolean>,
|
|
870
|
+
): Promise<boolean> {
|
|
871
|
+
return program[TERMINAL_BASE_RETRY]
|
|
872
|
+
? program[TERMINAL_BASE_RETRY](commit, operation)
|
|
873
|
+
: operation();
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
private addCleanupResidual(
|
|
877
|
+
program: Manageable<any>,
|
|
878
|
+
failure: FailedTerminalCall,
|
|
879
|
+
cleanupLease?: object,
|
|
880
|
+
): void {
|
|
881
|
+
const residual = this.reserveCleanupResidual(program, cleanupLease);
|
|
882
|
+
if (
|
|
883
|
+
cleanupLease &&
|
|
884
|
+
residual.cleanupLease &&
|
|
885
|
+
cleanupLease !== residual.cleanupLease
|
|
886
|
+
) {
|
|
887
|
+
program[TERMINAL_OUTER_CLEANUP_RELEASE]?.(cleanupLease);
|
|
888
|
+
}
|
|
889
|
+
residual.failures.push(failure);
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
private reserveCleanupResidual(
|
|
893
|
+
program: Manageable<any>,
|
|
894
|
+
cleanupLease?: object,
|
|
895
|
+
): CleanupResidual {
|
|
896
|
+
const existing = this._cleanupResiduals.get(program);
|
|
897
|
+
if (existing) {
|
|
898
|
+
return existing;
|
|
899
|
+
}
|
|
900
|
+
const retainedLease =
|
|
901
|
+
cleanupLease ?? program[TERMINAL_OUTER_CLEANUP_RETAIN]?.();
|
|
902
|
+
const residual: CleanupResidual = {
|
|
903
|
+
program,
|
|
904
|
+
failures: [],
|
|
905
|
+
cleanupLease: retainedLease,
|
|
906
|
+
activeReservations: 0,
|
|
907
|
+
};
|
|
908
|
+
this._cleanupResiduals.set(program, residual);
|
|
909
|
+
return residual;
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
private acquireCleanupReservation(program: Manageable<any>): CleanupResidual {
|
|
913
|
+
const residual = this.reserveCleanupResidual(program);
|
|
914
|
+
residual.activeReservations += 1;
|
|
915
|
+
return residual;
|
|
916
|
+
}
|
|
917
|
+
|
|
918
|
+
private releaseCleanupReservation(residual: CleanupResidual): void {
|
|
919
|
+
residual.activeReservations -= 1;
|
|
920
|
+
if (residual.activeReservations < 0) {
|
|
921
|
+
throw new Error("Cleanup residual reservation underflow");
|
|
922
|
+
}
|
|
923
|
+
if (
|
|
924
|
+
residual.activeReservations === 0 &&
|
|
925
|
+
residual.failures.length === 0 &&
|
|
926
|
+
residual.terminalState == null
|
|
927
|
+
) {
|
|
928
|
+
this.releaseCleanupResidual(residual);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
|
|
932
|
+
private releaseCleanupResidual(residual: CleanupResidual): void {
|
|
933
|
+
if (residual.activeReservations > 0) {
|
|
934
|
+
throw new Error(
|
|
935
|
+
"Cleanup residual cannot be released while rollback cleanup is active",
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
if (residual.terminalState != null) {
|
|
939
|
+
throw new Error(
|
|
940
|
+
"Cleanup residual cannot be released while terminal identity is retained",
|
|
941
|
+
);
|
|
942
|
+
}
|
|
943
|
+
if (residual.cleanupLease) {
|
|
944
|
+
residual.program[TERMINAL_OUTER_CLEANUP_RELEASE]?.(residual.cleanupLease);
|
|
945
|
+
residual.cleanupLease = undefined;
|
|
946
|
+
}
|
|
947
|
+
this._cleanupResiduals.delete(residual.program);
|
|
948
|
+
}
|
|
949
|
+
|
|
950
|
+
private releaseRetainedTerminalIdentity(
|
|
951
|
+
state: TerminalOperationState<T>,
|
|
952
|
+
): void {
|
|
953
|
+
const residual = this._cleanupResiduals.get(state.program);
|
|
954
|
+
if (residual?.terminalState !== state) {
|
|
955
|
+
return;
|
|
956
|
+
}
|
|
957
|
+
residual.terminalState = undefined;
|
|
958
|
+
if (residual.activeReservations === 0 && residual.failures.length === 0) {
|
|
959
|
+
this.releaseCleanupResidual(residual);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
private async retryCleanupResidual(
|
|
964
|
+
residual: CleanupResidual,
|
|
965
|
+
failure: FailedTerminalCall,
|
|
966
|
+
): Promise<void> {
|
|
967
|
+
const operation = (
|
|
968
|
+
residual.program as Manageable<any> & {
|
|
969
|
+
drop?: (...args: any[]) => Promise<boolean>;
|
|
970
|
+
}
|
|
971
|
+
)[failure.type];
|
|
972
|
+
if (!operation) {
|
|
973
|
+
throw new Error(
|
|
974
|
+
`Program at ${residual.program.address} cannot retry its failed ${failure.type} cleanup`,
|
|
975
|
+
);
|
|
976
|
+
}
|
|
977
|
+
const checkpoint = this.terminalCheckpoint(residual.program);
|
|
978
|
+
const invoke = () =>
|
|
979
|
+
this.invokeLifecycleMethod(() =>
|
|
980
|
+
operation.apply(residual.program, failure.args),
|
|
981
|
+
);
|
|
982
|
+
try {
|
|
983
|
+
if (failure.commit) {
|
|
984
|
+
await this.retryCommittedTerminalCall(
|
|
985
|
+
residual.program,
|
|
986
|
+
failure.commit,
|
|
987
|
+
invoke,
|
|
988
|
+
);
|
|
989
|
+
} else {
|
|
990
|
+
await invoke();
|
|
991
|
+
}
|
|
992
|
+
} catch (error) {
|
|
993
|
+
failure.commit ??= this.terminalCommit(
|
|
994
|
+
residual.program,
|
|
995
|
+
checkpoint,
|
|
996
|
+
failure.type,
|
|
997
|
+
failure.args[0],
|
|
998
|
+
);
|
|
999
|
+
throw error;
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
private finishTerminalOperation(state: TerminalOperationState<T>): void {
|
|
1004
|
+
if (
|
|
1005
|
+
state.activeCalls > 0 ||
|
|
1006
|
+
state.failed ||
|
|
1007
|
+
!state.terminalCompleted ||
|
|
1008
|
+
!state.program.closed
|
|
1009
|
+
) {
|
|
1010
|
+
return;
|
|
1011
|
+
}
|
|
1012
|
+
if (this.items.get(state.address) === state.program) {
|
|
1013
|
+
this.items.delete(state.address);
|
|
1014
|
+
}
|
|
1015
|
+
this.restoreTerminalOperations(state);
|
|
1016
|
+
if (this._failedInitializations.get(state.address) === state.program) {
|
|
1017
|
+
this._failedInitializations.delete(state.address);
|
|
1018
|
+
}
|
|
1019
|
+
if (this._terminalOperationsByAddress.get(state.address) === state) {
|
|
1020
|
+
this._terminalOperationsByAddress.delete(state.address);
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
private restoreTerminalOperations(state: TerminalOperationState<T>): void {
|
|
1025
|
+
if (
|
|
1026
|
+
state.closeWrapper &&
|
|
1027
|
+
state.closeOperation &&
|
|
1028
|
+
state.program.close === state.closeWrapper
|
|
1029
|
+
) {
|
|
1030
|
+
state.program.close = state.closeOperation;
|
|
1031
|
+
}
|
|
1032
|
+
const droppable = state.program as T & {
|
|
1033
|
+
drop?: (...args: any[]) => Promise<boolean>;
|
|
1034
|
+
};
|
|
1035
|
+
if (
|
|
1036
|
+
state.dropWrapper &&
|
|
1037
|
+
state.dropOperation &&
|
|
1038
|
+
droppable.drop === state.dropWrapper
|
|
1039
|
+
) {
|
|
1040
|
+
droppable.drop = state.dropOperation;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
private findMonitoredPrograms(address: Address): T[] {
|
|
1045
|
+
const targetAddress = address.toString();
|
|
1046
|
+
const pending: Manageable<any>[] = [...this.items.values()];
|
|
1047
|
+
const visited = new Set<Manageable<any>>();
|
|
1048
|
+
const matches: T[] = [];
|
|
1049
|
+
while (pending.length > 0) {
|
|
1050
|
+
const candidate = pending.pop()!;
|
|
1051
|
+
if (visited.has(candidate)) continue;
|
|
1052
|
+
visited.add(candidate);
|
|
1053
|
+
if (candidate.address.toString() === targetAddress) {
|
|
1054
|
+
matches.push(candidate as T);
|
|
1055
|
+
}
|
|
1056
|
+
for (const child of candidate.children ?? []) {
|
|
1057
|
+
pending.push(child);
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
return matches;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1063
|
+
private findMonitoredProgram(
|
|
1064
|
+
address: Address,
|
|
1065
|
+
liveOnly = false,
|
|
1066
|
+
preferred?: Manageable<any>,
|
|
1067
|
+
): T | undefined {
|
|
1068
|
+
const matches = this.findMonitoredPrograms(address);
|
|
1069
|
+
if (
|
|
1070
|
+
preferred &&
|
|
1071
|
+
matches.includes(preferred as T) &&
|
|
1072
|
+
(!liveOnly || !preferred.closed)
|
|
1073
|
+
) {
|
|
1074
|
+
return preferred as T;
|
|
1075
|
+
}
|
|
1076
|
+
return liveOnly
|
|
1077
|
+
? matches.find((candidate) => !candidate.closed)
|
|
1078
|
+
: (matches.find((candidate) => !candidate.closed) ?? matches[0]);
|
|
1079
|
+
}
|
|
1080
|
+
|
|
1081
|
+
private attachExistingProgram<S extends T>(
|
|
1082
|
+
address: Address,
|
|
1083
|
+
program: S,
|
|
1084
|
+
parent?: Manageable<any>,
|
|
1085
|
+
): S {
|
|
1086
|
+
if (parent) this.assertParentCanOwnOpen(parent);
|
|
1087
|
+
addParent(program, parent);
|
|
1088
|
+
if (parent == null && this.items.get(address) !== program) {
|
|
1089
|
+
const direct = this.items.get(address);
|
|
1090
|
+
if (direct && !direct.closed && direct !== program) {
|
|
1091
|
+
throw new Error(
|
|
1092
|
+
`Program at ${address} is already monitored as another live instance`,
|
|
1093
|
+
);
|
|
1094
|
+
}
|
|
1095
|
+
this.monitorTerminalOperations(address, program);
|
|
1096
|
+
this.items.set(address, program);
|
|
1097
|
+
}
|
|
1098
|
+
return program;
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
private assertNoTerminalOperation(
|
|
1102
|
+
address: Address,
|
|
1103
|
+
requestedProgram?: Manageable<any>,
|
|
1104
|
+
allowSettledAncestorRepair = requestedProgram != null,
|
|
1105
|
+
): void {
|
|
1106
|
+
const residual = [...this._cleanupResiduals.values()].find(
|
|
1107
|
+
(candidate) =>
|
|
1108
|
+
candidate.program.address.toString() === address.toString(),
|
|
1109
|
+
);
|
|
1110
|
+
if (residual) {
|
|
1111
|
+
throw new Error(
|
|
1112
|
+
`Program at ${address} has pending cleanup residuals and cannot be reopened before cleanup is retried`,
|
|
1113
|
+
);
|
|
1114
|
+
}
|
|
1115
|
+
const state = this._terminalOperationsByAddress.get(address);
|
|
1116
|
+
if (state?.activeCalls) {
|
|
1117
|
+
throw new Error(
|
|
1118
|
+
`Program is terminating and cannot accept another parent while its close or drop operation is finishing (${address})`,
|
|
1119
|
+
);
|
|
1120
|
+
}
|
|
1121
|
+
if (state?.failed) {
|
|
1122
|
+
throw new Error(
|
|
1123
|
+
`Program at ${address} failed terminal cleanup and cannot be reopened before cleanup is retried`,
|
|
1124
|
+
);
|
|
1125
|
+
}
|
|
1126
|
+
const targetAddress = address.toString();
|
|
1127
|
+
const pending: {
|
|
1128
|
+
program: Manageable<any>;
|
|
1129
|
+
ancestorActive: boolean;
|
|
1130
|
+
ancestorSettled: boolean;
|
|
1131
|
+
ancestorResidual: boolean;
|
|
1132
|
+
}[] = [...this.items.values()].map((program) => ({
|
|
1133
|
+
program,
|
|
1134
|
+
ancestorActive: false,
|
|
1135
|
+
ancestorSettled: false,
|
|
1136
|
+
ancestorResidual: false,
|
|
1137
|
+
}));
|
|
1138
|
+
for (const cleanupResidual of this._cleanupResiduals.values()) {
|
|
1139
|
+
pending.push({
|
|
1140
|
+
program: cleanupResidual.program,
|
|
1141
|
+
ancestorActive: false,
|
|
1142
|
+
ancestorSettled: false,
|
|
1143
|
+
ancestorResidual: true,
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
const visited = new Map<Manageable<any>, number>();
|
|
1147
|
+
while (pending.length > 0) {
|
|
1148
|
+
const {
|
|
1149
|
+
program: candidate,
|
|
1150
|
+
ancestorActive,
|
|
1151
|
+
ancestorSettled,
|
|
1152
|
+
ancestorResidual,
|
|
1153
|
+
} = pending.pop()!;
|
|
1154
|
+
const candidateState = this._terminalOperationsByProgram.get(
|
|
1155
|
+
candidate as T,
|
|
1156
|
+
);
|
|
1157
|
+
const ownActive = (candidateState?.activeCalls ?? 0) > 0;
|
|
1158
|
+
const ownFailed = candidateState?.failed === true;
|
|
1159
|
+
const ownAttachmentFence =
|
|
1160
|
+
candidate.acceptsParentAttachments === false &&
|
|
1161
|
+
(!candidate.closed ||
|
|
1162
|
+
candidate.pendingTerminalOperation != null ||
|
|
1163
|
+
ownActive ||
|
|
1164
|
+
ownFailed);
|
|
1165
|
+
const stateMask =
|
|
1166
|
+
(ancestorActive ? 1 : 0) |
|
|
1167
|
+
(ancestorSettled ? 2 : 0) |
|
|
1168
|
+
(ancestorResidual ? 4 : 0);
|
|
1169
|
+
const previousMask = visited.get(candidate) ?? -1;
|
|
1170
|
+
if (previousMask !== -1 && (previousMask | stateMask) === previousMask) {
|
|
1171
|
+
continue;
|
|
1172
|
+
}
|
|
1173
|
+
visited.set(
|
|
1174
|
+
candidate,
|
|
1175
|
+
previousMask === -1 ? stateMask : previousMask | stateMask,
|
|
1176
|
+
);
|
|
1177
|
+
if (candidate.address.toString() === targetAddress) {
|
|
1178
|
+
const exactRepair =
|
|
1179
|
+
allowSettledAncestorRepair && requestedProgram === candidate;
|
|
1180
|
+
if (
|
|
1181
|
+
ownActive ||
|
|
1182
|
+
ownFailed ||
|
|
1183
|
+
ownAttachmentFence ||
|
|
1184
|
+
ancestorActive ||
|
|
1185
|
+
ancestorResidual ||
|
|
1186
|
+
(ancestorSettled && !exactRepair)
|
|
1187
|
+
) {
|
|
1188
|
+
throw new Error(
|
|
1189
|
+
`Program is terminating and cannot accept another parent while its cleanup is finishing (${address})`,
|
|
1190
|
+
);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
const descendantActive = ancestorActive || ownActive;
|
|
1194
|
+
const descendantSettled =
|
|
1195
|
+
ancestorSettled || (!ownActive && (ownFailed || ownAttachmentFence));
|
|
1196
|
+
for (const child of candidate.children ?? []) {
|
|
1197
|
+
pending.push({
|
|
1198
|
+
program: child,
|
|
1199
|
+
ancestorActive: descendantActive,
|
|
1200
|
+
ancestorSettled: descendantSettled,
|
|
1201
|
+
ancestorResidual,
|
|
1202
|
+
});
|
|
1203
|
+
}
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
|
|
1207
|
+
private assertTerminalGraphCanStart(program: Manageable<any>): void {
|
|
1208
|
+
if (this._initializationRollbacks.has(program)) {
|
|
1209
|
+
return;
|
|
1210
|
+
}
|
|
1211
|
+
const replacing = this._replacementClosures.has(program);
|
|
1212
|
+
const pending = [program];
|
|
1213
|
+
const visited = new Set<Manageable<any>>();
|
|
1214
|
+
while (pending.length > 0) {
|
|
1215
|
+
const candidate = pending.pop()!;
|
|
1216
|
+
if (visited.has(candidate)) continue;
|
|
1217
|
+
visited.add(candidate);
|
|
1218
|
+
if ((this._parentAttachmentReservations.get(candidate) ?? 0) > 0) {
|
|
1219
|
+
throw new TerminalOperationNotStartedError(
|
|
1220
|
+
`Program at ${candidate.address} has an opening child attachment that must finish first`,
|
|
1221
|
+
);
|
|
1222
|
+
}
|
|
1223
|
+
const address = candidate.address.toString();
|
|
1224
|
+
const rootReplacementOpen = replacing && candidate === program;
|
|
1225
|
+
if (
|
|
1226
|
+
(this._openingPromises.has(address) && !rootReplacementOpen) ||
|
|
1227
|
+
(this._openingReservationsByAddress.get(address)?.size ?? 0) > 0
|
|
1228
|
+
) {
|
|
1229
|
+
throw new TerminalOperationNotStartedError(
|
|
1230
|
+
`Program at ${address} cannot terminate while an open generation owns its address`,
|
|
1231
|
+
);
|
|
1232
|
+
}
|
|
1233
|
+
for (const child of candidate.children ?? []) {
|
|
1234
|
+
pending.push(child);
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
private async waitForTerminalOperations(
|
|
1240
|
+
program: Manageable<any>,
|
|
1241
|
+
): Promise<void> {
|
|
1242
|
+
const state = this._terminalOperationsByProgram.get(program);
|
|
1243
|
+
while (state && state.pending.size > 0) {
|
|
1244
|
+
// A rejected direct close is not itself the result of stop(). Once it has
|
|
1245
|
+
// settled, closeCompletely() retries the retained instance below.
|
|
1246
|
+
await Promise.allSettled([...state.pending]);
|
|
1247
|
+
}
|
|
1248
|
+
}
|
|
1249
|
+
|
|
1250
|
+
private async closeCompletely(program: Manageable<any>): Promise<void> {
|
|
1251
|
+
await this.waitForTerminalOperations(program);
|
|
1252
|
+
let terminalState = this._terminalOperationsByProgram.get(program);
|
|
1253
|
+
while (!program.closed || terminalState?.failed) {
|
|
1254
|
+
if (terminalState) {
|
|
1255
|
+
// Applications/tests may replace a failed wrapped method before stop.
|
|
1256
|
+
// Re-wrap any replacement in the existing identity-specific state before
|
|
1257
|
+
// invoking it, so both a fresh failure and exact recovery remain monitored.
|
|
1258
|
+
this.refreshTerminalOperationWrappers(terminalState);
|
|
1259
|
+
}
|
|
1260
|
+
const wasRecovering = terminalState?.failed === true;
|
|
1261
|
+
const failedCall = terminalState?.failedCall;
|
|
1262
|
+
const ownersBefore = [...(program.parents ?? [])];
|
|
1263
|
+
const owner = wasRecovering
|
|
1264
|
+
? (failedCall?.args[0] as Manageable<any> | undefined)
|
|
1265
|
+
: program.closed
|
|
1266
|
+
? undefined
|
|
1267
|
+
: ownersBefore[0];
|
|
1268
|
+
const ownerReferencesBefore = ownersBefore.filter(
|
|
1269
|
+
(candidate) => candidate === owner,
|
|
1270
|
+
).length;
|
|
1271
|
+
const operationType: TerminalOperation =
|
|
1272
|
+
failedCall?.type ??
|
|
1273
|
+
(terminalState?.failedOperation === "drop" ? "drop" : "close");
|
|
1274
|
+
const operation = (
|
|
1275
|
+
program as Manageable<any> & {
|
|
1276
|
+
drop?: (from?: Manageable<any>) => Promise<boolean>;
|
|
1277
|
+
}
|
|
1278
|
+
)[operationType];
|
|
1279
|
+
if (!operation) {
|
|
1280
|
+
throw new Error(
|
|
1281
|
+
`Program at ${program.address} cannot retry its failed ${operationType} cleanup`,
|
|
1282
|
+
);
|
|
1283
|
+
}
|
|
1284
|
+
const operationArgs = wasRecovering ? (failedCall?.args ?? []) : [owner];
|
|
1285
|
+
const closed = await (
|
|
1286
|
+
operation as (...args: any[]) => Promise<boolean>
|
|
1287
|
+
).apply(program, operationArgs);
|
|
1288
|
+
await this.waitForTerminalOperations(program);
|
|
1289
|
+
terminalState = this._terminalOperationsByProgram.get(program);
|
|
1290
|
+
if (
|
|
1291
|
+
terminalState?.failed &&
|
|
1292
|
+
terminalState.failedCall === failedCall &&
|
|
1293
|
+
terminalState.failedOperation === operationType &&
|
|
1294
|
+
((closed && program.closed) ||
|
|
1295
|
+
(failedCall?.commit != null && closed === failedCall.commit.result))
|
|
1296
|
+
) {
|
|
1297
|
+
// This also supports a test/application replacing the wrapped method
|
|
1298
|
+
// after open: closeCompletely awaited the full replacement call, so it
|
|
1299
|
+
// is safe to mark the previously retained cleanup as recovered. A
|
|
1300
|
+
// committed non-terminal release must reproduce its recorded `false`;
|
|
1301
|
+
// the fresh loop below then drains the owners that remain.
|
|
1302
|
+
terminalState.failed = false;
|
|
1303
|
+
terminalState.failedOperation = undefined;
|
|
1304
|
+
terminalState.failedCall = undefined;
|
|
1305
|
+
terminalState.terminalCompleted = closed && program.closed;
|
|
1306
|
+
this.releaseRetainedTerminalIdentity(terminalState);
|
|
1307
|
+
if (terminalState.cleanupLease) {
|
|
1308
|
+
program[TERMINAL_OUTER_CLEANUP_RELEASE]?.(terminalState.cleanupLease);
|
|
1309
|
+
terminalState.cleanupLease = undefined;
|
|
1310
|
+
}
|
|
1311
|
+
this.finishTerminalOperation(terminalState);
|
|
1312
|
+
}
|
|
1313
|
+
|
|
1314
|
+
const ownersAfter = program.parents ?? [];
|
|
1315
|
+
const ownerReferencesAfter = ownersAfter.filter(
|
|
1316
|
+
(candidate) => candidate === owner,
|
|
1317
|
+
).length;
|
|
1318
|
+
if (owner) {
|
|
1319
|
+
const inverseEdges =
|
|
1320
|
+
owner.children?.filter((candidate) => candidate === program).length ??
|
|
1321
|
+
0;
|
|
1322
|
+
this.removeReleasedParentChildReferences(
|
|
1323
|
+
program,
|
|
1324
|
+
owner,
|
|
1325
|
+
Math.max(0, inverseEdges - ownerReferencesAfter),
|
|
1326
|
+
);
|
|
1327
|
+
}
|
|
1328
|
+
if (program.closed && !terminalState?.failed) {
|
|
1329
|
+
return;
|
|
1330
|
+
}
|
|
1331
|
+
if (wasRecovering && !terminalState?.failed) {
|
|
1332
|
+
// The failed outer call has now completed. Its base ownership transition
|
|
1333
|
+
// may already have committed, so evaluate remaining owners on a fresh loop.
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
// A non-terminal Program.close(from) is valid only when it releases the
|
|
1337
|
+
// requested owner and strictly reduces the total owner count. The latter is
|
|
1338
|
+
// a monotonic measure that bounds this loop.
|
|
1339
|
+
if (
|
|
1340
|
+
closed ||
|
|
1341
|
+
ownersAfter.length >= ownersBefore.length ||
|
|
1342
|
+
ownerReferencesAfter >= ownerReferencesBefore
|
|
1343
|
+
) {
|
|
1344
|
+
throw new Error(
|
|
1345
|
+
`Program at ${program.address} did not make ownership progress while stopping (${ownersBefore.length} -> ${ownersAfter.length})`,
|
|
1346
|
+
);
|
|
1347
|
+
}
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
private removeReleasedParentChildReferences(
|
|
1352
|
+
program: Manageable<any>,
|
|
1353
|
+
parent: Manageable<any> | undefined,
|
|
1354
|
+
releasedReferences: number,
|
|
1355
|
+
) {
|
|
1356
|
+
if (!parent) {
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1359
|
+
while (releasedReferences > 0) {
|
|
1360
|
+
const childIndex = parent.children?.indexOf(program) ?? -1;
|
|
1361
|
+
if (childIndex === -1) {
|
|
1362
|
+
return;
|
|
1363
|
+
}
|
|
1364
|
+
parent.children.splice(childIndex, 1);
|
|
1365
|
+
releasedReferences -= 1;
|
|
1366
|
+
}
|
|
1367
|
+
}
|
|
1368
|
+
|
|
1369
|
+
private assertNoFailedInitialization(address: Address) {
|
|
1370
|
+
const failed = this._failedInitializations.get(address);
|
|
1371
|
+
if (!failed) {
|
|
1372
|
+
return;
|
|
1373
|
+
}
|
|
1374
|
+
if (failed.closed) {
|
|
1375
|
+
if (this._terminalOperationsByAddress.get(address)?.failed) {
|
|
1376
|
+
return;
|
|
1377
|
+
}
|
|
1378
|
+
this._failedInitializations.delete(address);
|
|
1379
|
+
if (this.items.get(address) === failed) {
|
|
1380
|
+
this.items.delete(address);
|
|
1381
|
+
}
|
|
1382
|
+
return;
|
|
1383
|
+
}
|
|
1384
|
+
throw new Error(
|
|
1385
|
+
`Program at ${address} failed initialization cleanup and cannot be reopened before it is stopped`,
|
|
1386
|
+
);
|
|
1387
|
+
}
|
|
1388
|
+
|
|
1389
|
+
private async stopOnce(): Promise<void> {
|
|
1390
|
+
// Do not PQueue.clear(): p-queue drops queued callbacks without settling the
|
|
1391
|
+
// promises returned to callers. Admitted opens must run to completion (or
|
|
1392
|
+
// rejection) before their programs can be closed below.
|
|
1393
|
+
await Promise.all([...this._openAdmissions]);
|
|
103
1394
|
await Promise.all(
|
|
104
|
-
[...this._openQueue.values()].map((
|
|
105
|
-
x.clear();
|
|
106
|
-
return x.onIdle();
|
|
107
|
-
}),
|
|
1395
|
+
[...this._openQueue.values()].map((queue) => queue.onIdle()),
|
|
108
1396
|
);
|
|
109
1397
|
this._openQueue.clear();
|
|
110
1398
|
|
|
@@ -114,60 +1402,959 @@ export class Handler<T extends Manageable<any>> {
|
|
|
114
1402
|
await Promise.allSettled([...this._openingPromises.values()]);
|
|
115
1403
|
}
|
|
116
1404
|
this._openingPromises.clear();
|
|
1405
|
+
this._openingReservations.clear();
|
|
1406
|
+
this._openingReservationsByAddress.clear();
|
|
117
1407
|
|
|
118
|
-
//
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
1408
|
+
// A failed child can have its wrapped terminal method replaced before stop.
|
|
1409
|
+
// Refresh every retained failed identity before closing any parent root, so
|
|
1410
|
+
// parent traversal consumes the recorded commit through the new wrapper
|
|
1411
|
+
// instead of calling the replacement outside Handler monitoring.
|
|
1412
|
+
const pendingTerminalGraphs: Manageable<any>[] = [
|
|
1413
|
+
...this.items.values(),
|
|
1414
|
+
...this._cleanupResiduals.keys(),
|
|
1415
|
+
];
|
|
1416
|
+
const visitedTerminalGraphs = new Set<Manageable<any>>();
|
|
1417
|
+
while (pendingTerminalGraphs.length > 0) {
|
|
1418
|
+
const candidate = pendingTerminalGraphs.pop()!;
|
|
1419
|
+
if (visitedTerminalGraphs.has(candidate)) continue;
|
|
1420
|
+
visitedTerminalGraphs.add(candidate);
|
|
1421
|
+
const terminalState = this._terminalOperationsByProgram.get(candidate);
|
|
1422
|
+
if (terminalState?.failed) {
|
|
1423
|
+
this.refreshTerminalOperationWrappers(terminalState);
|
|
1424
|
+
}
|
|
1425
|
+
for (const child of candidate.children ?? []) {
|
|
1426
|
+
pendingTerminalGraphs.push(child);
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
|
|
1430
|
+
// A close can legitimately return false when it only releases one of several
|
|
1431
|
+
// owners. Drain every ownership reference so stop cannot discard a still-live
|
|
1432
|
+
// program from Handler state.
|
|
1433
|
+
const closeErrors: unknown[] = [];
|
|
1434
|
+
for (const [address, program] of [...this.items]) {
|
|
1435
|
+
try {
|
|
1436
|
+
await this.closeCompletely(program);
|
|
1437
|
+
} catch (error) {
|
|
1438
|
+
closeErrors.push(error);
|
|
1439
|
+
}
|
|
1440
|
+
const terminalState = this._terminalOperationsByProgram.get(program);
|
|
1441
|
+
if (
|
|
1442
|
+
program.closed &&
|
|
1443
|
+
!terminalState?.failed &&
|
|
1444
|
+
(terminalState?.activeCalls ?? 0) === 0 &&
|
|
1445
|
+
this.items.get(address) === program
|
|
1446
|
+
) {
|
|
1447
|
+
this.items.delete(address);
|
|
1448
|
+
}
|
|
1449
|
+
if (
|
|
1450
|
+
program.closed &&
|
|
1451
|
+
!terminalState?.failed &&
|
|
1452
|
+
this._failedInitializations.get(address) === program
|
|
1453
|
+
) {
|
|
1454
|
+
this._failedInitializations.delete(address);
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
for (const [program, residual] of [...this._cleanupResiduals]) {
|
|
1458
|
+
try {
|
|
1459
|
+
for (const failure of [...residual.failures]) {
|
|
1460
|
+
await this.retryCleanupResidual(residual, failure);
|
|
1461
|
+
const failureIndex = residual.failures.indexOf(failure);
|
|
1462
|
+
if (failureIndex !== -1) residual.failures.splice(failureIndex, 1);
|
|
1463
|
+
}
|
|
1464
|
+
await this.closeCompletely(program);
|
|
1465
|
+
} catch (error) {
|
|
1466
|
+
closeErrors.push(error);
|
|
1467
|
+
}
|
|
1468
|
+
const terminalState = this._terminalOperationsByProgram.get(program);
|
|
1469
|
+
if (
|
|
1470
|
+
program.closed &&
|
|
1471
|
+
residual.failures.length === 0 &&
|
|
1472
|
+
residual.terminalState == null &&
|
|
1473
|
+
!terminalState?.failed &&
|
|
1474
|
+
(terminalState?.activeCalls ?? 0) === 0
|
|
1475
|
+
) {
|
|
1476
|
+
this.releaseCleanupResidual(residual);
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
if (closeErrors.length > 0) {
|
|
1480
|
+
throw closeErrors[0];
|
|
1481
|
+
}
|
|
122
1482
|
|
|
123
|
-
// Remove all databases from the state
|
|
124
1483
|
this.items = new Map();
|
|
1484
|
+
this._failedInitializations.clear();
|
|
1485
|
+
for (const residual of [...this._cleanupResiduals.values()]) {
|
|
1486
|
+
this.releaseCleanupResidual(residual);
|
|
1487
|
+
}
|
|
1488
|
+
for (const state of new Set(this._terminalOperationsByAddress.values())) {
|
|
1489
|
+
this.restoreTerminalOperations(state);
|
|
1490
|
+
}
|
|
1491
|
+
this._terminalOperationsByAddress.clear();
|
|
125
1492
|
}
|
|
126
1493
|
|
|
127
1494
|
private _onProgamClose(program: Manageable<any>) {
|
|
128
|
-
|
|
129
|
-
|
|
1495
|
+
const address = program.address!.toString();
|
|
1496
|
+
const terminalState = this._terminalOperationsByProgram.get(program);
|
|
1497
|
+
if (terminalState?.activeCalls || terminalState?.failed) {
|
|
1498
|
+
// The base Program has ended, but the dynamically-dispatched subclass
|
|
1499
|
+
// close/drop promise is still running. Its wrapper owns final eviction.
|
|
1500
|
+
return;
|
|
1501
|
+
}
|
|
1502
|
+
if (this.items.get(address) === program) {
|
|
1503
|
+
this.items.delete(address);
|
|
1504
|
+
}
|
|
1505
|
+
if (this._failedInitializations.get(address) === program) {
|
|
1506
|
+
this._failedInitializations.delete(address);
|
|
1507
|
+
}
|
|
130
1508
|
// TODO remove item from this._openQueue?
|
|
131
1509
|
}
|
|
132
1510
|
|
|
1511
|
+
private async closeForReplacement(
|
|
1512
|
+
address: Address,
|
|
1513
|
+
program: Manageable<any>,
|
|
1514
|
+
parent?: Manageable<any>,
|
|
1515
|
+
): Promise<void> {
|
|
1516
|
+
const owners = program.parents ?? [];
|
|
1517
|
+
if (owners.length > 1 || (owners.length === 1 && owners[0] !== parent)) {
|
|
1518
|
+
throw new Error(
|
|
1519
|
+
`Program at ${address} cannot be replaced while it has other owners`,
|
|
1520
|
+
);
|
|
1521
|
+
}
|
|
1522
|
+
const parentReferencesBefore = owners.filter(
|
|
1523
|
+
(candidate) => candidate === parent,
|
|
1524
|
+
).length;
|
|
1525
|
+
let closed: boolean;
|
|
1526
|
+
this._replacementClosures.add(program);
|
|
1527
|
+
try {
|
|
1528
|
+
closed = await (
|
|
1529
|
+
program.close as unknown as (from?: Manageable<any>) => Promise<boolean>
|
|
1530
|
+
).call(program, parent);
|
|
1531
|
+
} finally {
|
|
1532
|
+
this._replacementClosures.delete(program);
|
|
1533
|
+
}
|
|
1534
|
+
const parentReferencesAfter = (program.parents ?? []).filter(
|
|
1535
|
+
(candidate) => candidate === parent,
|
|
1536
|
+
).length;
|
|
1537
|
+
this.removeReleasedParentChildReferences(
|
|
1538
|
+
program,
|
|
1539
|
+
parent,
|
|
1540
|
+
parentReferencesBefore - parentReferencesAfter,
|
|
1541
|
+
);
|
|
1542
|
+
if (!closed || !program.closed || parentReferencesAfter > 0) {
|
|
1543
|
+
throw new Error(
|
|
1544
|
+
`Program at ${address} cannot be replaced because close was not terminal`,
|
|
1545
|
+
);
|
|
1546
|
+
}
|
|
1547
|
+
if (this.items.get(address) === program) {
|
|
1548
|
+
this.items.delete(address);
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
|
|
133
1552
|
private async checkProcessExisting<S extends T>(
|
|
134
1553
|
address: Address,
|
|
135
1554
|
toOpen: Manageable<any>,
|
|
136
1555
|
mergeSrategy: ProgramMergeStrategy = "reject",
|
|
1556
|
+
parent?: Manageable<any>,
|
|
137
1557
|
): Promise<S | undefined> {
|
|
138
|
-
|
|
1558
|
+
this.assertNoFailedInitialization(address);
|
|
1559
|
+
this.assertNoTerminalOperation(address, toOpen);
|
|
1560
|
+
const prev = this.findMonitoredProgram(address, false, toOpen);
|
|
1561
|
+
if (prev?.closed) {
|
|
1562
|
+
if (this.items.get(address) === prev) {
|
|
1563
|
+
this.items.delete(address);
|
|
1564
|
+
}
|
|
1565
|
+
return undefined;
|
|
1566
|
+
}
|
|
139
1567
|
if (mergeSrategy === "reject") {
|
|
140
1568
|
if (prev) {
|
|
141
1569
|
throw new Error(`Program at ${address} is already open`);
|
|
142
1570
|
}
|
|
143
1571
|
} else if (mergeSrategy === "replace") {
|
|
144
1572
|
if (prev && prev !== toOpen) {
|
|
145
|
-
await
|
|
1573
|
+
await this.closeForReplacement(address, prev, parent);
|
|
146
1574
|
}
|
|
147
1575
|
} else if (mergeSrategy === "reuse") {
|
|
1576
|
+
if (prev) {
|
|
1577
|
+
this.attachExistingProgram(address, prev, parent);
|
|
1578
|
+
}
|
|
148
1579
|
return prev as S;
|
|
149
1580
|
}
|
|
150
1581
|
}
|
|
151
1582
|
|
|
1583
|
+
private async processParentExisting<S extends T>(
|
|
1584
|
+
address: Address,
|
|
1585
|
+
requestedProgram: Manageable<any> | undefined,
|
|
1586
|
+
mergeStrategy: ProgramMergeStrategy,
|
|
1587
|
+
parent: Manageable<any>,
|
|
1588
|
+
): Promise<S | undefined> {
|
|
1589
|
+
this.assertNoFailedInitialization(address);
|
|
1590
|
+
this.assertNoTerminalOperation(address, requestedProgram);
|
|
1591
|
+
const existing = this.findMonitoredProgram(
|
|
1592
|
+
address,
|
|
1593
|
+
false,
|
|
1594
|
+
requestedProgram,
|
|
1595
|
+
);
|
|
1596
|
+
if (existing?.closed) {
|
|
1597
|
+
if (this.items.get(address) === existing) {
|
|
1598
|
+
this.items.delete(address);
|
|
1599
|
+
}
|
|
1600
|
+
return undefined;
|
|
1601
|
+
}
|
|
1602
|
+
if (!existing) {
|
|
1603
|
+
return undefined;
|
|
1604
|
+
}
|
|
1605
|
+
|
|
1606
|
+
if (mergeStrategy === "reject") {
|
|
1607
|
+
throw new Error(`Program at ${address} is already open`);
|
|
1608
|
+
}
|
|
1609
|
+
if (mergeStrategy === "replace" && existing !== requestedProgram) {
|
|
1610
|
+
await this.closeForReplacement(address, existing, parent);
|
|
1611
|
+
return undefined;
|
|
1612
|
+
}
|
|
1613
|
+
|
|
1614
|
+
this.attachExistingProgram(address, existing, parent);
|
|
1615
|
+
try {
|
|
1616
|
+
// A program explicitly opened with a parent is a weak/addressable
|
|
1617
|
+
// reference. Structural beforeOpen() may already have made the exact
|
|
1618
|
+
// instance live without persisting its standalone block.
|
|
1619
|
+
const blocks = this.properties.client.services.blocks;
|
|
1620
|
+
if (!(await blocks.has(address))) {
|
|
1621
|
+
await existing.save(blocks, { skipOnAddress: false });
|
|
1622
|
+
}
|
|
1623
|
+
} catch (error) {
|
|
1624
|
+
const parentIndex = existing.parents?.lastIndexOf(parent) ?? -1;
|
|
1625
|
+
if (parentIndex !== -1) existing.parents.splice(parentIndex, 1);
|
|
1626
|
+
const childIndex = parent.children?.lastIndexOf(existing) ?? -1;
|
|
1627
|
+
if (childIndex !== -1) parent.children.splice(childIndex, 1);
|
|
1628
|
+
throw error;
|
|
1629
|
+
}
|
|
1630
|
+
return existing as S;
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
private async rollbackFailedInitialization(
|
|
1634
|
+
address: Address,
|
|
1635
|
+
program: Manageable<any>,
|
|
1636
|
+
state: {
|
|
1637
|
+
parents?: (Manageable<any> | undefined)[];
|
|
1638
|
+
children?: Manageable<any>[];
|
|
1639
|
+
parent?: Manageable<any>;
|
|
1640
|
+
parentChildReferences: number;
|
|
1641
|
+
},
|
|
1642
|
+
): Promise<void> {
|
|
1643
|
+
if (this.items.get(address) === program) {
|
|
1644
|
+
this.items.delete(address);
|
|
1645
|
+
}
|
|
1646
|
+
|
|
1647
|
+
const cleanupErrors: unknown[] = [];
|
|
1648
|
+
const childrenAfterFailure = [...(program.children ?? [])];
|
|
1649
|
+
try {
|
|
1650
|
+
if (!program.closed) {
|
|
1651
|
+
await program.close();
|
|
1652
|
+
}
|
|
1653
|
+
} catch (cleanupError) {
|
|
1654
|
+
cleanupErrors.push(cleanupError);
|
|
1655
|
+
logger.error(
|
|
1656
|
+
`Failed to close partially opened program at ${address}: ${String(
|
|
1657
|
+
cleanupError,
|
|
1658
|
+
)}`,
|
|
1659
|
+
);
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1662
|
+
// beforeOpen() can attach nested children before the parent itself becomes
|
|
1663
|
+
// open. Remove only relationships added by this failed generation, then
|
|
1664
|
+
// restore the exact caller-visible parent/child arrays captured at entry.
|
|
1665
|
+
const baselineChildCounts = new Map<Manageable<any>, number>();
|
|
1666
|
+
for (const child of state.children ?? []) {
|
|
1667
|
+
baselineChildCounts.set(child, (baselineChildCounts.get(child) ?? 0) + 1);
|
|
1668
|
+
}
|
|
1669
|
+
const currentChildCounts = new Map<Manageable<any>, number>();
|
|
1670
|
+
for (const child of childrenAfterFailure) {
|
|
1671
|
+
currentChildCounts.set(child, (currentChildCounts.get(child) ?? 0) + 1);
|
|
1672
|
+
}
|
|
1673
|
+
for (const [child, currentCount] of currentChildCounts) {
|
|
1674
|
+
const baselineCount = baselineChildCounts.get(child) ?? 0;
|
|
1675
|
+
let extra = currentCount - baselineCount;
|
|
1676
|
+
while (extra > 0) {
|
|
1677
|
+
const referencesBefore =
|
|
1678
|
+
child.parents?.filter((parent) => parent === program).length ?? 0;
|
|
1679
|
+
let retainedFailure = false;
|
|
1680
|
+
let cleanupReservation: CleanupResidual | undefined;
|
|
1681
|
+
if (!child.closed && referencesBefore > 0) {
|
|
1682
|
+
const checkpoint = this.terminalCheckpoint(child);
|
|
1683
|
+
cleanupReservation = this.acquireCleanupReservation(child);
|
|
1684
|
+
try {
|
|
1685
|
+
await (
|
|
1686
|
+
child.close as unknown as (
|
|
1687
|
+
from?: Manageable<any>,
|
|
1688
|
+
) => Promise<boolean>
|
|
1689
|
+
).call(child, program);
|
|
1690
|
+
} catch (cleanupError) {
|
|
1691
|
+
retainedFailure = true;
|
|
1692
|
+
const terminalState = this._terminalOperationsByProgram.get(
|
|
1693
|
+
child as T,
|
|
1694
|
+
);
|
|
1695
|
+
const handlerOwnsFailure =
|
|
1696
|
+
terminalState?.failed === true &&
|
|
1697
|
+
terminalState.failedCall?.type === "close" &&
|
|
1698
|
+
this.sameTerminalArgs(terminalState.failedCall.args, [program]);
|
|
1699
|
+
if (handlerOwnsFailure) {
|
|
1700
|
+
} else {
|
|
1701
|
+
cleanupReservation.failures.push({
|
|
1702
|
+
type: "close",
|
|
1703
|
+
args: [program],
|
|
1704
|
+
commit: this.terminalCommit(
|
|
1705
|
+
child,
|
|
1706
|
+
checkpoint,
|
|
1707
|
+
"close",
|
|
1708
|
+
program,
|
|
1709
|
+
),
|
|
1710
|
+
});
|
|
1711
|
+
}
|
|
1712
|
+
this.releaseCleanupReservation(cleanupReservation);
|
|
1713
|
+
cleanupReservation = undefined;
|
|
1714
|
+
cleanupErrors.push(cleanupError);
|
|
1715
|
+
logger.error(
|
|
1716
|
+
`Failed to close partially opened child of ${address}: ${String(
|
|
1717
|
+
cleanupError,
|
|
1718
|
+
)}`,
|
|
1719
|
+
);
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
let referencesAfter =
|
|
1723
|
+
child.parents?.filter((parent) => parent === program).length ?? 0;
|
|
1724
|
+
if (
|
|
1725
|
+
!retainedFailure &&
|
|
1726
|
+
referencesAfter >= referencesBefore &&
|
|
1727
|
+
referencesAfter > 0
|
|
1728
|
+
) {
|
|
1729
|
+
const parentIndex = child.parents.lastIndexOf(program);
|
|
1730
|
+
child.parents.splice(parentIndex, 1);
|
|
1731
|
+
referencesAfter -= 1;
|
|
1732
|
+
}
|
|
1733
|
+
if (cleanupReservation) {
|
|
1734
|
+
this.releaseCleanupReservation(cleanupReservation);
|
|
1735
|
+
}
|
|
1736
|
+
extra -= 1;
|
|
1737
|
+
}
|
|
1738
|
+
const remainingProgramReferences =
|
|
1739
|
+
child.parents?.filter((parent) => parent === program).length ?? 0;
|
|
1740
|
+
if (
|
|
1741
|
+
!child.closed &&
|
|
1742
|
+
baselineCount === 0 &&
|
|
1743
|
+
(remainingProgramReferences > 0 || (child.parents?.length ?? 0) === 0)
|
|
1744
|
+
) {
|
|
1745
|
+
// A failed nested cleanup must remain reachable by stop(); restoring the
|
|
1746
|
+
// caller-visible arrays alone would otherwise orphan a live resource.
|
|
1747
|
+
if (!this._cleanupResiduals.has(child)) {
|
|
1748
|
+
this.addCleanupResidual(child, {
|
|
1749
|
+
type: "close",
|
|
1750
|
+
args: [remainingProgramReferences > 0 ? program : undefined],
|
|
1751
|
+
});
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
(program as { parents?: (Manageable<any> | undefined)[] }).parents =
|
|
1757
|
+
state.parents ? [...state.parents] : undefined;
|
|
1758
|
+
(program as { children?: Manageable<any>[] }).children = state.children
|
|
1759
|
+
? [...state.children]
|
|
1760
|
+
: undefined;
|
|
1761
|
+
if (state.parent) {
|
|
1762
|
+
let currentReferences =
|
|
1763
|
+
state.parent.children?.filter((child) => child === program).length ?? 0;
|
|
1764
|
+
while (currentReferences > state.parentChildReferences) {
|
|
1765
|
+
const childIndex = state.parent.children.lastIndexOf(program);
|
|
1766
|
+
if (childIndex === -1) {
|
|
1767
|
+
break;
|
|
1768
|
+
}
|
|
1769
|
+
state.parent.children.splice(childIndex, 1);
|
|
1770
|
+
currentReferences -= 1;
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
if (!program.closed) {
|
|
1775
|
+
// Never manufacture a terminal state after close() failed. Keep the
|
|
1776
|
+
// partially initialized instance identity-tracked so later opens cannot
|
|
1777
|
+
// reuse it and stop() can retry cleanup.
|
|
1778
|
+
this.items.set(address, program as T);
|
|
1779
|
+
this._failedInitializations.set(address, program as T);
|
|
1780
|
+
} else {
|
|
1781
|
+
this._failedInitializations.delete(address);
|
|
1782
|
+
}
|
|
1783
|
+
|
|
1784
|
+
if (cleanupErrors.length > 0) {
|
|
1785
|
+
logger.error(
|
|
1786
|
+
`Program initialization rollback at ${address} encountered ${cleanupErrors.length} cleanup error(s)`,
|
|
1787
|
+
);
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
private dependencyGraph(program: T): Manageable<any>[] {
|
|
1792
|
+
const dependencies = this.properties.getDependencies?.(program) ?? [];
|
|
1793
|
+
return [...new Set<Manageable<any>>([program, ...dependencies])];
|
|
1794
|
+
}
|
|
1795
|
+
|
|
1796
|
+
private async resolveDependencyAddresses(
|
|
1797
|
+
program: T,
|
|
1798
|
+
): Promise<Map<Manageable<any>, string>> {
|
|
1799
|
+
const addresses = new Map<Manageable<any>, string>();
|
|
1800
|
+
for (const candidate of this.dependencyGraph(program)) {
|
|
1801
|
+
const address = await candidate.save(
|
|
1802
|
+
this.properties.client.services.blocks,
|
|
1803
|
+
{
|
|
1804
|
+
skipOnAddress: true,
|
|
1805
|
+
save: () => false,
|
|
1806
|
+
},
|
|
1807
|
+
);
|
|
1808
|
+
addresses.set(candidate, address.toString());
|
|
1809
|
+
}
|
|
1810
|
+
return addresses;
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
private cleanupFailedOpeningMonitoring(
|
|
1814
|
+
reservation: OpeningReservation<T>,
|
|
1815
|
+
): void {
|
|
1816
|
+
for (const state of reservation.provisionalTerminalStates) {
|
|
1817
|
+
if (
|
|
1818
|
+
!state.program.closed ||
|
|
1819
|
+
state.activeCalls > 0 ||
|
|
1820
|
+
state.pending.size > 0 ||
|
|
1821
|
+
state.failed ||
|
|
1822
|
+
state.cleanupLease ||
|
|
1823
|
+
state.program.pendingTerminalOperation != null ||
|
|
1824
|
+
this._cleanupResiduals.has(state.program)
|
|
1825
|
+
) {
|
|
1826
|
+
continue;
|
|
1827
|
+
}
|
|
1828
|
+
this.restoreTerminalOperations(state);
|
|
1829
|
+
if (this._terminalOperationsByAddress.get(state.address) === state) {
|
|
1830
|
+
this._terminalOperationsByAddress.delete(state.address);
|
|
1831
|
+
}
|
|
1832
|
+
this._terminalOperationsByProgram.delete(state.program);
|
|
1833
|
+
}
|
|
1834
|
+
reservation.provisionalTerminalStates.clear();
|
|
1835
|
+
}
|
|
1836
|
+
|
|
1837
|
+
private releaseOpeningReservation(
|
|
1838
|
+
reservation: OpeningReservation<T>,
|
|
1839
|
+
failed = false,
|
|
1840
|
+
): void {
|
|
1841
|
+
for (const address of reservation.programsByAddress.keys()) {
|
|
1842
|
+
const reservations = this._openingReservationsByAddress.get(address);
|
|
1843
|
+
reservations?.delete(reservation);
|
|
1844
|
+
if (reservations?.size === 0) {
|
|
1845
|
+
this._openingReservationsByAddress.delete(address);
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1848
|
+
if (failed) {
|
|
1849
|
+
for (const participant of reservation.contributedParticipants) {
|
|
1850
|
+
const references =
|
|
1851
|
+
reservation.group.participantReferences.get(participant) ?? 0;
|
|
1852
|
+
if (references <= 1) {
|
|
1853
|
+
reservation.group.participantReferences.delete(participant);
|
|
1854
|
+
} else {
|
|
1855
|
+
reservation.group.participantReferences.set(
|
|
1856
|
+
participant,
|
|
1857
|
+
references - 1,
|
|
1858
|
+
);
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
reservation.group.reservations.delete(reservation);
|
|
1863
|
+
this._openingReservations.delete(reservation);
|
|
1864
|
+
reservation.programsByAddress.clear();
|
|
1865
|
+
reservation.programs.clear();
|
|
1866
|
+
reservation.adoptedParents.clear();
|
|
1867
|
+
reservation.contributedParticipants.clear();
|
|
1868
|
+
reservation.wakeWaiters.clear();
|
|
1869
|
+
if (failed) {
|
|
1870
|
+
this.cleanupFailedOpeningMonitoring(reservation);
|
|
1871
|
+
} else {
|
|
1872
|
+
reservation.provisionalTerminalStates.clear();
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
private reservationContains(
|
|
1877
|
+
reservation: OpeningReservation<T>,
|
|
1878
|
+
program: Manageable<any>,
|
|
1879
|
+
): boolean {
|
|
1880
|
+
return reservation.group.participantReferences.has(program);
|
|
1881
|
+
}
|
|
1882
|
+
|
|
1883
|
+
private contributeOpeningParticipant(
|
|
1884
|
+
reservation: OpeningReservation<T>,
|
|
1885
|
+
program: Manageable<any>,
|
|
1886
|
+
): void {
|
|
1887
|
+
if (reservation.contributedParticipants.has(program)) return;
|
|
1888
|
+
reservation.contributedParticipants.add(program);
|
|
1889
|
+
reservation.group.participantReferences.set(
|
|
1890
|
+
program,
|
|
1891
|
+
(reservation.group.participantReferences.get(program) ?? 0) + 1,
|
|
1892
|
+
);
|
|
1893
|
+
}
|
|
1894
|
+
|
|
1895
|
+
private mergeOpeningReservationGroups(
|
|
1896
|
+
left: OpeningReservation<T>,
|
|
1897
|
+
right: OpeningReservation<T>,
|
|
1898
|
+
): void {
|
|
1899
|
+
if (left.group === right.group) return;
|
|
1900
|
+
const target = left.group;
|
|
1901
|
+
const source = right.group;
|
|
1902
|
+
for (const [participant, references] of source.participantReferences) {
|
|
1903
|
+
target.participantReferences.set(
|
|
1904
|
+
participant,
|
|
1905
|
+
(target.participantReferences.get(participant) ?? 0) + references,
|
|
1906
|
+
);
|
|
1907
|
+
}
|
|
1908
|
+
for (const reservation of source.reservations) {
|
|
1909
|
+
reservation.group = target;
|
|
1910
|
+
target.reservations.add(reservation);
|
|
1911
|
+
}
|
|
1912
|
+
source.reservations.clear();
|
|
1913
|
+
source.participantReferences.clear();
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
private canShareOpeningReservation(
|
|
1917
|
+
existing: OpeningReservation<T>,
|
|
1918
|
+
reservation: OpeningReservation<T>,
|
|
1919
|
+
address: string,
|
|
1920
|
+
candidates: Set<Manageable<any>>,
|
|
1921
|
+
parent?: Manageable<any>,
|
|
1922
|
+
): boolean {
|
|
1923
|
+
const existingCandidates = existing.programsByAddress.get(address);
|
|
1924
|
+
if (
|
|
1925
|
+
!existingCandidates ||
|
|
1926
|
+
[...candidates].some((candidate) => !existingCandidates.has(candidate))
|
|
1927
|
+
) {
|
|
1928
|
+
return false;
|
|
1929
|
+
}
|
|
1930
|
+
if (existing.group === reservation.group) return true;
|
|
1931
|
+
if (parent && this.reservationContains(existing, parent)) return true;
|
|
1932
|
+
return [...reservation.adoptedParents].some((adoptedParent) =>
|
|
1933
|
+
this.reservationContains(existing, adoptedParent),
|
|
1934
|
+
);
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
private canOpenAlongsideReservation(
|
|
1938
|
+
existing: OpeningReservation<T>,
|
|
1939
|
+
address: string,
|
|
1940
|
+
candidates: Set<Manageable<any>>,
|
|
1941
|
+
rootProgram: Manageable<any>,
|
|
1942
|
+
): boolean {
|
|
1943
|
+
const existingCandidates = existing.programsByAddress.get(address);
|
|
1944
|
+
if (!existingCandidates) return false;
|
|
1945
|
+
if (
|
|
1946
|
+
[...candidates].some((candidate) => existingCandidates.has(candidate))
|
|
1947
|
+
) {
|
|
1948
|
+
return false;
|
|
1949
|
+
}
|
|
1950
|
+
return (
|
|
1951
|
+
!candidates.has(rootProgram) &&
|
|
1952
|
+
(existing.rootProgram == null ||
|
|
1953
|
+
!existingCandidates.has(existing.rootProgram))
|
|
1954
|
+
);
|
|
1955
|
+
}
|
|
1956
|
+
|
|
1957
|
+
private adoptOpeningReservation(
|
|
1958
|
+
promise: Promise<T>,
|
|
1959
|
+
parent: Manageable<any>,
|
|
1960
|
+
): void {
|
|
1961
|
+
const reservation = this._openingReservationsByPromise.get(promise);
|
|
1962
|
+
if (!reservation || reservation.phase !== "opening") return;
|
|
1963
|
+
reservation.adoptedParents.add(parent);
|
|
1964
|
+
this.contributeOpeningParticipant(reservation, parent);
|
|
1965
|
+
for (const wake of [...reservation.wakeWaiters]) wake();
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
private async waitForOpeningConflictChange(
|
|
1969
|
+
reservation: OpeningReservation<T>,
|
|
1970
|
+
waits: Set<Promise<T>>,
|
|
1971
|
+
): Promise<void> {
|
|
1972
|
+
let wake!: () => void;
|
|
1973
|
+
const adopted = new Promise<void>((resolve) => {
|
|
1974
|
+
wake = resolve;
|
|
1975
|
+
});
|
|
1976
|
+
reservation.wakeWaiters.add(wake);
|
|
1977
|
+
try {
|
|
1978
|
+
await Promise.race([Promise.allSettled(waits), adopted]);
|
|
1979
|
+
} finally {
|
|
1980
|
+
reservation.wakeWaiters.delete(wake);
|
|
1981
|
+
}
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
private assertParentCanOwnOpen(parent: Manageable<any>): void {
|
|
1985
|
+
if (this._initializationRollbacks.has(parent)) {
|
|
1986
|
+
throw new Error(
|
|
1987
|
+
"Parent program is finishing initialization rollback cleanup",
|
|
1988
|
+
);
|
|
1989
|
+
}
|
|
1990
|
+
if (
|
|
1991
|
+
this._cleanupResiduals.has(parent) ||
|
|
1992
|
+
[...this._failedInitializations.values()].some(
|
|
1993
|
+
(failed) => (failed as Manageable<any>) === parent,
|
|
1994
|
+
)
|
|
1995
|
+
) {
|
|
1996
|
+
throw new Error("Parent program has failed lifecycle cleanup");
|
|
1997
|
+
}
|
|
1998
|
+
const terminalState = this._terminalOperationsByProgram.get(parent);
|
|
1999
|
+
if (
|
|
2000
|
+
(terminalState?.activeCalls ?? 0) > 0 ||
|
|
2001
|
+
terminalState?.failed ||
|
|
2002
|
+
parent.acceptsParentAttachments === false
|
|
2003
|
+
) {
|
|
2004
|
+
throw new Error("Parent program is finishing terminal cleanup");
|
|
2005
|
+
}
|
|
2006
|
+
if (parent.closed) {
|
|
2007
|
+
throw new Error("Parent program is closed");
|
|
2008
|
+
}
|
|
2009
|
+
}
|
|
2010
|
+
|
|
2011
|
+
private retainParentAttachment(parent: Manageable<any>): void {
|
|
2012
|
+
this._parentAttachmentReservations.set(
|
|
2013
|
+
parent,
|
|
2014
|
+
(this._parentAttachmentReservations.get(parent) ?? 0) + 1,
|
|
2015
|
+
);
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
private releaseParentAttachment(parent: Manageable<any>): void {
|
|
2019
|
+
const reservations = this._parentAttachmentReservations.get(parent) ?? 0;
|
|
2020
|
+
if (reservations <= 1) {
|
|
2021
|
+
this._parentAttachmentReservations.delete(parent);
|
|
2022
|
+
} else {
|
|
2023
|
+
this._parentAttachmentReservations.set(parent, reservations - 1);
|
|
2024
|
+
}
|
|
2025
|
+
}
|
|
2026
|
+
|
|
2027
|
+
private isReservedNestedUse(
|
|
2028
|
+
address: string,
|
|
2029
|
+
program: Manageable<any> | undefined,
|
|
2030
|
+
parent: Manageable<any>,
|
|
2031
|
+
): boolean {
|
|
2032
|
+
return [...(this._openingReservationsByAddress.get(address) ?? [])].some(
|
|
2033
|
+
(reservation) =>
|
|
2034
|
+
this.reservationContains(reservation, parent) &&
|
|
2035
|
+
(program == null ||
|
|
2036
|
+
reservation.programsByAddress.get(address)?.has(program) === true),
|
|
2037
|
+
);
|
|
2038
|
+
}
|
|
2039
|
+
|
|
2040
|
+
private async waitForOpeningReservations(
|
|
2041
|
+
address: string,
|
|
2042
|
+
program?: Manageable<any>,
|
|
2043
|
+
parent?: Manageable<any>,
|
|
2044
|
+
observedReservations: Set<OpeningReservation<T>> = new Set(),
|
|
2045
|
+
): Promise<void> {
|
|
2046
|
+
while (true) {
|
|
2047
|
+
const waits = new Set<Promise<T>>();
|
|
2048
|
+
for (const reservation of this._openingReservationsByAddress.get(
|
|
2049
|
+
address,
|
|
2050
|
+
) ?? []) {
|
|
2051
|
+
if (
|
|
2052
|
+
parent &&
|
|
2053
|
+
this.reservationContains(reservation, parent) &&
|
|
2054
|
+
(program == null ||
|
|
2055
|
+
reservation.programsByAddress.get(address)?.has(program) === true)
|
|
2056
|
+
) {
|
|
2057
|
+
continue;
|
|
2058
|
+
}
|
|
2059
|
+
if (
|
|
2060
|
+
reservation.phase === "rollback" &&
|
|
2061
|
+
!observedReservations.has(reservation)
|
|
2062
|
+
) {
|
|
2063
|
+
throw new Error(
|
|
2064
|
+
`Program at ${address} is finishing initialization rollback cleanup`,
|
|
2065
|
+
);
|
|
2066
|
+
}
|
|
2067
|
+
waits.add(reservation.promise);
|
|
2068
|
+
}
|
|
2069
|
+
if (waits.size === 0) return;
|
|
2070
|
+
await Promise.allSettled(waits);
|
|
2071
|
+
}
|
|
2072
|
+
}
|
|
2073
|
+
|
|
2074
|
+
private async prepareOpeningGraph(
|
|
2075
|
+
program: T,
|
|
2076
|
+
reservation: OpeningReservation<T>,
|
|
2077
|
+
parent?: Manageable<any>,
|
|
2078
|
+
resolvedAddresses?: Map<Manageable<any>, string>,
|
|
2079
|
+
): Promise<Map<Manageable<any>, string>> {
|
|
2080
|
+
const addresses =
|
|
2081
|
+
resolvedAddresses ?? (await this.resolveDependencyAddresses(program));
|
|
2082
|
+
reservation.rootProgram = program;
|
|
2083
|
+
const programs = [...addresses.keys()];
|
|
2084
|
+
for (const candidate of programs) {
|
|
2085
|
+
this.contributeOpeningParticipant(reservation, candidate);
|
|
2086
|
+
}
|
|
2087
|
+
const programsByAddress = new Map<string, Set<Manageable<any>>>();
|
|
2088
|
+
for (const [candidate, address] of addresses) {
|
|
2089
|
+
const candidates = programsByAddress.get(address) ?? new Set();
|
|
2090
|
+
candidates.add(candidate);
|
|
2091
|
+
programsByAddress.set(address, candidates);
|
|
2092
|
+
}
|
|
2093
|
+
|
|
2094
|
+
while (true) {
|
|
2095
|
+
const waits = new Set<Promise<T>>();
|
|
2096
|
+
for (const [address, candidates] of programsByAddress) {
|
|
2097
|
+
for (const existing of this._openingReservationsByAddress.get(
|
|
2098
|
+
address,
|
|
2099
|
+
) ?? []) {
|
|
2100
|
+
if (existing === reservation) continue;
|
|
2101
|
+
if (
|
|
2102
|
+
this.canShareOpeningReservation(
|
|
2103
|
+
existing,
|
|
2104
|
+
reservation,
|
|
2105
|
+
address,
|
|
2106
|
+
candidates,
|
|
2107
|
+
parent,
|
|
2108
|
+
)
|
|
2109
|
+
) {
|
|
2110
|
+
this.mergeOpeningReservationGroups(reservation, existing);
|
|
2111
|
+
continue;
|
|
2112
|
+
}
|
|
2113
|
+
if (
|
|
2114
|
+
existing.phase === "rollback" &&
|
|
2115
|
+
!reservation.observedReservations.has(existing)
|
|
2116
|
+
) {
|
|
2117
|
+
throw new Error(
|
|
2118
|
+
`Program at ${address} is finishing initialization rollback cleanup`,
|
|
2119
|
+
);
|
|
2120
|
+
}
|
|
2121
|
+
if (
|
|
2122
|
+
this.canOpenAlongsideReservation(
|
|
2123
|
+
existing,
|
|
2124
|
+
address,
|
|
2125
|
+
candidates,
|
|
2126
|
+
program,
|
|
2127
|
+
)
|
|
2128
|
+
) {
|
|
2129
|
+
continue;
|
|
2130
|
+
}
|
|
2131
|
+
const existingPrograms = existing.programsByAddress.get(address);
|
|
2132
|
+
if (
|
|
2133
|
+
existingPrograms &&
|
|
2134
|
+
![...candidates].some((candidate) =>
|
|
2135
|
+
existingPrograms.has(candidate),
|
|
2136
|
+
)
|
|
2137
|
+
) {
|
|
2138
|
+
throw new Error(
|
|
2139
|
+
`Program at ${address} is already opening as another instance`,
|
|
2140
|
+
);
|
|
2141
|
+
}
|
|
2142
|
+
waits.add(existing.promise);
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
if (waits.size === 0) break;
|
|
2146
|
+
await this.waitForOpeningConflictChange(reservation, waits);
|
|
2147
|
+
}
|
|
2148
|
+
|
|
2149
|
+
for (const [address, candidates] of programsByAddress) {
|
|
2150
|
+
this.assertNoFailedInitialization(address);
|
|
2151
|
+
for (const candidate of candidates) {
|
|
2152
|
+
this.assertNoTerminalOperation(
|
|
2153
|
+
address,
|
|
2154
|
+
candidate,
|
|
2155
|
+
candidate === program,
|
|
2156
|
+
);
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
// No await separates the final conflict check from claiming every address.
|
|
2161
|
+
// Terminal admission reads the same map synchronously, so either the open
|
|
2162
|
+
// generation owns the whole graph or cleanup wins and the checks above fail.
|
|
2163
|
+
reservation.programs = new Set(programs);
|
|
2164
|
+
for (const [address, candidates] of programsByAddress) {
|
|
2165
|
+
for (const existing of this._openingReservationsByAddress.get(address) ??
|
|
2166
|
+
[]) {
|
|
2167
|
+
if (existing === reservation) continue;
|
|
2168
|
+
if (
|
|
2169
|
+
this.canShareOpeningReservation(
|
|
2170
|
+
existing,
|
|
2171
|
+
reservation,
|
|
2172
|
+
address,
|
|
2173
|
+
candidates,
|
|
2174
|
+
parent,
|
|
2175
|
+
)
|
|
2176
|
+
) {
|
|
2177
|
+
this.mergeOpeningReservationGroups(reservation, existing);
|
|
2178
|
+
} else if (
|
|
2179
|
+
this.canOpenAlongsideReservation(
|
|
2180
|
+
existing,
|
|
2181
|
+
address,
|
|
2182
|
+
candidates,
|
|
2183
|
+
program,
|
|
2184
|
+
)
|
|
2185
|
+
) {
|
|
2186
|
+
continue;
|
|
2187
|
+
} else {
|
|
2188
|
+
throw new TerminalOperationNotStartedError(
|
|
2189
|
+
`Program at ${address} became reserved by another open generation`,
|
|
2190
|
+
);
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
reservation.programsByAddress.set(address, new Set(candidates));
|
|
2194
|
+
const reservations =
|
|
2195
|
+
this._openingReservationsByAddress.get(address) ?? new Set();
|
|
2196
|
+
reservations.add(reservation);
|
|
2197
|
+
this._openingReservationsByAddress.set(address, reservations);
|
|
2198
|
+
}
|
|
2199
|
+
for (const [address, candidates] of programsByAddress) {
|
|
2200
|
+
for (const candidate of candidates) {
|
|
2201
|
+
const monitoring = this.monitorTerminalOperations(
|
|
2202
|
+
address,
|
|
2203
|
+
candidate as T,
|
|
2204
|
+
);
|
|
2205
|
+
if (monitoring.newLifecycle) {
|
|
2206
|
+
reservation.provisionalTerminalStates.add(monitoring.state);
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
}
|
|
2210
|
+
return addresses;
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
private trackOpening<S extends T>(
|
|
2214
|
+
address: string,
|
|
2215
|
+
open: (reservation: OpeningReservation<T>) => Promise<S>,
|
|
2216
|
+
observedReservations: Set<OpeningReservation<T>>,
|
|
2217
|
+
): Promise<S> {
|
|
2218
|
+
let resolve!: (program: S) => void;
|
|
2219
|
+
let reject!: (error: unknown) => void;
|
|
2220
|
+
const openPromise = new Promise<S>((promiseResolve, promiseReject) => {
|
|
2221
|
+
resolve = promiseResolve;
|
|
2222
|
+
reject = promiseReject;
|
|
2223
|
+
});
|
|
2224
|
+
const group: OpeningReservationGroup = {
|
|
2225
|
+
participantReferences: new Map(),
|
|
2226
|
+
reservations: new Set(),
|
|
2227
|
+
};
|
|
2228
|
+
const reservation: OpeningReservation<T> = {
|
|
2229
|
+
promise: openPromise as Promise<T>,
|
|
2230
|
+
phase: "opening",
|
|
2231
|
+
observedReservations,
|
|
2232
|
+
adoptedParents: new Set(),
|
|
2233
|
+
contributedParticipants: new Set(),
|
|
2234
|
+
wakeWaiters: new Set(),
|
|
2235
|
+
group,
|
|
2236
|
+
programsByAddress: new Map(),
|
|
2237
|
+
programs: new Set(),
|
|
2238
|
+
provisionalTerminalStates: new Set(),
|
|
2239
|
+
};
|
|
2240
|
+
group.reservations.add(reservation);
|
|
2241
|
+
this._openingReservations.add(reservation);
|
|
2242
|
+
this._openingReservationsByPromise.set(
|
|
2243
|
+
openPromise as Promise<T>,
|
|
2244
|
+
reservation,
|
|
2245
|
+
);
|
|
2246
|
+
this._openingPromises.set(address, openPromise as Promise<T>);
|
|
2247
|
+
let execution: Promise<S>;
|
|
2248
|
+
try {
|
|
2249
|
+
execution = open(reservation);
|
|
2250
|
+
} catch (error) {
|
|
2251
|
+
execution = Promise.reject(error);
|
|
2252
|
+
}
|
|
2253
|
+
void execution.then(
|
|
2254
|
+
(program) => {
|
|
2255
|
+
this.releaseOpeningReservation(reservation);
|
|
2256
|
+
if (this._openingPromises.get(address) === openPromise) {
|
|
2257
|
+
this._openingPromises.delete(address);
|
|
2258
|
+
}
|
|
2259
|
+
resolve(program);
|
|
2260
|
+
},
|
|
2261
|
+
(error) => {
|
|
2262
|
+
this.releaseOpeningReservation(reservation, true);
|
|
2263
|
+
if (this._openingPromises.get(address) === openPromise) {
|
|
2264
|
+
this._openingPromises.delete(address);
|
|
2265
|
+
}
|
|
2266
|
+
reject(error);
|
|
2267
|
+
},
|
|
2268
|
+
);
|
|
2269
|
+
return openPromise;
|
|
2270
|
+
}
|
|
2271
|
+
|
|
152
2272
|
async open<S extends T>(
|
|
153
2273
|
storeOrAddress: S | Address | string,
|
|
154
2274
|
options: OpenOptions<S> = {},
|
|
155
2275
|
): Promise<S> {
|
|
156
|
-
|
|
2276
|
+
if (!this._acceptingOpens) {
|
|
2277
|
+
throw new Error("Program handler is stopping or stopped");
|
|
2278
|
+
}
|
|
2279
|
+
const attachmentParent =
|
|
2280
|
+
options.parent && options.parent !== storeOrAddress
|
|
2281
|
+
? options.parent
|
|
2282
|
+
: undefined;
|
|
2283
|
+
if (attachmentParent) {
|
|
2284
|
+
this.assertParentCanOwnOpen(attachmentParent);
|
|
2285
|
+
this.retainParentAttachment(attachmentParent);
|
|
2286
|
+
}
|
|
2287
|
+
const observedReservations = new Set(
|
|
2288
|
+
[...this._openingReservations].filter(
|
|
2289
|
+
(reservation) => reservation.phase === "opening",
|
|
2290
|
+
),
|
|
2291
|
+
);
|
|
2292
|
+
let completeAdmission!: () => void;
|
|
2293
|
+
const admission = new Promise<void>((resolve) => {
|
|
2294
|
+
completeAdmission = resolve;
|
|
2295
|
+
});
|
|
2296
|
+
this._openAdmissions.add(admission);
|
|
2297
|
+
try {
|
|
2298
|
+
return await this.openAdmitted(
|
|
2299
|
+
storeOrAddress,
|
|
2300
|
+
options,
|
|
2301
|
+
observedReservations,
|
|
2302
|
+
);
|
|
2303
|
+
} finally {
|
|
2304
|
+
if (attachmentParent) {
|
|
2305
|
+
this.releaseParentAttachment(attachmentParent);
|
|
2306
|
+
}
|
|
2307
|
+
this._openAdmissions.delete(admission);
|
|
2308
|
+
completeAdmission();
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
|
|
2312
|
+
private async openAdmitted<S extends T>(
|
|
2313
|
+
storeOrAddress: S | Address | string,
|
|
2314
|
+
options: OpenOptions<S> = {},
|
|
2315
|
+
observedReservations: Set<OpeningReservation<T>> = new Set(),
|
|
2316
|
+
): Promise<S> {
|
|
2317
|
+
if (
|
|
2318
|
+
typeof storeOrAddress !== "string" &&
|
|
2319
|
+
this._cleanupResiduals.has(storeOrAddress)
|
|
2320
|
+
) {
|
|
2321
|
+
throw new Error(
|
|
2322
|
+
`Program at ${storeOrAddress.address} has pending cleanup residuals and cannot be reopened before cleanup is retried`,
|
|
2323
|
+
);
|
|
2324
|
+
}
|
|
2325
|
+
if (options.parent && options.parent !== storeOrAddress) {
|
|
2326
|
+
this.assertParentCanOwnOpen(options.parent);
|
|
2327
|
+
}
|
|
2328
|
+
// Parent opens historically share subprograms by default. Explicit parent
|
|
2329
|
+
// strategies still have to retain their reject/replace semantics.
|
|
2330
|
+
const mergeStrategy: ProgramMergeStrategy | undefined = options.parent
|
|
2331
|
+
? (options.existing ?? "reuse")
|
|
2332
|
+
: options.existing;
|
|
2333
|
+
const fn = async (
|
|
2334
|
+
openingReservation?: OpeningReservation<T>,
|
|
2335
|
+
): Promise<S> => {
|
|
157
2336
|
// TODO add locks for store lifecycle, e.g. what happens if we try to open and close a store at the same time?
|
|
158
2337
|
let program = storeOrAddress as S;
|
|
159
2338
|
if (typeof storeOrAddress === "string") {
|
|
160
2339
|
const address = storeOrAddress.toString();
|
|
161
2340
|
try {
|
|
162
|
-
|
|
2341
|
+
this.assertNoFailedInitialization(address);
|
|
2342
|
+
this.assertNoTerminalOperation(address);
|
|
2343
|
+
const existing = this.findMonitoredProgram(address);
|
|
163
2344
|
if (existing) {
|
|
164
2345
|
// Be defensive: stale handles shouldn't be returned from the cache.
|
|
165
2346
|
if (existing.closed) {
|
|
166
|
-
this.items.
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
} else if (
|
|
170
|
-
|
|
2347
|
+
if (this.items.get(address) === existing) {
|
|
2348
|
+
this.items.delete(address);
|
|
2349
|
+
}
|
|
2350
|
+
} else if (mergeStrategy === "reuse") {
|
|
2351
|
+
return this.attachExistingProgram(
|
|
2352
|
+
address,
|
|
2353
|
+
existing,
|
|
2354
|
+
options.parent,
|
|
2355
|
+
) as S;
|
|
2356
|
+
} else if (mergeStrategy === "replace") {
|
|
2357
|
+
await this.closeForReplacement(address, existing, options.parent);
|
|
171
2358
|
} else {
|
|
172
2359
|
throw new Error(`Program at ${address} is already open`);
|
|
173
2360
|
}
|
|
@@ -201,16 +2388,22 @@ export class Handler<T extends Manageable<any>> {
|
|
|
201
2388
|
}
|
|
202
2389
|
|
|
203
2390
|
if (!program.closed) {
|
|
204
|
-
|
|
2391
|
+
this.assertNoFailedInitialization(program.address);
|
|
2392
|
+
this.assertNoTerminalOperation(program.address, program);
|
|
2393
|
+
const existing = this.findMonitoredProgram(program.address);
|
|
205
2394
|
if (existing === program) {
|
|
206
|
-
|
|
207
|
-
|
|
2395
|
+
return this.attachExistingProgram(
|
|
2396
|
+
program.address,
|
|
2397
|
+
existing,
|
|
2398
|
+
options.parent,
|
|
2399
|
+
) as S;
|
|
208
2400
|
} else if (existing) {
|
|
209
2401
|
// we got existing, but it is not the same instance
|
|
210
2402
|
const existing = await this.checkProcessExisting(
|
|
211
2403
|
program.address,
|
|
212
2404
|
program,
|
|
213
|
-
|
|
2405
|
+
mergeStrategy,
|
|
2406
|
+
options.parent,
|
|
214
2407
|
);
|
|
215
2408
|
|
|
216
2409
|
if (existing) {
|
|
@@ -226,16 +2419,31 @@ export class Handler<T extends Manageable<any>> {
|
|
|
226
2419
|
`Program at ${program.address} is already opened with a different client`,
|
|
227
2420
|
);
|
|
228
2421
|
}
|
|
2422
|
+
assertCanAttachParent(program);
|
|
2423
|
+
if (!openingReservation) {
|
|
2424
|
+
throw new Error("Missing opening graph reservation");
|
|
2425
|
+
}
|
|
2426
|
+
resolvedDependencyAddresses = await this.prepareOpeningGraph(
|
|
2427
|
+
program,
|
|
2428
|
+
openingReservation,
|
|
2429
|
+
options.parent,
|
|
2430
|
+
resolvedDependencyAddresses,
|
|
2431
|
+
);
|
|
229
2432
|
|
|
230
2433
|
// assume new instance was not added to monitored items, just add it
|
|
231
2434
|
// and return it as we would opened it normally
|
|
232
2435
|
|
|
233
2436
|
await program.save(this.properties.client.services.blocks, {
|
|
234
2437
|
skipOnAddress: false,
|
|
235
|
-
|
|
236
|
-
return !this.
|
|
2438
|
+
save: (address) => {
|
|
2439
|
+
return !this.findMonitoredProgram(address, true);
|
|
237
2440
|
},
|
|
238
2441
|
});
|
|
2442
|
+
assertCanAttachParent(program);
|
|
2443
|
+
if (options.parent) {
|
|
2444
|
+
this.assertParentCanOwnOpen(options.parent);
|
|
2445
|
+
}
|
|
2446
|
+
this.monitorTerminalOperations(program.address, program);
|
|
239
2447
|
this.items.set(program.address, program);
|
|
240
2448
|
addParent(program, options.parent);
|
|
241
2449
|
return program;
|
|
@@ -243,93 +2451,241 @@ export class Handler<T extends Manageable<any>> {
|
|
|
243
2451
|
}
|
|
244
2452
|
}
|
|
245
2453
|
|
|
2454
|
+
const existingBeforeSave = await this.checkProcessExisting(
|
|
2455
|
+
program.address,
|
|
2456
|
+
program,
|
|
2457
|
+
mergeStrategy,
|
|
2458
|
+
options.parent,
|
|
2459
|
+
);
|
|
2460
|
+
if (existingBeforeSave) {
|
|
2461
|
+
return existingBeforeSave as S;
|
|
2462
|
+
}
|
|
2463
|
+
if (!openingReservation) {
|
|
2464
|
+
throw new Error("Missing opening graph reservation");
|
|
2465
|
+
}
|
|
2466
|
+
resolvedDependencyAddresses = await this.prepareOpeningGraph(
|
|
2467
|
+
program,
|
|
2468
|
+
openingReservation,
|
|
2469
|
+
options.parent,
|
|
2470
|
+
resolvedDependencyAddresses,
|
|
2471
|
+
);
|
|
2472
|
+
|
|
246
2473
|
logger.trace(`Open database '${program.constructor.name}`);
|
|
247
2474
|
|
|
248
2475
|
// todo make this nicer
|
|
249
2476
|
let address = await program.save(this.properties.client.services.blocks, {
|
|
250
|
-
skipOnAddress:
|
|
251
|
-
|
|
252
|
-
return !this.
|
|
2477
|
+
skipOnAddress: !resolvedWithoutSaving,
|
|
2478
|
+
save: (address) => {
|
|
2479
|
+
return !this.findMonitoredProgram(address, true);
|
|
253
2480
|
},
|
|
254
2481
|
});
|
|
255
2482
|
|
|
256
2483
|
const existing = await this.checkProcessExisting(
|
|
257
2484
|
address,
|
|
258
2485
|
program,
|
|
259
|
-
|
|
2486
|
+
mergeStrategy,
|
|
2487
|
+
options.parent,
|
|
260
2488
|
);
|
|
261
2489
|
if (existing) {
|
|
262
2490
|
return existing as S;
|
|
263
2491
|
}
|
|
264
2492
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
2493
|
+
const rollbackState = {
|
|
2494
|
+
parents: program.parents ? [...program.parents] : undefined,
|
|
2495
|
+
children: program.children ? [...program.children] : undefined,
|
|
2496
|
+
parent: options.parent,
|
|
2497
|
+
parentChildReferences:
|
|
2498
|
+
options.parent?.children?.filter((child) => child === program)
|
|
2499
|
+
.length ?? 0,
|
|
2500
|
+
};
|
|
2501
|
+
this.monitorTerminalOperations(address, program);
|
|
2502
|
+
try {
|
|
2503
|
+
const {
|
|
2504
|
+
onBeforeOpen: userOnBeforeOpen,
|
|
2505
|
+
onOpen: userOnOpen,
|
|
2506
|
+
onClose: userOnClose,
|
|
2507
|
+
onDrop: userOnDrop,
|
|
2508
|
+
...programOptions
|
|
2509
|
+
} = options;
|
|
2510
|
+
if (options.parent) {
|
|
2511
|
+
this.assertParentCanOwnOpen(options.parent);
|
|
2512
|
+
}
|
|
2513
|
+
await this.invokeLifecycleMethod(() =>
|
|
2514
|
+
program.beforeOpen(this.properties.client, {
|
|
2515
|
+
...programOptions,
|
|
2516
|
+
onBeforeOpen: async (p) => {
|
|
2517
|
+
if (this.properties.shouldMonitor(program)) {
|
|
2518
|
+
this.items.set(address, program);
|
|
2519
|
+
}
|
|
2520
|
+
if (userOnBeforeOpen) {
|
|
2521
|
+
this._openLifecycleCallbacks += 1;
|
|
2522
|
+
try {
|
|
2523
|
+
await userOnBeforeOpen(p);
|
|
2524
|
+
} finally {
|
|
2525
|
+
this._openLifecycleCallbacks -= 1;
|
|
2526
|
+
}
|
|
2527
|
+
}
|
|
2528
|
+
},
|
|
2529
|
+
onOpen: async (p) => {
|
|
2530
|
+
if (!userOnOpen) return;
|
|
2531
|
+
this._openLifecycleCallbacks += 1;
|
|
2532
|
+
try {
|
|
2533
|
+
await userOnOpen(p);
|
|
2534
|
+
} finally {
|
|
2535
|
+
this._openLifecycleCallbacks -= 1;
|
|
2536
|
+
}
|
|
2537
|
+
},
|
|
2538
|
+
onClose: async (p) => {
|
|
2539
|
+
if (this.properties.shouldMonitor(p)) {
|
|
2540
|
+
this._onProgamClose(p as T); // TODO types
|
|
2541
|
+
}
|
|
2542
|
+
if (userOnClose) {
|
|
2543
|
+
this._terminalLifecycleCallbacks += 1;
|
|
2544
|
+
try {
|
|
2545
|
+
await userOnClose(p);
|
|
2546
|
+
} finally {
|
|
2547
|
+
this._terminalLifecycleCallbacks -= 1;
|
|
2548
|
+
}
|
|
2549
|
+
}
|
|
2550
|
+
},
|
|
2551
|
+
onDrop: async (p) => {
|
|
2552
|
+
if (this.properties.shouldMonitor(p)) {
|
|
2553
|
+
this._onProgamClose(p);
|
|
2554
|
+
}
|
|
2555
|
+
if (userOnDrop) {
|
|
2556
|
+
this._terminalLifecycleCallbacks += 1;
|
|
2557
|
+
try {
|
|
2558
|
+
await userOnDrop(p);
|
|
2559
|
+
} finally {
|
|
2560
|
+
this._terminalLifecycleCallbacks -= 1;
|
|
2561
|
+
}
|
|
2562
|
+
}
|
|
2563
|
+
},
|
|
2564
|
+
// If the program opens more programs
|
|
2565
|
+
// reset: options.reset,
|
|
2566
|
+
}),
|
|
2567
|
+
);
|
|
2568
|
+
await this.invokeLifecycleMethod(() => program.open(options.args));
|
|
2569
|
+
await this.invokeLifecycleMethod(() => program.afterOpen());
|
|
2570
|
+
return program as S;
|
|
2571
|
+
} catch (error) {
|
|
2572
|
+
try {
|
|
2573
|
+
if (openingReservation) {
|
|
2574
|
+
openingReservation.phase = "rollback";
|
|
269
2575
|
}
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
2576
|
+
const rollbackPrograms = new Set<Manageable<any>>([
|
|
2577
|
+
program,
|
|
2578
|
+
...(resolvedDependencyAddresses?.keys() ?? []),
|
|
2579
|
+
]);
|
|
2580
|
+
for (const rollbackProgram of rollbackPrograms) {
|
|
2581
|
+
this._initializationRollbacks.add(rollbackProgram);
|
|
274
2582
|
}
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
2583
|
+
try {
|
|
2584
|
+
await this.rollbackFailedInitialization(
|
|
2585
|
+
address,
|
|
2586
|
+
program,
|
|
2587
|
+
rollbackState,
|
|
2588
|
+
);
|
|
2589
|
+
} finally {
|
|
2590
|
+
for (const rollbackProgram of rollbackPrograms) {
|
|
2591
|
+
this._initializationRollbacks.delete(rollbackProgram);
|
|
2592
|
+
}
|
|
279
2593
|
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
return program as S;
|
|
2594
|
+
} catch (cleanupError) {
|
|
2595
|
+
logger.error(
|
|
2596
|
+
`Failed to roll back program at ${address}: ${String(cleanupError)}`,
|
|
2597
|
+
);
|
|
2598
|
+
}
|
|
2599
|
+
throw error;
|
|
2600
|
+
}
|
|
288
2601
|
};
|
|
289
2602
|
|
|
290
2603
|
// Helper to resolve address from storeOrAddress
|
|
2604
|
+
let resolvedWithoutSaving = false;
|
|
2605
|
+
let resolvedDependencyAddresses: Map<Manageable<any>, string> | undefined;
|
|
291
2606
|
const resolveAddress = async (): Promise<string> => {
|
|
292
2607
|
if (typeof storeOrAddress === "string") {
|
|
293
2608
|
return storeOrAddress;
|
|
294
2609
|
}
|
|
295
|
-
|
|
296
|
-
|
|
2610
|
+
resolvedWithoutSaving = storeOrAddress.closed;
|
|
2611
|
+
resolvedDependencyAddresses =
|
|
2612
|
+
await this.resolveDependencyAddresses(storeOrAddress);
|
|
2613
|
+
const address = resolvedDependencyAddresses.get(storeOrAddress);
|
|
2614
|
+
if (!address) {
|
|
2615
|
+
throw new Error("Failed to resolve the root program address");
|
|
297
2616
|
}
|
|
298
|
-
return
|
|
299
|
-
skipOnAddress: true,
|
|
300
|
-
condition: (addr) => !this.items.has(addr.toString()),
|
|
301
|
-
});
|
|
2617
|
+
return address;
|
|
302
2618
|
};
|
|
303
2619
|
|
|
304
2620
|
const address = await resolveAddress();
|
|
2621
|
+
await this.waitForOpeningReservations(
|
|
2622
|
+
address,
|
|
2623
|
+
typeof storeOrAddress === "string" ? undefined : storeOrAddress,
|
|
2624
|
+
options.parent,
|
|
2625
|
+
observedReservations,
|
|
2626
|
+
);
|
|
2627
|
+
if (!this._openingPromises.has(address)) {
|
|
2628
|
+
this.assertNoFailedInitialization(address);
|
|
2629
|
+
this.assertNoTerminalOperation(
|
|
2630
|
+
address,
|
|
2631
|
+
typeof storeOrAddress === "string" ? undefined : storeOrAddress,
|
|
2632
|
+
);
|
|
2633
|
+
}
|
|
305
2634
|
|
|
306
2635
|
// For parent opens, check if already opened or in-progress and return early
|
|
307
2636
|
// This prevents race conditions while avoiding deadlocks
|
|
308
2637
|
if (options?.parent) {
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
2638
|
+
while (true) {
|
|
2639
|
+
// Check if there's an open in progress FIRST - wait for it. This
|
|
2640
|
+
// must precede items because beforeOpen() adds to items before open()
|
|
2641
|
+
// completes.
|
|
2642
|
+
const existingPromise = this._openingPromises.get(address);
|
|
2643
|
+
if (existingPromise) {
|
|
2644
|
+
const requestedProgram =
|
|
2645
|
+
typeof storeOrAddress === "string" ? undefined : storeOrAddress;
|
|
2646
|
+
this.adoptOpeningReservation(existingPromise, options.parent);
|
|
2647
|
+
if (
|
|
2648
|
+
this.isReservedNestedUse(address, requestedProgram, options.parent)
|
|
2649
|
+
) {
|
|
2650
|
+
const existing = await this.processParentExisting<S>(
|
|
2651
|
+
address,
|
|
2652
|
+
requestedProgram,
|
|
2653
|
+
mergeStrategy ?? "reuse",
|
|
2654
|
+
options.parent,
|
|
2655
|
+
);
|
|
2656
|
+
if (existing) return existing;
|
|
2657
|
+
}
|
|
2658
|
+
try {
|
|
2659
|
+
await existingPromise;
|
|
2660
|
+
} catch {
|
|
2661
|
+
// The failed generation has completed its identity-safe rollback.
|
|
2662
|
+
// This admitted request still owns its independent open semantics.
|
|
2663
|
+
}
|
|
2664
|
+
continue;
|
|
2665
|
+
}
|
|
325
2666
|
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
2667
|
+
const existing = await this.processParentExisting<S>(
|
|
2668
|
+
address,
|
|
2669
|
+
typeof storeOrAddress === "string" ? undefined : storeOrAddress,
|
|
2670
|
+
mergeStrategy ?? "reuse",
|
|
2671
|
+
options.parent,
|
|
2672
|
+
);
|
|
2673
|
+
if (existing) {
|
|
2674
|
+
return existing;
|
|
2675
|
+
}
|
|
2676
|
+
|
|
2677
|
+
// processParentExisting() can await replacement and even its empty
|
|
2678
|
+
// async path yields once. Re-read both gates after that yield. With no
|
|
2679
|
+
// await between this final check and trackOpening(), registering a new
|
|
2680
|
+
// parent generation is an atomic singleflight claim on this JS turn.
|
|
2681
|
+
if (this._openingPromises.has(address) || this.items.has(address)) {
|
|
2682
|
+
continue;
|
|
2683
|
+
}
|
|
2684
|
+
return this.trackOpening(
|
|
2685
|
+
address,
|
|
2686
|
+
(reservation) => fn(reservation),
|
|
2687
|
+
observedReservations,
|
|
2688
|
+
);
|
|
333
2689
|
}
|
|
334
2690
|
}
|
|
335
2691
|
|
|
@@ -339,6 +2695,49 @@ export class Handler<T extends Manageable<any>> {
|
|
|
339
2695
|
queue = new PQueue({ concurrency: 1 });
|
|
340
2696
|
this._openQueue.set(address, queue);
|
|
341
2697
|
}
|
|
342
|
-
return queue.add(
|
|
2698
|
+
return queue.add(async () => {
|
|
2699
|
+
while (true) {
|
|
2700
|
+
// Parent opens bypass the queue to avoid nested-open deadlocks. Wait for
|
|
2701
|
+
// such a generation here before applying this root open's own
|
|
2702
|
+
// reject/reuse/replace semantics. Loop so a newer parent generation that
|
|
2703
|
+
// claimed the address while this waiter resumed is also observed.
|
|
2704
|
+
const existingPromise = this._openingPromises.get(address);
|
|
2705
|
+
if (existingPromise) {
|
|
2706
|
+
try {
|
|
2707
|
+
await existingPromise;
|
|
2708
|
+
} catch {
|
|
2709
|
+
// Re-read Handler state and apply this root request's own strategy
|
|
2710
|
+
// after the failed generation has rolled back.
|
|
2711
|
+
}
|
|
2712
|
+
continue;
|
|
2713
|
+
}
|
|
2714
|
+
|
|
2715
|
+
// Existing live non-replacement operations finish from Handler state
|
|
2716
|
+
// and do not represent a new initialization generation. In particular,
|
|
2717
|
+
// don't make a concurrent parent inherit a duplicate-open rejection. A
|
|
2718
|
+
// stale closed cache entry, however, is evicted before fn() starts the
|
|
2719
|
+
// new generation visible to parent opens.
|
|
2720
|
+
const existing = this.findMonitoredProgram(address);
|
|
2721
|
+
if (existing?.closed) {
|
|
2722
|
+
this.assertNoTerminalOperation(
|
|
2723
|
+
address,
|
|
2724
|
+
typeof storeOrAddress === "string" ? undefined : storeOrAddress,
|
|
2725
|
+
);
|
|
2726
|
+
if (this.items.get(address) === existing) {
|
|
2727
|
+
this.items.delete(address);
|
|
2728
|
+
}
|
|
2729
|
+
} else if (existing && mergeStrategy !== "replace") {
|
|
2730
|
+
return fn();
|
|
2731
|
+
}
|
|
2732
|
+
|
|
2733
|
+
// No await separates the empty gate from registration, so a parent
|
|
2734
|
+
// open cannot claim this address in between.
|
|
2735
|
+
return this.trackOpening(
|
|
2736
|
+
address,
|
|
2737
|
+
(reservation) => fn(reservation),
|
|
2738
|
+
observedReservations,
|
|
2739
|
+
);
|
|
2740
|
+
}
|
|
2741
|
+
}) as any as S;
|
|
343
2742
|
}
|
|
344
2743
|
}
|