@spine-event-engine/deployment 2.0.0-snapshot.23 → 2.0.0-snapshot.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/internal/backend-membership-kernel.d.ts +34 -4
- package/dist/internal/backend-membership-kernel.d.ts.map +1 -1
- package/dist/internal/backend-membership-kernel.js +515 -86
- package/dist/internal/backend-membership-kernel.js.map +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +5 -5
|
@@ -11,10 +11,16 @@
|
|
|
11
11
|
* or implied. See the License for the specific language governing permissions and limitations under
|
|
12
12
|
* the License.
|
|
13
13
|
*/
|
|
14
|
+
var _a;
|
|
14
15
|
const childCleanupTimeoutMs = 1_000;
|
|
15
16
|
/**
|
|
16
17
|
* Manages ephemeral backend membership and its native subscription children.
|
|
17
|
-
* Logical-definition persistence
|
|
18
|
+
* Logical-definition persistence remains with callers.
|
|
19
|
+
*
|
|
20
|
+
* @typeParam Member A discovered backend member.
|
|
21
|
+
* @typeParam Request The forwarded unary request.
|
|
22
|
+
* @typeParam Child The native child subscription handle.
|
|
23
|
+
* @typeParam Update The native child update.
|
|
18
24
|
*/
|
|
19
25
|
export class BackendMembershipKernel {
|
|
20
26
|
#options;
|
|
@@ -78,26 +84,15 @@ export class BackendMembershipKernel {
|
|
|
78
84
|
throw new Error("subscription ID is required");
|
|
79
85
|
if (signal.aborted || this.#closed || this.#nodes.length === 0)
|
|
80
86
|
throw new Error("backend membership is unavailable.");
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
definition: definition.slice(),
|
|
84
|
-
maxChildBytes,
|
|
85
|
-
children: new Map(),
|
|
86
|
-
starts: new Set(),
|
|
87
|
-
active: false,
|
|
88
|
-
activationController: undefined,
|
|
89
|
-
updates: undefined,
|
|
90
|
-
failure: undefined,
|
|
91
|
-
});
|
|
92
|
-
const state = this.#definitions.get(key);
|
|
93
|
-
if (state === undefined)
|
|
94
|
-
throw new Error("subscription creation was cancelled");
|
|
87
|
+
const required = new Set(this.#nodes.map((member) => this.#options.memberKey(member)));
|
|
88
|
+
const state = this.#retainDefinition(key, definition, maxChildBytes);
|
|
95
89
|
await this.#schedule(this.#nodes, false);
|
|
96
90
|
// Re-evaluate cancellation after reconciliation can yield to close().
|
|
97
91
|
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
|
|
98
92
|
if (signal.aborted || this.#isClosed() || !this.#definitions.has(key))
|
|
99
93
|
throw new Error("subscription creation was cancelled");
|
|
100
|
-
if (state.failure !== undefined ||
|
|
94
|
+
if (state.failure !== undefined ||
|
|
95
|
+
[...required].some((memberKey) => !state.children.has(memberKey))) {
|
|
101
96
|
await this.#removeDefinition(key, new AbortController().signal);
|
|
102
97
|
throw state.failure ?? new Error("backend membership is unavailable.");
|
|
103
98
|
}
|
|
@@ -117,8 +112,24 @@ export class BackendMembershipKernel {
|
|
|
117
112
|
const key = this.#options.definitionKey(definition);
|
|
118
113
|
if (key === undefined || key.length === 0)
|
|
119
114
|
throw new Error("subscription ID is required");
|
|
120
|
-
|
|
121
|
-
|
|
115
|
+
this.#retainDefinition(key, definition, maxChildBytes);
|
|
116
|
+
await this.#schedule(this.#nodes, false);
|
|
117
|
+
const failure = this.#definitions.get(key)?.failure;
|
|
118
|
+
if (failure !== undefined)
|
|
119
|
+
throw failure;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Retains one logical definition and its child lifecycle state.
|
|
123
|
+
*
|
|
124
|
+
* @param key The logical subscription identifier.
|
|
125
|
+
* @param definition The public subscription definition.
|
|
126
|
+
* @param maxChildBytes The maximum native child envelope size.
|
|
127
|
+
* @returns The retained definition state.
|
|
128
|
+
*/
|
|
129
|
+
#retainDefinition(key, definition, maxChildBytes) {
|
|
130
|
+
let state = this.#definitions.get(key);
|
|
131
|
+
if (state === undefined) {
|
|
132
|
+
state = {
|
|
122
133
|
definition: definition.slice(),
|
|
123
134
|
maxChildBytes,
|
|
124
135
|
children: new Map(),
|
|
@@ -127,19 +138,21 @@ export class BackendMembershipKernel {
|
|
|
127
138
|
activationController: undefined,
|
|
128
139
|
updates: undefined,
|
|
129
140
|
failure: undefined,
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
141
|
+
readiness: undefined,
|
|
142
|
+
lifetime: undefined,
|
|
143
|
+
};
|
|
144
|
+
this.#definitions.set(key, state);
|
|
145
|
+
}
|
|
146
|
+
return state;
|
|
135
147
|
}
|
|
136
148
|
/**
|
|
137
|
-
*
|
|
149
|
+
* Activates a retained logical definition until cancellation or terminal child failure.
|
|
138
150
|
*
|
|
139
151
|
* @param definition Supplies the retained logical definition.
|
|
140
152
|
* @param updates Receives relayed child updates.
|
|
141
153
|
* @param signal Cancels update relay.
|
|
142
|
-
* @returns
|
|
154
|
+
* @returns Normal completion after cancellation; rejects with a current child failure, possibly
|
|
155
|
+
* wrapped with a bounded cleanup failure.
|
|
143
156
|
*/
|
|
144
157
|
async activate(definition, updates, signal) {
|
|
145
158
|
const key = this.#options.definitionKey(definition);
|
|
@@ -149,38 +162,184 @@ export class BackendMembershipKernel {
|
|
|
149
162
|
if (state.activationController !== undefined)
|
|
150
163
|
throw new Error("subscription activation is already active");
|
|
151
164
|
const controller = new AbortController();
|
|
152
|
-
|
|
153
|
-
state.active = true;
|
|
154
|
-
state.updates = updates;
|
|
165
|
+
const { readiness, lifetime } = this.#beginActivation(state, updates, controller);
|
|
155
166
|
const abort = () => {
|
|
156
|
-
controller
|
|
157
|
-
for (const start of state.starts)
|
|
158
|
-
start.abort();
|
|
159
|
-
for (const child of state.children.values())
|
|
160
|
-
child.activationController?.abort();
|
|
167
|
+
this.#abortActivation(state, readiness, controller);
|
|
161
168
|
};
|
|
162
169
|
if (signal.aborted) {
|
|
163
170
|
abort();
|
|
164
|
-
state
|
|
165
|
-
state.active = false;
|
|
171
|
+
this.#finishActivation(state, controller);
|
|
166
172
|
return;
|
|
167
173
|
}
|
|
168
174
|
signal.addEventListener("abort", abort, { once: true });
|
|
169
175
|
try {
|
|
170
|
-
await this.#
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
]);
|
|
176
|
+
await this.#waitActivation(state, readiness, lifetime, signal, controller);
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
await this.#failActivation(definition, readiness, signal, error);
|
|
175
180
|
}
|
|
176
181
|
finally {
|
|
177
182
|
signal.removeEventListener("abort", abort);
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
183
|
+
this.#finishActivation(state, controller);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Captures one logical activation before any child relay starts.
|
|
188
|
+
* @param state The retained logical definition.
|
|
189
|
+
* @param updates Receives relayed child updates.
|
|
190
|
+
* @param controller Cancels this activation.
|
|
191
|
+
* @returns The readiness and terminal-failure gates for this activation.
|
|
192
|
+
*/
|
|
193
|
+
#beginActivation(state, updates, controller) {
|
|
194
|
+
state.activationController = controller;
|
|
195
|
+
state.active = true;
|
|
196
|
+
state.updates = updates;
|
|
197
|
+
const readiness = this.#newReadiness();
|
|
198
|
+
state.readiness = readiness;
|
|
199
|
+
const lifetime = this.#newLifetime();
|
|
200
|
+
state.lifetime = lifetime;
|
|
201
|
+
return { readiness, lifetime };
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Clears a logical relay only when its activation generation is still current.
|
|
205
|
+
*
|
|
206
|
+
* @param state The retained definition.
|
|
207
|
+
* @param controller The completed activation controller.
|
|
208
|
+
*/
|
|
209
|
+
#finishActivation(state, controller) {
|
|
210
|
+
if (state.activationController !== controller)
|
|
211
|
+
return;
|
|
212
|
+
state.activationController = undefined;
|
|
213
|
+
state.active = false;
|
|
214
|
+
state.updates = undefined;
|
|
215
|
+
state.readiness = undefined;
|
|
216
|
+
state.lifetime = undefined;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Creates the acknowledgement gate for a strict child activation.
|
|
220
|
+
*
|
|
221
|
+
* @returns The gate, or undefined for a generic non-subscription kernel.
|
|
222
|
+
*/
|
|
223
|
+
#newReadiness() {
|
|
224
|
+
if (this.#options.isActivationAck === undefined)
|
|
225
|
+
return undefined;
|
|
226
|
+
const readiness = {
|
|
227
|
+
gate: Promise.withResolvers(),
|
|
228
|
+
seen: new Set(),
|
|
229
|
+
expected: undefined,
|
|
230
|
+
first: undefined,
|
|
231
|
+
emitting: false,
|
|
232
|
+
reconciling: false,
|
|
233
|
+
settled: false,
|
|
234
|
+
failure: undefined,
|
|
235
|
+
};
|
|
236
|
+
// spine-log-boundary: deployment.membership_readiness_rejection_observer
|
|
237
|
+
void readiness.gate.promise.catch(() => undefined);
|
|
238
|
+
return readiness;
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Creates an observed terminal signal for one logical activation.
|
|
242
|
+
* @returns The unsettled activation lifetime.
|
|
243
|
+
*/
|
|
244
|
+
#newLifetime() {
|
|
245
|
+
const lifetime = {
|
|
246
|
+
gate: Promise.withResolvers(),
|
|
247
|
+
failure: undefined,
|
|
248
|
+
};
|
|
249
|
+
// spine-log-boundary: deployment.membership_lifetime_rejection_observer
|
|
250
|
+
void lifetime.gate.promise.catch(() => undefined);
|
|
251
|
+
return lifetime;
|
|
252
|
+
}
|
|
253
|
+
/**
|
|
254
|
+
* Cancels a logical activation and every child attempt.
|
|
255
|
+
*
|
|
256
|
+
* @param state The retained definition.
|
|
257
|
+
* @param readiness Its acknowledgement gate when configured.
|
|
258
|
+
* @param controller The logical activation controller.
|
|
259
|
+
*/
|
|
260
|
+
#abortActivation(state, readiness, controller) {
|
|
261
|
+
controller.abort();
|
|
262
|
+
if (readiness !== undefined && !readiness.settled) {
|
|
263
|
+
readiness.settled = true;
|
|
264
|
+
readiness.gate.resolve(undefined);
|
|
265
|
+
}
|
|
266
|
+
for (const start of state.starts)
|
|
267
|
+
start.abort();
|
|
268
|
+
for (const child of state.children.values())
|
|
269
|
+
child.activationController?.abort();
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Waits for the required child acknowledgements and later cancellation.
|
|
273
|
+
*
|
|
274
|
+
* @param state The retained definition.
|
|
275
|
+
* @param readiness Its acknowledgement gate when configured.
|
|
276
|
+
* @param signal The public stream cancellation signal.
|
|
277
|
+
* @param controller The logical activation controller.
|
|
278
|
+
* @returns Completion after activation cancellation or readiness failure.
|
|
279
|
+
*/
|
|
280
|
+
async #waitActivation(state, readiness, lifetime, signal, controller) {
|
|
281
|
+
await Promise.race([
|
|
282
|
+
this.#schedule(this.#nodes, false),
|
|
283
|
+
_a.waitForAbort(signal),
|
|
284
|
+
_a.waitForAbort(controller.signal),
|
|
285
|
+
lifetime.gate.promise,
|
|
286
|
+
]);
|
|
287
|
+
this.#throwLifetimeFailure(lifetime);
|
|
288
|
+
if (signal.aborted || controller.signal.aborted)
|
|
289
|
+
return;
|
|
290
|
+
if (readiness !== undefined)
|
|
291
|
+
await Promise.race([this.#awaitReadiness(state, readiness, signal), lifetime.gate.promise]);
|
|
292
|
+
this.#throwLifetimeFailure(lifetime);
|
|
293
|
+
await Promise.race([
|
|
294
|
+
_a.waitForAbort(signal),
|
|
295
|
+
_a.waitForAbort(controller.signal),
|
|
296
|
+
lifetime.gate.promise,
|
|
297
|
+
]);
|
|
298
|
+
this.#throwLifetimeFailure(lifetime);
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Preserves a child failure when cancellation and failure settle together.
|
|
302
|
+
* @param lifetime The current logical activation lifetime.
|
|
303
|
+
*/
|
|
304
|
+
#throwLifetimeFailure(lifetime) {
|
|
305
|
+
if (lifetime.failure !== undefined)
|
|
306
|
+
throw lifetime.failure;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Removes failed strict activation state while preserving the original cause.
|
|
310
|
+
*
|
|
311
|
+
* @param definition The logical subscription definition.
|
|
312
|
+
* @param readiness Its acknowledgement gate when configured.
|
|
313
|
+
* @param signal The public stream cancellation signal.
|
|
314
|
+
* @param error The activation failure.
|
|
315
|
+
*/
|
|
316
|
+
async #failActivation(definition, readiness, signal, error) {
|
|
317
|
+
// The signal can abort while scheduling or waiting for child readiness.
|
|
318
|
+
if (!signal.aborted) {
|
|
319
|
+
try {
|
|
320
|
+
await this.cancel(definition, new AbortController().signal);
|
|
321
|
+
}
|
|
322
|
+
catch (cleanupError) {
|
|
323
|
+
throw new AggregateError([error, cleanupError], "Child activation cleanup failed.");
|
|
182
324
|
}
|
|
183
325
|
}
|
|
326
|
+
throw error;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Sets the activation's required child snapshot and waits for its single logical ACK.
|
|
330
|
+
*
|
|
331
|
+
* @param state The active logical definition.
|
|
332
|
+
* @param readiness The current activation's child acknowledgements.
|
|
333
|
+
* @param signal The public stream cancellation signal.
|
|
334
|
+
*/
|
|
335
|
+
async #awaitReadiness(state, readiness, signal) {
|
|
336
|
+
if (state.failure !== undefined)
|
|
337
|
+
throw state.failure;
|
|
338
|
+
readiness.expected = new Set(this.#nodes.map((member) => this.#options.memberKey(member)));
|
|
339
|
+
if (!readiness.reconciling && [...readiness.expected].some((key) => !state.children.has(key)))
|
|
340
|
+
throw new Error("backend membership is unavailable.");
|
|
341
|
+
this.#emitReady(state, readiness);
|
|
342
|
+
await Promise.race([readiness.gate.promise, _a.waitForAbort(signal)]);
|
|
184
343
|
}
|
|
185
344
|
/**
|
|
186
345
|
* Clears one logical definition and its live child subscriptions.
|
|
@@ -245,9 +404,12 @@ export class BackendMembershipKernel {
|
|
|
245
404
|
this.#nodes = [...members];
|
|
246
405
|
this.#pending = { members: this.#nodes, generation: ++this.#generation };
|
|
247
406
|
if (abortStarts)
|
|
248
|
-
for (const state of this.#definitions.values())
|
|
407
|
+
for (const state of this.#definitions.values()) {
|
|
408
|
+
if (state.readiness !== undefined && !state.readiness.settled)
|
|
409
|
+
state.readiness.reconciling = true;
|
|
249
410
|
for (const start of state.starts)
|
|
250
411
|
start.abort();
|
|
412
|
+
}
|
|
251
413
|
if (this.#running === undefined) {
|
|
252
414
|
this.#completion = new Promise((resolve) => {
|
|
253
415
|
this.#complete = resolve;
|
|
@@ -257,22 +419,48 @@ export class BackendMembershipKernel {
|
|
|
257
419
|
return this.#completion;
|
|
258
420
|
}
|
|
259
421
|
async #run() {
|
|
422
|
+
let finalFailure;
|
|
260
423
|
while (this.#pending !== undefined && !this.#closed) {
|
|
261
424
|
const pending = this.#pending;
|
|
262
425
|
this.#pending = undefined;
|
|
263
426
|
try {
|
|
264
427
|
await this.#replace(pending.members, pending.generation);
|
|
428
|
+
finalFailure = undefined;
|
|
265
429
|
}
|
|
266
430
|
catch (error) {
|
|
267
431
|
// spine-log-boundary: deployment.membership_reconciliation
|
|
268
|
-
|
|
432
|
+
finalFailure = error;
|
|
269
433
|
/* a later snapshot can recover */
|
|
270
434
|
}
|
|
271
435
|
}
|
|
272
436
|
this.#running = undefined;
|
|
437
|
+
this.#finishReadinessReconcile(finalFailure);
|
|
273
438
|
this.#complete?.();
|
|
274
439
|
this.#complete = undefined;
|
|
275
440
|
}
|
|
441
|
+
/**
|
|
442
|
+
* Releases pending ACK barriers only after the final desired member snapshot is installed.
|
|
443
|
+
*
|
|
444
|
+
* @param failure The final reconciliation failure, when one occurred.
|
|
445
|
+
*/
|
|
446
|
+
#finishReadinessReconcile(failure) {
|
|
447
|
+
const required = new Set(this.#nodes.map((member) => this.#options.memberKey(member)));
|
|
448
|
+
for (const state of this.#definitions.values()) {
|
|
449
|
+
const readiness = state.readiness;
|
|
450
|
+
if (readiness === undefined || !readiness.reconciling || readiness.settled)
|
|
451
|
+
continue;
|
|
452
|
+
readiness.reconciling = false;
|
|
453
|
+
if (failure !== undefined ||
|
|
454
|
+
state.failure !== undefined ||
|
|
455
|
+
required.size === 0 ||
|
|
456
|
+
[...required].some((key) => !state.children.has(key))) {
|
|
457
|
+
this.#failReadiness(readiness, failure ?? state.failure ?? new Error("backend membership is unavailable."));
|
|
458
|
+
continue;
|
|
459
|
+
}
|
|
460
|
+
readiness.expected = new Set(required);
|
|
461
|
+
this.#emitReady(state, readiness);
|
|
462
|
+
}
|
|
463
|
+
}
|
|
276
464
|
async #replace(members, generation) {
|
|
277
465
|
if (generation !== this.#generation || this.#closed)
|
|
278
466
|
return;
|
|
@@ -343,10 +531,16 @@ export class BackendMembershipKernel {
|
|
|
343
531
|
this.#members.set(key, { member, client, incarnation: ++this.#incarnation, eligible: false });
|
|
344
532
|
}
|
|
345
533
|
async #syncDefinition(state, generation) {
|
|
534
|
+
if (state.lifetime?.failure !== undefined)
|
|
535
|
+
return;
|
|
346
536
|
state.failure = undefined;
|
|
347
537
|
for (const [key, child] of state.children)
|
|
348
538
|
if (!this.#members.has(key)) {
|
|
349
539
|
state.children.delete(key);
|
|
540
|
+
state.readiness?.expected?.delete(key);
|
|
541
|
+
state.readiness?.seen.delete(key);
|
|
542
|
+
if (state.readiness !== undefined)
|
|
543
|
+
this.#emitReady(state, state.readiness);
|
|
350
544
|
child.controller.abort();
|
|
351
545
|
child.activationController?.abort();
|
|
352
546
|
// spine-log-boundary: deployment.membership_child_completion
|
|
@@ -374,7 +568,7 @@ export class BackendMembershipKernel {
|
|
|
374
568
|
}
|
|
375
569
|
}
|
|
376
570
|
async #startChild(state, key, current, generation) {
|
|
377
|
-
if (this.#closed || generation !== this.#generation)
|
|
571
|
+
if (this.#closed || generation !== this.#generation || state.lifetime?.failure !== undefined)
|
|
378
572
|
return;
|
|
379
573
|
const controller = new AbortController();
|
|
380
574
|
state.starts.add(controller);
|
|
@@ -387,12 +581,29 @@ export class BackendMembershipKernel {
|
|
|
387
581
|
state.starts.delete(controller);
|
|
388
582
|
this.#childStarts.delete(controller);
|
|
389
583
|
}
|
|
584
|
+
await this.#installChild(state, key, current, generation, controller, child);
|
|
585
|
+
}
|
|
586
|
+
/**
|
|
587
|
+
* Installs a created child only while its definition and member remain current.
|
|
588
|
+
*
|
|
589
|
+
* @param state The retained logical definition.
|
|
590
|
+
* @param key The backend member identifier.
|
|
591
|
+
* @param current The member that created the child.
|
|
592
|
+
* @param generation The reconciliation generation.
|
|
593
|
+
* @param controller The child creation controller.
|
|
594
|
+
* @param child The created native child.
|
|
595
|
+
* @returns Completion after installation or compensation.
|
|
596
|
+
*/
|
|
597
|
+
async #installChild(state, key, current, generation, controller, child) {
|
|
390
598
|
if (this.#options.childSize(child) > state.maxChildBytes) {
|
|
391
599
|
await this.#compensateChild(current.client, child, controller.signal);
|
|
392
600
|
throw new Error("backend-envelope-too-large");
|
|
393
601
|
}
|
|
394
602
|
if (generation !== this.#generation ||
|
|
395
603
|
this.#members.get(key)?.incarnation !== current.incarnation ||
|
|
604
|
+
controller.signal.aborted ||
|
|
605
|
+
this.#definitions.get(this.#options.definitionKey(state.definition) ?? "") !== state ||
|
|
606
|
+
state.lifetime?.failure !== undefined ||
|
|
396
607
|
this.#isClosed()) {
|
|
397
608
|
await this.#compensateChild(current.client, child, controller.signal);
|
|
398
609
|
return;
|
|
@@ -409,33 +620,235 @@ export class BackendMembershipKernel {
|
|
|
409
620
|
this.#activateChild(state, key, stored, current.client);
|
|
410
621
|
}
|
|
411
622
|
#activateChild(state, key, child, client) {
|
|
412
|
-
if (child.active || state.updates === undefined)
|
|
623
|
+
if (child.active || state.updates === undefined || state.lifetime?.failure !== undefined)
|
|
413
624
|
return;
|
|
414
625
|
child.active = true;
|
|
415
626
|
const controller = new AbortController();
|
|
416
627
|
child.activationController = controller;
|
|
628
|
+
const readiness = state.readiness;
|
|
629
|
+
const lifetime = state.lifetime;
|
|
630
|
+
if (readiness !== undefined && !readiness.emitting && !readiness.settled)
|
|
631
|
+
readiness.expected?.add(key);
|
|
632
|
+
const logical = state.activationController;
|
|
633
|
+
const sink = state.updates;
|
|
634
|
+
const onUpdate = (update) => this.#relayChild(state, key, readiness, logical, sink, update);
|
|
635
|
+
const activation = { state, key, child, client, controller, logical, lifetime, readiness };
|
|
636
|
+
this.#observeChildActivation(activation, onUpdate);
|
|
637
|
+
}
|
|
638
|
+
/**
|
|
639
|
+
* Observes one captured child relay through failure and completion.
|
|
640
|
+
* @param activation The captured child identity and cancellation state.
|
|
641
|
+
* @param onUpdate Receives child updates for the logical relay.
|
|
642
|
+
*/
|
|
643
|
+
#observeChildActivation(activation, onUpdate) {
|
|
644
|
+
const { child, client, controller } = activation;
|
|
417
645
|
// spine-log-boundary: deployment.membership_child_activation
|
|
418
646
|
child.activation = client
|
|
419
|
-
.activate(child.child,
|
|
420
|
-
.catch(() =>
|
|
647
|
+
.activate(child.child, onUpdate, controller.signal)
|
|
648
|
+
.catch((error) => {
|
|
649
|
+
this.#failChildActivation(activation, error);
|
|
650
|
+
})
|
|
421
651
|
.then(() => {
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
652
|
+
this.#finishChildActivation(activation);
|
|
653
|
+
});
|
|
654
|
+
}
|
|
655
|
+
/**
|
|
656
|
+
* Fences the current logical relay when one required child fails or ends.
|
|
657
|
+
* @param activation The captured child identity and cancellation state.
|
|
658
|
+
* @param error The child failure or unexpected stream-end cause.
|
|
659
|
+
*/
|
|
660
|
+
#failChildActivation(activation, error) {
|
|
661
|
+
const { state, key, child, controller, logical, lifetime, readiness } = activation;
|
|
662
|
+
if (lifetime === undefined ||
|
|
663
|
+
logical === undefined ||
|
|
664
|
+
lifetime.failure !== undefined ||
|
|
665
|
+
controller.signal.aborted ||
|
|
666
|
+
logical.signal.aborted ||
|
|
667
|
+
this.#closed ||
|
|
668
|
+
state.lifetime !== lifetime ||
|
|
669
|
+
state.activationController !== logical ||
|
|
670
|
+
state.children.get(key) !== child ||
|
|
671
|
+
child.activationController !== controller)
|
|
672
|
+
return;
|
|
673
|
+
lifetime.failure =
|
|
674
|
+
error instanceof Error ? error : new Error("Backend child failed.", { cause: error });
|
|
675
|
+
for (const start of state.starts)
|
|
676
|
+
start.abort();
|
|
677
|
+
for (const current of state.children.values())
|
|
678
|
+
current.activationController?.abort();
|
|
679
|
+
lifetime.gate.reject(lifetime.failure);
|
|
680
|
+
if (readiness !== undefined)
|
|
681
|
+
this.#failReadiness(readiness, error);
|
|
682
|
+
}
|
|
683
|
+
/**
|
|
684
|
+
* Restarts a cancelled child relay or disposes a child whose stream ended.
|
|
685
|
+
*
|
|
686
|
+
* @param activation The captured child identity and cancellation state.
|
|
687
|
+
*/
|
|
688
|
+
#finishChildActivation(activation) {
|
|
689
|
+
const { state, key, child, client, controller, lifetime, readiness } = activation;
|
|
690
|
+
if (child.activationController !== controller)
|
|
691
|
+
return;
|
|
692
|
+
if (!controller.signal.aborted) {
|
|
693
|
+
const failure = new Error("Backend child stream ended unexpectedly.");
|
|
694
|
+
this.#failChildActivation(activation, failure);
|
|
695
|
+
}
|
|
696
|
+
child.activationController = undefined;
|
|
697
|
+
child.active = false;
|
|
698
|
+
if (controller.signal.aborted &&
|
|
699
|
+
lifetime?.failure === undefined &&
|
|
700
|
+
state.children.get(key) === child &&
|
|
701
|
+
state.active &&
|
|
702
|
+
state.activationController !== undefined &&
|
|
703
|
+
!state.activationController.signal.aborted &&
|
|
704
|
+
state.updates !== undefined) {
|
|
705
|
+
this.#activateChild(state, key, child, client);
|
|
706
|
+
return;
|
|
707
|
+
}
|
|
708
|
+
this.#dropEndedChild(state, key, child, client, controller, readiness);
|
|
709
|
+
}
|
|
710
|
+
/**
|
|
711
|
+
* Disposes an ended child only while its logical relay still expects updates.
|
|
712
|
+
*
|
|
713
|
+
* @param state The retained logical definition.
|
|
714
|
+
* @param key The backend member identifier.
|
|
715
|
+
* @param child The ended child state.
|
|
716
|
+
* @param client The member client that created the child.
|
|
717
|
+
* @param controller The child activation controller.
|
|
718
|
+
* @param readiness The current logical acknowledgement gate.
|
|
719
|
+
*/
|
|
720
|
+
#dropEndedChild(state, key, child, client, controller, readiness) {
|
|
721
|
+
if (state.children.get(key) !== child ||
|
|
722
|
+
controller.signal.aborted ||
|
|
723
|
+
!state.active ||
|
|
724
|
+
state.activationController === undefined ||
|
|
725
|
+
state.activationController.signal.aborted ||
|
|
726
|
+
readiness?.failure !== undefined)
|
|
727
|
+
return;
|
|
728
|
+
state.children.delete(key);
|
|
729
|
+
readiness?.seen.delete(key);
|
|
730
|
+
void this.#disposeEndedChild(client, child.child);
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* Tracks cleanup of a child whose stream ended independently of logical cancellation.
|
|
734
|
+
*
|
|
735
|
+
* @param client The member client that created the child.
|
|
736
|
+
* @param child The ended child subscription.
|
|
737
|
+
*/
|
|
738
|
+
async #disposeEndedChild(client, child) {
|
|
739
|
+
const running = this.#compensateChild(client, child, new AbortController().signal);
|
|
740
|
+
const cleanups = this.#childCleanup.get(client) ?? new Set();
|
|
741
|
+
const tracked = running.then(() => undefined);
|
|
742
|
+
cleanups.add(tracked);
|
|
743
|
+
this.#childCleanup.set(client, cleanups);
|
|
744
|
+
try {
|
|
745
|
+
await tracked;
|
|
746
|
+
}
|
|
747
|
+
finally {
|
|
748
|
+
cleanups.delete(tracked);
|
|
749
|
+
if (cleanups.size === 0)
|
|
750
|
+
this.#childCleanup.delete(client);
|
|
751
|
+
}
|
|
752
|
+
}
|
|
753
|
+
/**
|
|
754
|
+
* Consumes one child's ACK locally and gates its business updates behind logical readiness.
|
|
755
|
+
*
|
|
756
|
+
* @param state The retained logical definition.
|
|
757
|
+
* @param key The child member key.
|
|
758
|
+
* @param readiness The readiness state captured when this child started.
|
|
759
|
+
* @param logical The logical activation controller captured when this child started.
|
|
760
|
+
* @param sink The original logical update sink captured when this child started.
|
|
761
|
+
* @param update The child update to inspect or forward.
|
|
762
|
+
*/
|
|
763
|
+
async #relayChild(state, key, readiness, logical, sink, update) {
|
|
764
|
+
if (logical === undefined || !this.#relayIsCurrent(state, logical))
|
|
765
|
+
return;
|
|
766
|
+
if (readiness === undefined) {
|
|
767
|
+
await sink(update);
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
if (state.readiness !== readiness || state.activationController !== logical)
|
|
771
|
+
return;
|
|
772
|
+
const isAck = this.#options.isActivationAck?.(update) ?? false;
|
|
773
|
+
if (!readiness.seen.has(key)) {
|
|
774
|
+
if (!isAck)
|
|
775
|
+
throw new Error("Backend child activation ACK is missing or invalid.");
|
|
776
|
+
readiness.seen.add(key);
|
|
777
|
+
readiness.first ??= update;
|
|
778
|
+
this.#emitReady(state, readiness);
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
if (isAck)
|
|
782
|
+
throw new Error("Backend child sent a duplicate activation ACK.");
|
|
783
|
+
await readiness.gate.promise;
|
|
784
|
+
if (state.readiness === readiness && this.#relayIsCurrent(state, logical))
|
|
785
|
+
await state.updates?.(update);
|
|
786
|
+
}
|
|
787
|
+
/**
|
|
788
|
+
* Checks generation, cancellation, and terminal failure before a child update.
|
|
789
|
+
* @param state The retained logical definition.
|
|
790
|
+
* @param logical The child relay's captured activation controller.
|
|
791
|
+
* @returns Whether the captured activation still accepts updates.
|
|
792
|
+
*/
|
|
793
|
+
#relayIsCurrent(state, logical) {
|
|
794
|
+
return (state.activationController === logical &&
|
|
795
|
+
!logical.signal.aborted &&
|
|
796
|
+
state.lifetime?.failure === undefined);
|
|
797
|
+
}
|
|
798
|
+
/**
|
|
799
|
+
* Forwards exactly one logical ACK after every required child has acknowledged attachment.
|
|
800
|
+
*
|
|
801
|
+
* @param state The retained logical definition.
|
|
802
|
+
* @param readiness The current activation's child acknowledgement state.
|
|
803
|
+
*/
|
|
804
|
+
#emitReady(state, readiness) {
|
|
805
|
+
if (readiness.reconciling ||
|
|
806
|
+
readiness.emitting ||
|
|
807
|
+
readiness.settled ||
|
|
808
|
+
readiness.expected === undefined ||
|
|
809
|
+
readiness.expected.size === 0 ||
|
|
810
|
+
![...readiness.expected].every((key) => readiness.seen.has(key)))
|
|
811
|
+
return;
|
|
812
|
+
const update = readiness.first;
|
|
813
|
+
const sink = state.updates;
|
|
814
|
+
if (update === undefined || sink === undefined) {
|
|
815
|
+
this.#failReadiness(readiness, new Error("Backend child activation ACK is unavailable."));
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
readiness.emitting = true;
|
|
819
|
+
let logicalAck;
|
|
820
|
+
try {
|
|
821
|
+
logicalAck = this.#options.logicalActivationAck?.(state.definition, update) ?? update;
|
|
822
|
+
}
|
|
823
|
+
catch (error) {
|
|
824
|
+
this.#failReadiness(readiness, error);
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
void sink(logicalAck).then(() => {
|
|
828
|
+
if (readiness.settled)
|
|
433
829
|
return;
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
830
|
+
readiness.settled = true;
|
|
831
|
+
readiness.gate.resolve(undefined);
|
|
832
|
+
}, (error) => {
|
|
833
|
+
this.#failReadiness(readiness, error);
|
|
437
834
|
});
|
|
438
835
|
}
|
|
836
|
+
/**
|
|
837
|
+
* Rejects pending readiness once while retaining the original child or sink failure.
|
|
838
|
+
*
|
|
839
|
+
* @param readiness The current activation's child acknowledgement state.
|
|
840
|
+
* @param error The failing child or update-sink cause.
|
|
841
|
+
*/
|
|
842
|
+
#failReadiness(readiness, error) {
|
|
843
|
+
if (readiness.settled)
|
|
844
|
+
return;
|
|
845
|
+
readiness.settled = true;
|
|
846
|
+
readiness.failure =
|
|
847
|
+
error instanceof Error
|
|
848
|
+
? error
|
|
849
|
+
: new Error("Backend child activation failed.", { cause: error });
|
|
850
|
+
readiness.gate.reject(readiness.failure);
|
|
851
|
+
}
|
|
439
852
|
async #removeDefinition(key, signal) {
|
|
440
853
|
const state = this.#definitions.get(key);
|
|
441
854
|
if (state === undefined)
|
|
@@ -457,6 +870,10 @@ export class BackendMembershipKernel {
|
|
|
457
870
|
if (child === undefined)
|
|
458
871
|
continue;
|
|
459
872
|
state.children.delete(key);
|
|
873
|
+
state.readiness?.expected?.delete(key);
|
|
874
|
+
state.readiness?.seen.delete(key);
|
|
875
|
+
if (state.readiness !== undefined)
|
|
876
|
+
this.#emitReady(state, state.readiness);
|
|
460
877
|
try {
|
|
461
878
|
await this.#cleanupChild(child, client);
|
|
462
879
|
}
|
|
@@ -521,8 +938,6 @@ export class BackendMembershipKernel {
|
|
|
521
938
|
continue;
|
|
522
939
|
}
|
|
523
940
|
async #disposeChild(pending, signal) {
|
|
524
|
-
if (pending.running !== undefined)
|
|
525
|
-
return false;
|
|
526
941
|
const controller = new AbortController();
|
|
527
942
|
const abort = () => {
|
|
528
943
|
controller.abort();
|
|
@@ -531,15 +946,35 @@ export class BackendMembershipKernel {
|
|
|
531
946
|
if (signal.aborted)
|
|
532
947
|
abort();
|
|
533
948
|
try {
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
949
|
+
await this.#awaitChildDisposal(pending, controller);
|
|
950
|
+
return true;
|
|
951
|
+
}
|
|
952
|
+
catch {
|
|
953
|
+
controller.abort();
|
|
954
|
+
return false;
|
|
955
|
+
}
|
|
956
|
+
finally {
|
|
957
|
+
signal.removeEventListener("abort", abort);
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
/**
|
|
961
|
+
* Races one shared child disposal against its existing bounded wait.
|
|
962
|
+
*
|
|
963
|
+
* @param pending The retained child disposal attempt.
|
|
964
|
+
* @param controller Cancels the entered disposal when its wait fails.
|
|
965
|
+
* @returns Completion when the child disposal settles within its limit.
|
|
966
|
+
*/
|
|
967
|
+
async #awaitChildDisposal(pending, controller) {
|
|
968
|
+
let timer;
|
|
969
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
970
|
+
timer = setTimeout(() => {
|
|
971
|
+
reject(new Error("backend child cleanup timed out"));
|
|
972
|
+
}, childCleanupTimeoutMs);
|
|
973
|
+
timer.unref();
|
|
974
|
+
});
|
|
975
|
+
try {
|
|
976
|
+
const running = pending.running ?? pending.client.dispose(pending.child, controller.signal);
|
|
977
|
+
if (pending.running === undefined) {
|
|
543
978
|
pending.running = running;
|
|
544
979
|
const settled = () => {
|
|
545
980
|
if (pending.running === running)
|
|
@@ -547,20 +982,12 @@ export class BackendMembershipKernel {
|
|
|
547
982
|
};
|
|
548
983
|
// spine-log-boundary: deployment.membership_child_cleanup_retry
|
|
549
984
|
void running.then(settled, settled);
|
|
550
|
-
await Promise.race([running, timeout]);
|
|
551
985
|
}
|
|
552
|
-
|
|
553
|
-
if (timer !== undefined)
|
|
554
|
-
clearTimeout(timer);
|
|
555
|
-
}
|
|
556
|
-
return true;
|
|
557
|
-
}
|
|
558
|
-
catch {
|
|
559
|
-
controller.abort();
|
|
560
|
-
return false;
|
|
986
|
+
await Promise.race([running, timeout]);
|
|
561
987
|
}
|
|
562
988
|
finally {
|
|
563
|
-
|
|
989
|
+
if (timer !== undefined)
|
|
990
|
+
clearTimeout(timer);
|
|
564
991
|
}
|
|
565
992
|
}
|
|
566
993
|
#retainFailedChildCleanup(client, child) {
|
|
@@ -601,6 +1028,7 @@ export class BackendMembershipKernel {
|
|
|
601
1028
|
start.abort();
|
|
602
1029
|
await this.#running;
|
|
603
1030
|
await Promise.allSettled([...this.#definitions.keys()].map((key) => this.#removeDefinition(key, new AbortController().signal)));
|
|
1031
|
+
await Promise.allSettled([...this.#childCleanup.values()].flatMap((pending) => [...pending]));
|
|
604
1032
|
await this.#retryChildCleanup();
|
|
605
1033
|
await Promise.all([...this.#members.values()].map(({ client }) => this.#dispose(client)));
|
|
606
1034
|
this.#members.clear();
|
|
@@ -611,4 +1039,5 @@ export class BackendMembershipKernel {
|
|
|
611
1039
|
throw new Error("backend child cleanup remains incomplete.");
|
|
612
1040
|
}
|
|
613
1041
|
}
|
|
1042
|
+
_a = BackendMembershipKernel;
|
|
614
1043
|
//# sourceMappingURL=backend-membership-kernel.js.map
|