@zlink-systems/framework 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/contracts/Channels/IZLinkFanoutClient.d.ts +0 -6
- package/dist/contracts/RouteMesh/RuntimeTopology.d.ts +8 -0
- package/dist/internal.d.ts +1 -0
- package/dist/internal.js +1 -0
- package/dist/runtime/activation-admission.d.ts +29 -0
- package/dist/runtime/activation-admission.js +61 -0
- package/dist/runtime/actors/actor-context.js +6 -6
- package/dist/runtime/actors/actor-handoff.d.ts +9 -3
- package/dist/runtime/actors/actor-handoff.js +28 -17
- package/dist/runtime/actors/actor-local-native-join.js +2 -18
- package/dist/runtime/actors/actor-remote-joiner.d.ts +2 -2
- package/dist/runtime/actors/actor-remote-joiner.js +6 -8
- package/dist/runtime/actors/actor-runtime-contracts.d.ts +4 -2
- package/dist/runtime/actors/actor-source-transfer.d.ts +5 -5
- package/dist/runtime/actors/deferred-join-completion.d.ts +14 -0
- package/dist/runtime/actors/deferred-join-completion.js +20 -0
- package/dist/runtime/actors/index.d.ts +1 -1
- package/dist/runtime/actors/index.js +15 -5
- package/dist/runtime/admission.d.ts +5 -1
- package/dist/runtime/admission.js +4 -0
- package/dist/runtime/backend/contracts/index.d.ts +2 -10
- package/dist/runtime/backend/mesh-actor-session-node-adapter.js +2 -9
- package/dist/runtime/backend/node/node-backend-adapter-factory.d.ts +1 -2
- package/dist/runtime/backend/node/node-backend-adapter-factory.js +4 -18
- package/dist/runtime/backend/node/node-backend-adapter-support.d.ts +6 -3
- package/dist/runtime/backend/node/node-backend-adapter-support.js +12 -28
- package/dist/runtime/backend/node/node-event-loop-poller.d.ts +1 -1
- package/dist/runtime/backend/node/node-event-loop-poller.js +8 -2
- package/dist/runtime/backend/node/node-raw-binding-port.d.ts +1 -1
- package/dist/runtime/backend/node/node-raw-binding-port.js +11 -3
- package/dist/runtime/backend/node/node-raw-mesh-backend.d.ts +12 -9
- package/dist/runtime/backend/node/node-raw-mesh-backend.js +61 -29
- package/dist/runtime/backend/node/node-socket-backend-adapter.js +1 -1
- package/dist/runtime/backend/raw-binding-port.d.ts +12 -0
- package/dist/runtime/channels/channel-clients.d.ts +0 -1
- package/dist/runtime/channels/channel-clients.js +0 -8
- package/dist/runtime/channels/channel-outbound-operations.js +2 -10
- package/dist/runtime/channels/channel-runtime-manager.d.ts +2 -5
- package/dist/runtime/channels/channel-runtime-manager.js +1 -12
- package/dist/runtime/channels/channel-socket-registry.d.ts +17 -4
- package/dist/runtime/channels/channel-socket-registry.js +49 -62
- package/dist/runtime/channels/channel-transports.d.ts +0 -3
- package/dist/runtime/channels/fanout-location-runtime.js +3 -9
- package/dist/runtime/execution/index.d.ts +18 -8
- package/dist/runtime/execution/index.js +55 -29
- package/dist/runtime/execution/serial-execution-queue.d.ts +5 -6
- package/dist/runtime/execution/serial-execution-queue.js +49 -15
- package/dist/runtime/foundation/listener-records.d.ts +11 -0
- package/dist/runtime/foundation/listener-records.js +20 -0
- package/dist/runtime/foundation/raw-service-mesh-runtime.d.ts +10 -21
- package/dist/runtime/foundation/raw-service-mesh-runtime.js +66 -392
- package/dist/runtime/foundation/service-authority-payload-codec.d.ts +1 -0
- package/dist/runtime/foundation/service-authority-payload-codec.js +8 -0
- package/dist/runtime/foundation/service-maintenance-runtime.js +23 -2
- package/dist/runtime/foundation/service-relocation-coordinator.d.ts +9 -0
- package/dist/runtime/foundation/service-relocation-coordinator.js +14 -1
- package/dist/runtime/foundation/service-relocation-object-owner.d.ts +8 -0
- package/dist/runtime/foundation/service-relocation-object-owner.js +16 -4
- package/dist/runtime/foundation/service-session-binding-ingress-port.d.ts +6 -13
- package/dist/runtime/foundation/service-stateful-runtime.d.ts +10 -7
- package/dist/runtime/foundation/service-stateful-runtime.js +100 -60
- package/dist/runtime/foundation/service-stateful-wire-codec.d.ts +9 -4
- package/dist/runtime/foundation/service-stateful-wire-codec.js +153 -90
- package/dist/runtime/foundation/service-topology-registry.d.ts +1 -2
- package/dist/runtime/foundation/service-topology-registry.js +6 -30
- package/dist/runtime/host/actor-transfer-runtime.d.ts +17 -15
- package/dist/runtime/host/actor-transfer-runtime.js +73 -86
- package/dist/runtime/host/index.d.ts +6 -2
- package/dist/runtime/host/index.js +63 -37
- package/dist/runtime/host/remote-bound-session-relay.js +4 -3
- package/dist/runtime/host/route-mesh-runtime.d.ts +2 -0
- package/dist/runtime/host/route-mesh-runtime.js +7 -5
- package/dist/runtime/host/service-relocation-host-runtime.d.ts +50 -8
- package/dist/runtime/host/service-relocation-host-runtime.js +343 -270
- package/dist/runtime/host/spot-runtime-options-factory.d.ts +7 -4
- package/dist/runtime/host/spot-runtime-options-factory.js +4 -8
- package/dist/runtime/host/stateful-authority-route-runtime.js +6 -2
- package/dist/runtime/host/user-spot-creation-coordinator.d.ts +11 -3
- package/dist/runtime/host/user-spot-creation-coordinator.js +85 -87
- package/dist/runtime/locations/in-memory-provider-location-store.js +8 -6
- package/dist/runtime/locations/lifecycle.d.ts +1 -1
- package/dist/runtime/locations/lifecycle.js +2 -2
- package/dist/runtime/locations/resolvers.js +7 -2
- package/dist/runtime/locations/spot-location-claims.d.ts +2 -2
- package/dist/runtime/locations/spot-location-claims.js +3 -27
- package/dist/runtime/protocol/service_wire_codec.generated.d.ts +1 -0
- package/dist/runtime/protocol/service_wire_codec.generated.js +5 -1
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.d.ts +1 -0
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.js +3 -2
- package/dist/runtime/spots/formal-remote-actor-admission-registry.d.ts +3 -3
- package/dist/runtime/spots/formal-remote-actor-admission-registry.js +4 -4
- package/dist/runtime/spots/formal-remote-actor-transfer-registry.d.ts +3 -3
- package/dist/runtime/spots/index.d.ts +25 -19
- package/dist/runtime/spots/index.js +207 -205
- package/dist/runtime/spots/spot-activation-registry.d.ts +4 -2
- package/dist/runtime/spots/spot-activation-registry.js +34 -31
- package/dist/runtime/spots/spot-activation-state.d.ts +2 -11
- package/dist/runtime/spots/spot-activation-state.js +4 -17
- package/dist/runtime/spots/spot-activation.d.ts +8 -6
- package/dist/runtime/spots/spot-activation.js +54 -49
- package/dist/runtime/spots/spot-actor-admission-coordinator.d.ts +1 -1
- package/dist/runtime/spots/spot-actor-admission-coordinator.js +1 -1
- package/dist/runtime/spots/spot-actor-join-dispatch.d.ts +2 -2
- package/dist/runtime/spots/spot-actor-join-dispatch.js +3 -5
- package/dist/runtime/spots/spot-actor-lifecycle-drain.js +2 -2
- package/dist/runtime/spots/spot-actor-membership.d.ts +4 -0
- package/dist/runtime/spots/spot-actor-membership.js +26 -12
- package/dist/runtime/spots/spot-entry-activation.d.ts +1 -2
- package/dist/runtime/spots/spot-entry-activation.js +0 -1
- package/dist/runtime/spots/spot-manager-public.js +1 -1
- package/dist/runtime/spots/spot-native-actor-join-admission.d.ts +1 -1
- package/dist/runtime/spots/spot-native-actor-join-admission.js +2 -2
- package/dist/runtime/spots/spot-node-runtime-manager.d.ts +5 -5
- package/dist/runtime/spots/spot-node-runtime-manager.js +7 -11
- package/dist/runtime/spots/spot-outbound.js +1 -29
- package/dist/runtime/spots/spot-routed-actor-admission.d.ts +1 -0
- package/dist/runtime/spots/spot-routed-actor-admission.js +10 -0
- package/dist/runtime/spots/spot-routed-frame-dispatch.d.ts +2 -6
- package/dist/runtime/spots/spot-routed-frame-dispatch.js +2 -70
- package/dist/runtime/spots/spot-runtime-ports.d.ts +3 -5
- package/dist/runtime/spots/spot-serial-executor.d.ts +5 -0
- package/dist/runtime/spots/spot-serial-executor.js +8 -1
- package/dist/runtime/spots/spot-serial-turn-executor.d.ts +3 -0
- package/dist/runtime/spots/spot-serial-turn-executor.js +8 -4
- package/dist/runtime/streams/actor-session-binding-registry.d.ts +13 -15
- package/dist/runtime/streams/actor-session-binding-registry.js +19 -171
- package/dist/runtime/streams/actor-session-binding-runtime-owner.d.ts +3 -4
- package/dist/runtime/streams/bound-session-service.js +1 -18
- package/dist/runtime/streams/index.d.ts +2 -0
- package/dist/runtime/streams/index.js +19 -19
- package/dist/runtime/streams/session-serial-executor.js +2 -2
- package/package.json +3 -3
- package/src/contracts/Channels/IZLinkFanoutClient.ts +0 -7
- package/src/contracts/RouteMesh/RuntimeTopology.ts +10 -0
- package/src/internal.ts +1 -0
- package/src/runtime/activation-admission.ts +78 -0
- package/src/runtime/actors/actor-context.ts +6 -6
- package/src/runtime/actors/actor-handoff.ts +31 -23
- package/src/runtime/actors/actor-local-native-join.ts +3 -42
- package/src/runtime/actors/actor-remote-joiner.ts +6 -25
- package/src/runtime/actors/actor-runtime-contracts.ts +4 -10
- package/src/runtime/actors/actor-source-transfer.ts +5 -18
- package/src/runtime/actors/deferred-join-completion.ts +40 -0
- package/src/runtime/actors/index.ts +22 -9
- package/src/runtime/admission.ts +5 -1
- package/src/runtime/backend/contracts/index.ts +2 -13
- package/src/runtime/backend/mesh-actor-session-node-adapter.ts +2 -10
- package/src/runtime/backend/node/node-backend-adapter-factory.ts +5 -25
- package/src/runtime/backend/node/node-backend-adapter-support.ts +9 -29
- package/src/runtime/backend/node/node-event-loop-poller.ts +14 -2
- package/src/runtime/backend/node/node-raw-binding-port.ts +17 -3
- package/src/runtime/backend/node/node-raw-mesh-backend.ts +65 -32
- package/src/runtime/backend/node/node-socket-backend-adapter.ts +2 -2
- package/src/runtime/backend/raw-binding-port.ts +13 -0
- package/src/runtime/channels/channel-clients.ts +0 -11
- package/src/runtime/channels/channel-outbound-operations.ts +3 -11
- package/src/runtime/channels/channel-runtime-manager.ts +4 -15
- package/src/runtime/channels/channel-socket-registry.ts +69 -71
- package/src/runtime/channels/channel-transports.ts +0 -3
- package/src/runtime/channels/fanout-location-runtime.ts +4 -16
- package/src/runtime/execution/index.ts +68 -30
- package/src/runtime/execution/serial-execution-queue.ts +52 -20
- package/src/runtime/foundation/listener-records.ts +21 -0
- package/src/runtime/foundation/raw-service-mesh-runtime.ts +66 -466
- package/src/runtime/foundation/service-authority-payload-codec.ts +10 -0
- package/src/runtime/foundation/service-maintenance-runtime.ts +30 -9
- package/src/runtime/foundation/service-relocation-coordinator.ts +13 -0
- package/src/runtime/foundation/service-relocation-object-owner.ts +18 -5
- package/src/runtime/foundation/service-session-binding-ingress-port.ts +6 -14
- package/src/runtime/foundation/service-stateful-runtime.ts +151 -89
- package/src/runtime/foundation/service-stateful-wire-codec.ts +203 -121
- package/src/runtime/foundation/service-topology-registry.ts +8 -48
- package/src/runtime/host/actor-transfer-runtime.ts +94 -166
- package/src/runtime/host/index.ts +119 -54
- package/src/runtime/host/remote-bound-session-relay.ts +4 -3
- package/src/runtime/host/route-mesh-runtime.ts +9 -5
- package/src/runtime/host/service-relocation-host-runtime.ts +495 -376
- package/src/runtime/host/spot-runtime-options-factory.ts +24 -13
- package/src/runtime/host/stateful-authority-route-runtime.ts +8 -4
- package/src/runtime/host/user-spot-creation-coordinator.ts +147 -142
- package/src/runtime/locations/in-memory-provider-location-store.ts +11 -7
- package/src/runtime/locations/lifecycle.ts +3 -2
- package/src/runtime/locations/resolvers.ts +19 -13
- package/src/runtime/locations/spot-location-claims.ts +5 -34
- package/src/runtime/protocol/service_wire_codec.generated.ts +9 -2
- package/src/runtime/protocol/service_wire_pilot_codec.generated.ts +4 -4
- package/src/runtime/spots/formal-remote-actor-admission-registry.ts +6 -6
- package/src/runtime/spots/formal-remote-actor-transfer-registry.ts +3 -3
- package/src/runtime/spots/index.ts +341 -295
- package/src/runtime/spots/spot-activation-registry.ts +41 -31
- package/src/runtime/spots/spot-activation-state.ts +4 -16
- package/src/runtime/spots/spot-activation.ts +102 -80
- package/src/runtime/spots/spot-actor-admission-coordinator.ts +2 -2
- package/src/runtime/spots/spot-actor-join-dispatch.ts +6 -7
- package/src/runtime/spots/spot-actor-lifecycle-drain.ts +20 -18
- package/src/runtime/spots/spot-actor-membership.ts +81 -17
- package/src/runtime/spots/spot-entry-activation.ts +0 -3
- package/src/runtime/spots/spot-manager-public.ts +8 -4
- package/src/runtime/spots/spot-native-actor-join-admission.ts +2 -2
- package/src/runtime/spots/spot-node-runtime-manager.ts +12 -18
- package/src/runtime/spots/spot-outbound.ts +4 -47
- package/src/runtime/spots/spot-routed-actor-admission.ts +20 -0
- package/src/runtime/spots/spot-routed-frame-dispatch.ts +6 -76
- package/src/runtime/spots/spot-runtime-ports.ts +6 -17
- package/src/runtime/spots/spot-serial-executor.ts +9 -1
- package/src/runtime/spots/spot-serial-turn-executor.ts +10 -4
- package/src/runtime/streams/actor-session-binding-registry.ts +20 -308
- package/src/runtime/streams/actor-session-binding-runtime-owner.ts +2 -12
- package/src/runtime/streams/bound-session-service.ts +2 -36
- package/src/runtime/streams/index.ts +29 -27
- package/src/runtime/streams/session-serial-executor.ts +2 -2
- package/dist/runtime/actors/deferred-join-accepted-journal.d.ts +0 -47
- package/dist/runtime/actors/deferred-join-accepted-journal.js +0 -992
- package/src/runtime/actors/deferred-join-accepted-journal.ts +0 -1346
|
@@ -36,6 +36,7 @@ import type { ZLinkDomainLocationStore as ZLinkLocationStore } from '../location
|
|
|
36
36
|
import type { ZLinkTrackedInstanceAuthority } from '../locations/spot-location-claims';
|
|
37
37
|
import type { ZLinkProviderResolver } from '../../contracts/Common/ZLinkProviderResolver';
|
|
38
38
|
import type { ZLinkRuntimeEventPublisher, ZLinkRuntimeMetrics } from '../diagnostics';
|
|
39
|
+
import type { ZLinkActivationAdmission } from '../activation-admission';
|
|
39
40
|
import type { ZLinkFrameworkRegistration } from '../configuration';
|
|
40
41
|
import type { ZLinkBackendMeshNode, ZLinkMeshCompletionTable } from '../backend';
|
|
41
42
|
import { ReceiveKind, type ReceiveRecord } from '../foundation/service-runtime-contracts';
|
|
@@ -72,7 +73,10 @@ import {
|
|
|
72
73
|
restoreRelocationAdapterState,
|
|
73
74
|
type ZLinkRelocationStateAdapterLike
|
|
74
75
|
} from './relocation-state-adapter';
|
|
75
|
-
import {
|
|
76
|
+
import {
|
|
77
|
+
ServiceRelocationAuthorityError,
|
|
78
|
+
ServiceRelocationPostCommitError
|
|
79
|
+
} from '../foundation/service-relocation-coordinator';
|
|
76
80
|
import { ServiceMaintenanceRuntime } from '../foundation/service-maintenance-runtime';
|
|
77
81
|
import {
|
|
78
82
|
ServiceRelocationObjectCaptureOwner,
|
|
@@ -92,7 +96,6 @@ import { createProviderInstance } from '../spots/spot-provider';
|
|
|
92
96
|
import type { DefaultZLinkSpotManager, ZLinkSpotNodeRuntimeManager } from '../spots';
|
|
93
97
|
import type { ZLinkSpotActivation } from '../spots/spot-activation-state';
|
|
94
98
|
import type { DefaultZLinkActorManager } from '../actors';
|
|
95
|
-
import type { ZLinkDeferredJoinAcceptedRoot } from '../actors/deferred-join-accepted-journal';
|
|
96
99
|
import {
|
|
97
100
|
toFrameworkActorRef,
|
|
98
101
|
type ZLinkActorRuntimeState,
|
|
@@ -210,6 +213,7 @@ interface ZLinkHostRelocationOptions {
|
|
|
210
213
|
readonly reconcileStatefulAuthorityRoutes?: (signal?: AbortSignal) => Promise<void>;
|
|
211
214
|
readonly runtimeEventPublisher?: ZLinkRuntimeEventPublisher;
|
|
212
215
|
readonly metrics?: ZLinkRuntimeMetrics;
|
|
216
|
+
readonly activationAdmission?: ZLinkActivationAdmission;
|
|
213
217
|
}
|
|
214
218
|
|
|
215
219
|
interface RelocationTargetRequirement {
|
|
@@ -241,6 +245,8 @@ interface LocalStage {
|
|
|
241
245
|
readonly boundaryRelay: ServiceMaintenanceRelocationData[];
|
|
242
246
|
fallback?: ReturnType<typeof setTimeout>;
|
|
243
247
|
finalize?: Promise<void>;
|
|
248
|
+
/** The Restore's activation admission (MeshNode §5.1); held until target commit or abort. */
|
|
249
|
+
readonly releaseActivation: () => void;
|
|
244
250
|
}
|
|
245
251
|
|
|
246
252
|
interface TargetPayloadAssembly {
|
|
@@ -253,11 +259,11 @@ interface TargetPayloadAssembly {
|
|
|
253
259
|
interface SourceCutoverWindow {
|
|
254
260
|
readonly meshName: string;
|
|
255
261
|
readonly targetNodeRid: RoutingId;
|
|
256
|
-
/** Boundary batch frames plus the cutover frame, retained
|
|
262
|
+
/** Boundary batch frames plus the cutover frame, retained until authority settlement. */
|
|
257
263
|
frames?: readonly Buffer[];
|
|
264
|
+
/** A failed submit or a lost target connection requires the whole batch again. */
|
|
265
|
+
resendPending: boolean;
|
|
258
266
|
readonly cutoverSubmittedAtMs: number;
|
|
259
|
-
windowTimer?: ReturnType<typeof setTimeout>;
|
|
260
|
-
retryTimer?: ReturnType<typeof setTimeout>;
|
|
261
267
|
followTimer?: ReturnType<typeof setTimeout>;
|
|
262
268
|
windowClosed: boolean;
|
|
263
269
|
followExpired: boolean;
|
|
@@ -272,7 +278,6 @@ interface TargetRelocationOffer {
|
|
|
272
278
|
readonly prepareFingerprint: string;
|
|
273
279
|
readonly authenticatedSourceNodeRid: string;
|
|
274
280
|
readonly envelope: ServiceRelocationEnvelope;
|
|
275
|
-
readonly restoreDeadlineAtMs: number;
|
|
276
281
|
readonly reservation: TargetRelocationReservation;
|
|
277
282
|
}
|
|
278
283
|
|
|
@@ -502,8 +507,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
502
507
|
this.targetReadyFailures.clear();
|
|
503
508
|
this.targetReadyResponses.clear();
|
|
504
509
|
for (const window of this.sourceCutoverWindows.values()) {
|
|
505
|
-
if (window.windowTimer !== undefined) clearTimeout(window.windowTimer);
|
|
506
|
-
if (window.retryTimer !== undefined) clearTimeout(window.retryTimer);
|
|
507
510
|
if (window.followTimer !== undefined) clearTimeout(window.followTimer);
|
|
508
511
|
}
|
|
509
512
|
this.sourceCutoverWindows.clear();
|
|
@@ -661,22 +664,53 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
661
664
|
publishState: () => undefined,
|
|
662
665
|
forceStop: () => undefined
|
|
663
666
|
});
|
|
667
|
+
const started: Array<Promise<void>> = [];
|
|
664
668
|
for (const unit of work) {
|
|
665
669
|
maintenance.enqueue({
|
|
666
670
|
id: unit.id,
|
|
667
671
|
ready: () => true,
|
|
668
|
-
relocate:
|
|
672
|
+
relocate: (relocationSignal) => {
|
|
673
|
+
const run = unit.relocate(relocationSignal);
|
|
674
|
+
started.push(run);
|
|
675
|
+
return run;
|
|
676
|
+
}
|
|
669
677
|
});
|
|
670
678
|
}
|
|
671
679
|
const operationSignal = signal ?? new AbortController().signal;
|
|
672
680
|
const deadlineMs = signal === undefined ? 30_000 : 24 * 60 * 60 * 1000;
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
681
|
+
let failure: unknown;
|
|
682
|
+
try {
|
|
683
|
+
const result = await raceAbort(
|
|
684
|
+
maintenance.start('retire', deadlineMs, stopStartingSignal, operationSignal),
|
|
685
|
+
operationSignal
|
|
686
|
+
);
|
|
687
|
+
if (result.state === 'completed') return;
|
|
688
|
+
failure = result.terminalError ?? new Error(`Host relocation ended in '${result.state}'.`);
|
|
689
|
+
} catch (error) {
|
|
690
|
+
failure = error;
|
|
691
|
+
}
|
|
692
|
+
// The host result follows the settled authority of every started unit
|
|
693
|
+
// (spec 30 §13): a unit completes only after its target commit is
|
|
694
|
+
// confirmed, so a failure beside a completed unit leaves units on both
|
|
695
|
+
// sides.
|
|
696
|
+
const settled = await Promise.allSettled(started);
|
|
697
|
+
const targetSide = settled.filter(
|
|
698
|
+
(value) =>
|
|
699
|
+
value.status === 'fulfilled' || value.reason instanceof ServiceRelocationPostCommitError
|
|
700
|
+
).length;
|
|
701
|
+
const authorityError = settled.find(
|
|
702
|
+
(value): value is PromiseRejectedResult =>
|
|
703
|
+
value.status === 'rejected' && value.reason instanceof ServiceRelocationAuthorityError
|
|
676
704
|
);
|
|
677
|
-
if (
|
|
678
|
-
|
|
705
|
+
if (authorityError !== undefined) throw authorityError.reason;
|
|
706
|
+
// A unit that never started also stays on the source.
|
|
707
|
+
if (targetSide > 0 && targetSide < work.length) {
|
|
708
|
+
throw new ServiceRelocationAuthorityError(
|
|
709
|
+
'Relocation units settled on both the source and the target.',
|
|
710
|
+
{ cause: failure }
|
|
711
|
+
);
|
|
679
712
|
}
|
|
713
|
+
throw failure;
|
|
680
714
|
}
|
|
681
715
|
|
|
682
716
|
async tryHandleControl(
|
|
@@ -1385,6 +1419,28 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1385
1419
|
target: ZLinkMeshNodeDescriptor | undefined,
|
|
1386
1420
|
targetApplicationVersion: bigint | undefined,
|
|
1387
1421
|
signal?: AbortSignal
|
|
1422
|
+
): Promise<void> {
|
|
1423
|
+
await activation.serial.executeLifecycleOperation(() =>
|
|
1424
|
+
this.relocateSpotAggregateCore(
|
|
1425
|
+
meshName,
|
|
1426
|
+
activation,
|
|
1427
|
+
kind,
|
|
1428
|
+
actorStates,
|
|
1429
|
+
target,
|
|
1430
|
+
targetApplicationVersion,
|
|
1431
|
+
signal
|
|
1432
|
+
)
|
|
1433
|
+
);
|
|
1434
|
+
}
|
|
1435
|
+
|
|
1436
|
+
private async relocateSpotAggregateCore(
|
|
1437
|
+
meshName: string,
|
|
1438
|
+
activation: ZLinkSpotActivation,
|
|
1439
|
+
kind: 'user_spot' | 'instance_spot',
|
|
1440
|
+
actorStates: readonly ZLinkActorRuntimeState[],
|
|
1441
|
+
target: ZLinkMeshNodeDescriptor | undefined,
|
|
1442
|
+
targetApplicationVersion: bigint | undefined,
|
|
1443
|
+
signal?: AbortSignal
|
|
1388
1444
|
): Promise<void> {
|
|
1389
1445
|
const store = this.requireLocationStore();
|
|
1390
1446
|
const spotKey = encodeAuthorityKey(kind, String(activation.spotId));
|
|
@@ -1441,12 +1497,16 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1441
1497
|
objectGeneration: spotAuthority.objectGeneration,
|
|
1442
1498
|
authorityOwnerGeneration: spotAuthority.authorityOwnerGeneration,
|
|
1443
1499
|
seal: async (captureSignal) => {
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1500
|
+
const { spotCaptureOperation } = await this.afterSpotCloseDecision(
|
|
1501
|
+
meshName,
|
|
1502
|
+
activation.spotId,
|
|
1503
|
+
() => {
|
|
1504
|
+
interruptionStartedAt = performance.now();
|
|
1505
|
+
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
1506
|
+
// Both seals run in the same event-loop turn after the Close decision.
|
|
1507
|
+
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
1508
|
+
}
|
|
1509
|
+
);
|
|
1450
1510
|
const preparedSessions = Promise.all(
|
|
1451
1511
|
actorStates.map(async (state) => ({
|
|
1452
1512
|
state,
|
|
@@ -1623,7 +1683,11 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1623
1683
|
);
|
|
1624
1684
|
} catch (error) {
|
|
1625
1685
|
outcome = 'failed';
|
|
1626
|
-
if (
|
|
1686
|
+
if (
|
|
1687
|
+
readinessBoundaryConsumed &&
|
|
1688
|
+
!(error instanceof ServiceRelocationPostCommitError) &&
|
|
1689
|
+
!(error instanceof ServiceRelocationAuthorityError)
|
|
1690
|
+
) {
|
|
1627
1691
|
await activation
|
|
1628
1692
|
.completeConsumedRelocationBoundary(ZLinkSpotRelocationReadyOutcome.Continued)
|
|
1629
1693
|
.catch(() => undefined);
|
|
@@ -1763,6 +1827,24 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1763
1827
|
target: ZLinkMeshNodeDescriptor | undefined,
|
|
1764
1828
|
targetApplicationVersion: bigint | undefined,
|
|
1765
1829
|
signal?: AbortSignal
|
|
1830
|
+
): Promise<void> {
|
|
1831
|
+
await activation.serial.executeLifecycleOperation(() =>
|
|
1832
|
+
this.relocatePerActorSpotShellCore(
|
|
1833
|
+
meshName,
|
|
1834
|
+
activation,
|
|
1835
|
+
target,
|
|
1836
|
+
targetApplicationVersion,
|
|
1837
|
+
signal
|
|
1838
|
+
)
|
|
1839
|
+
);
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1842
|
+
private async relocatePerActorSpotShellCore(
|
|
1843
|
+
meshName: string,
|
|
1844
|
+
activation: ZLinkSpotActivation,
|
|
1845
|
+
target: ZLinkMeshNodeDescriptor | undefined,
|
|
1846
|
+
targetApplicationVersion: bigint | undefined,
|
|
1847
|
+
signal?: AbortSignal
|
|
1766
1848
|
): Promise<void> {
|
|
1767
1849
|
const spotKey = encodeAuthorityKey('user_spot', String(activation.spotId));
|
|
1768
1850
|
const spotAuthority = await requireAuthority(this.requireLocationStore(), spotKey, signal);
|
|
@@ -1796,9 +1878,16 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1796
1878
|
objectGeneration: spotAuthority.objectGeneration,
|
|
1797
1879
|
authorityOwnerGeneration: spotAuthority.authorityOwnerGeneration,
|
|
1798
1880
|
seal: async (captureSignal) => {
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1881
|
+
const { spotCaptureOperation } = await this.afterSpotCloseDecision(
|
|
1882
|
+
meshName,
|
|
1883
|
+
activation.spotId,
|
|
1884
|
+
() => {
|
|
1885
|
+
interruptionStartedAt = performance.now();
|
|
1886
|
+
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
1887
|
+
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
1888
|
+
}
|
|
1889
|
+
);
|
|
1890
|
+
spotCapture = await spotCaptureOperation;
|
|
1802
1891
|
return {
|
|
1803
1892
|
boundSessionState: Buffer.alloc(0),
|
|
1804
1893
|
queuedMessages: [],
|
|
@@ -1966,7 +2055,12 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1966
2055
|
},
|
|
1967
2056
|
abortSeal: async () => {
|
|
1968
2057
|
if (standalone && ownSession !== undefined) await ownSession.prepared.rollback();
|
|
1969
|
-
}
|
|
2058
|
+
},
|
|
2059
|
+
discardSeal: (reason) =>
|
|
2060
|
+
(standalone
|
|
2061
|
+
? ownSession
|
|
2062
|
+
: sessions.find((value) => value.state === state)
|
|
2063
|
+
)?.prepared.discard(reason)
|
|
1970
2064
|
};
|
|
1971
2065
|
}
|
|
1972
2066
|
|
|
@@ -2067,11 +2161,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2067
2161
|
advertisedReceiveChunkLimitBytes
|
|
2068
2162
|
);
|
|
2069
2163
|
const plan = planRelocationChunks(encoded, chunkLimitBytes);
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2164
|
+
// The Restore absolute deadline is this unit's relocation deadline: the
|
|
2165
|
+
// caller's signal, or the control deadline without one. From it the
|
|
2166
|
+
// source settles the authority with its Preserve fence (Location runtime
|
|
2167
|
+
// §10). The target keeps no deadline of its own.
|
|
2168
|
+
const restoreDeadlineReached = () =>
|
|
2169
|
+
signal?.aborted === true || performance.now() >= controlDeadlineAtMs;
|
|
2074
2170
|
let readyReceived = false;
|
|
2171
|
+
let settlement: 'target' | 'source' | 'lost' | undefined;
|
|
2075
2172
|
let sourceCommitted = false;
|
|
2076
2173
|
try {
|
|
2077
2174
|
const prepare = {
|
|
@@ -2169,9 +2266,9 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2169
2266
|
Buffer.concat(boundaryBatch.map((value) => value.canonicalBytes))
|
|
2170
2267
|
)
|
|
2171
2268
|
});
|
|
2172
|
-
// The
|
|
2173
|
-
//
|
|
2174
|
-
//
|
|
2269
|
+
// The cutover submit result is not the authority result. The memory
|
|
2270
|
+
// copy of the boundary batch and cutover stays until the authority
|
|
2271
|
+
// settlement below (spec 28 §4.4).
|
|
2175
2272
|
let submitFailure: unknown;
|
|
2176
2273
|
try {
|
|
2177
2274
|
for (const record of boundaryBatch) {
|
|
@@ -2182,7 +2279,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2182
2279
|
submitFailure = error;
|
|
2183
2280
|
console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
|
|
2184
2281
|
}
|
|
2185
|
-
this.beginSourceCutoverWindow(
|
|
2282
|
+
const window = this.beginSourceCutoverWindow(
|
|
2186
2283
|
meshName,
|
|
2187
2284
|
target.rid,
|
|
2188
2285
|
relayAuthorityId,
|
|
@@ -2190,53 +2287,63 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2190
2287
|
submitFailure !== undefined,
|
|
2191
2288
|
limits
|
|
2192
2289
|
);
|
|
2193
|
-
|
|
2194
|
-
await this.waitForTargetAuthorities(
|
|
2195
|
-
captured.envelope,
|
|
2196
|
-
targetFence,
|
|
2197
|
-
convergenceDeadlineAtMs,
|
|
2198
|
-
signal
|
|
2199
|
-
);
|
|
2200
|
-
relocationDebug('coordinator.target_authorities_observed', {
|
|
2201
|
-
aggregateId: captured.envelope.aggregateId
|
|
2202
|
-
});
|
|
2203
|
-
for (const session of sessions) session.prepared.setReplayResults([]);
|
|
2204
|
-
await captured.commitSource();
|
|
2205
|
-
sourceCommitted = true;
|
|
2206
|
-
await this.options.reconcileStatefulAuthorityRoutes?.(signal);
|
|
2207
|
-
} catch (error) {
|
|
2208
|
-
if (!readyReceived) throw error;
|
|
2209
2290
|
try {
|
|
2210
|
-
await this.
|
|
2291
|
+
settlement = await this.settleSourceAuthority(
|
|
2211
2292
|
captured.envelope,
|
|
2293
|
+
primary,
|
|
2212
2294
|
targetFence,
|
|
2213
|
-
|
|
2295
|
+
restoreDeadlineReached,
|
|
2296
|
+
window
|
|
2214
2297
|
);
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
);
|
|
2226
|
-
throw new
|
|
2227
|
-
authority
|
|
2228
|
-
{ id: `${captured.envelope.aggregateId}:${captured.envelope.aggregateGeneration}` },
|
|
2229
|
-
error
|
|
2298
|
+
} finally {
|
|
2299
|
+
this.closeSourceCutoverWindow(relayAuthorityId, window);
|
|
2300
|
+
}
|
|
2301
|
+
relocationDebug('coordinator.authority_settled', {
|
|
2302
|
+
aggregateId: captured.envelope.aggregateId,
|
|
2303
|
+
settlement
|
|
2304
|
+
});
|
|
2305
|
+
if (settlement === 'source') {
|
|
2306
|
+
// Preserve won: the captured queue, timers and held records return to
|
|
2307
|
+
// the source queue in their accepted order, and dispatch reopens.
|
|
2308
|
+
await captured.abortSource();
|
|
2309
|
+
throw new Error(
|
|
2310
|
+
`Relocation '${relayAuthorityId}' ended by the source Preserve fence; the source kept the authority.`
|
|
2230
2311
|
);
|
|
2231
|
-
}
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2312
|
+
}
|
|
2313
|
+
if (settlement === 'lost') {
|
|
2314
|
+
// Expired owner: dispatch and Store changes stop for good and every
|
|
2315
|
+
// pending request without a terminal gets one Unavailable.
|
|
2316
|
+
const unavailable = createInternalFrameworkException(
|
|
2317
|
+
ZLinkFrameworkInternalErrorKind.RouteNotConnected,
|
|
2318
|
+
`Relocation '${relayAuthorityId}' source owner lease ended before its authority settled.`,
|
|
2319
|
+
true
|
|
2238
2320
|
);
|
|
2321
|
+
await captured.discardSource(unavailable);
|
|
2322
|
+
throw new ServiceRelocationAuthorityError(unavailable.message, { cause: unavailable });
|
|
2239
2323
|
}
|
|
2324
|
+
for (const session of sessions) session.prepared.setReplayResults([]);
|
|
2325
|
+
await captured.commitSource();
|
|
2326
|
+
sourceCommitted = true;
|
|
2327
|
+
await this.options.reconcileStatefulAuthorityRoutes?.(signal);
|
|
2328
|
+
} catch (error) {
|
|
2329
|
+
if (settlement !== 'target') throw error;
|
|
2330
|
+
// Target commit is confirmed; later source cleanup failures cannot move
|
|
2331
|
+
// the authority back (spec 28 §9).
|
|
2332
|
+
if (!sourceCommitted) {
|
|
2333
|
+
for (const session of sessions) session.prepared.setReplayResults([]);
|
|
2334
|
+
await captured.commitSource().catch(() => undefined);
|
|
2335
|
+
}
|
|
2336
|
+
await this.options.reconcileStatefulAuthorityRoutes?.().catch(() => undefined);
|
|
2337
|
+
const authority = await requireAuthority(
|
|
2338
|
+
this.requireLocationStore(),
|
|
2339
|
+
{ value: primaryKey(captured.envelope) } as ZLinkAuthorityKey,
|
|
2340
|
+
undefined
|
|
2341
|
+
);
|
|
2342
|
+
throw new ServiceRelocationPostCommitError(
|
|
2343
|
+
authority,
|
|
2344
|
+
{ id: `${captured.envelope.aggregateId}:${captured.envelope.aggregateGeneration}` },
|
|
2345
|
+
error
|
|
2346
|
+
);
|
|
2240
2347
|
} finally {
|
|
2241
2348
|
if (!readyReceived) {
|
|
2242
2349
|
// Only an explicit failure before the Ready reply becomes accepted
|
|
@@ -2339,12 +2446,11 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2339
2446
|
}
|
|
2340
2447
|
|
|
2341
2448
|
/**
|
|
2342
|
-
* Starts the source
|
|
2343
|
-
*
|
|
2344
|
-
*
|
|
2345
|
-
*
|
|
2346
|
-
*
|
|
2347
|
-
* route removal point — closes the unit for SafeToShutdown.
|
|
2449
|
+
* Starts the source cutover window after the cutover submit terminal. The
|
|
2450
|
+
* boundary batch and cutover copies live in framework memory (not charged
|
|
2451
|
+
* to the in-flight budget) until the authority settlement closes the
|
|
2452
|
+
* window. S4 — the Message Follow route removal point — closes the unit for
|
|
2453
|
+
* SafeToShutdown.
|
|
2348
2454
|
*/
|
|
2349
2455
|
private beginSourceCutoverWindow(
|
|
2350
2456
|
meshName: string,
|
|
@@ -2353,29 +2459,18 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2353
2459
|
frames: readonly Buffer[],
|
|
2354
2460
|
initialSubmitFailed: boolean,
|
|
2355
2461
|
limits: ReturnType<ZLinkHostServiceRelocationRuntime['relocationLimits']>
|
|
2356
|
-
):
|
|
2462
|
+
): SourceCutoverWindow {
|
|
2357
2463
|
const submittedAt = performance.now();
|
|
2358
2464
|
const window: SourceCutoverWindow = {
|
|
2359
2465
|
meshName,
|
|
2360
2466
|
targetNodeRid,
|
|
2361
2467
|
frames,
|
|
2468
|
+
resendPending: initialSubmitFailed,
|
|
2362
2469
|
cutoverSubmittedAtMs: submittedAt,
|
|
2363
2470
|
windowClosed: false,
|
|
2364
2471
|
followExpired: false
|
|
2365
2472
|
};
|
|
2366
2473
|
this.sourceCutoverWindows.set(relocationKey, window);
|
|
2367
|
-
const settle = () => {
|
|
2368
|
-
if (window.windowClosed && window.followExpired) {
|
|
2369
|
-
this.sourceCutoverWindows.delete(relocationKey);
|
|
2370
|
-
}
|
|
2371
|
-
};
|
|
2372
|
-
window.windowTimer = setTimeout(() => {
|
|
2373
|
-
window.windowClosed = true;
|
|
2374
|
-
window.frames = undefined;
|
|
2375
|
-
if (window.retryTimer !== undefined) clearTimeout(window.retryTimer);
|
|
2376
|
-
settle();
|
|
2377
|
-
}, limits.relocationCutoverWaitTimeoutMs);
|
|
2378
|
-
window.windowTimer.unref();
|
|
2379
2474
|
window.followTimer = setTimeout(() => {
|
|
2380
2475
|
window.followExpired = true;
|
|
2381
2476
|
// S1→S4 route convergence: the span the source must keep its Message
|
|
@@ -2385,33 +2480,150 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2385
2480
|
Math.max(0, performance.now() - submittedAt) / 1000,
|
|
2386
2481
|
{ mesh_name: meshName }
|
|
2387
2482
|
);
|
|
2388
|
-
|
|
2483
|
+
this.releaseSourceCutoverWindow(relocationKey, window);
|
|
2389
2484
|
}, limits.messageFollowDurationMs);
|
|
2390
2485
|
window.followTimer.unref();
|
|
2391
|
-
|
|
2486
|
+
return window;
|
|
2392
2487
|
}
|
|
2393
2488
|
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
window.
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2489
|
+
/** The authority settlement ended: the retained batch copy is released. */
|
|
2490
|
+
private closeSourceCutoverWindow(relocationKey: string, window: SourceCutoverWindow): void {
|
|
2491
|
+
window.windowClosed = true;
|
|
2492
|
+
window.frames = undefined;
|
|
2493
|
+
this.releaseSourceCutoverWindow(relocationKey, window);
|
|
2494
|
+
}
|
|
2495
|
+
|
|
2496
|
+
private releaseSourceCutoverWindow(relocationKey: string, window: SourceCutoverWindow): void {
|
|
2497
|
+
if (
|
|
2498
|
+
window.windowClosed &&
|
|
2499
|
+
window.followExpired &&
|
|
2500
|
+
this.sourceCutoverWindows.get(relocationKey) === window
|
|
2501
|
+
) {
|
|
2502
|
+
this.sourceCutoverWindows.delete(relocationKey);
|
|
2503
|
+
}
|
|
2504
|
+
}
|
|
2505
|
+
|
|
2506
|
+
/**
|
|
2507
|
+
* Sends the whole boundary batch and cutover again after a failed submit
|
|
2508
|
+
* or an observed loss of the target connection (spec 28 §4.4). The target
|
|
2509
|
+
* replaces its partial span with the whole batch.
|
|
2510
|
+
*/
|
|
2511
|
+
private async resendSourceCutover(window: SourceCutoverWindow): Promise<void> {
|
|
2512
|
+
const frames = window.frames;
|
|
2513
|
+
if (frames === undefined) return;
|
|
2514
|
+
const peers = this.requireMeshNode(window.meshName).peers();
|
|
2515
|
+
const peer = peers.find(
|
|
2516
|
+
(value) =>
|
|
2517
|
+
value.routingId !== null && String(value.routingId) === String(window.targetNodeRid)
|
|
2518
|
+
);
|
|
2519
|
+
// Backend peer state 3 is Ready.
|
|
2520
|
+
if (peer !== undefined && peer.state !== 3) {
|
|
2521
|
+
window.resendPending = true;
|
|
2522
|
+
return;
|
|
2523
|
+
}
|
|
2524
|
+
if (!window.resendPending) return;
|
|
2525
|
+
try {
|
|
2526
|
+
for (const frame of frames) {
|
|
2527
|
+
await this.submitControlFrame(
|
|
2528
|
+
window.meshName,
|
|
2529
|
+
window.targetNodeRid,
|
|
2530
|
+
frame,
|
|
2531
|
+
'cutover retransmission'
|
|
2532
|
+
);
|
|
2533
|
+
}
|
|
2534
|
+
window.resendPending = false;
|
|
2535
|
+
} catch {
|
|
2536
|
+
// The next settlement round retries the whole batch.
|
|
2537
|
+
}
|
|
2538
|
+
}
|
|
2539
|
+
|
|
2540
|
+
/**
|
|
2541
|
+
* The one source authority settlement after Ready (Location runtime
|
|
2542
|
+
* §6.1·§10, spec 28 §4.4). It reads the Store until the target commit or
|
|
2543
|
+
* the Restore deadline. From the Restore deadline it runs the source
|
|
2544
|
+
* Preserve fence: a CAS on the StoreVersion expected by target NewOwner
|
|
2545
|
+
* excludes a late target commit. `target`: the
|
|
2546
|
+
* target committed. `source`: Preserve fenced the target out while the
|
|
2547
|
+
* source owner lease was valid. `lost`: the source owner lease ended first
|
|
2548
|
+
* or another authority owns the object. An uncertain Store or lease result
|
|
2549
|
+
* keeps the work retained and repeats the same decision.
|
|
2550
|
+
*/
|
|
2551
|
+
private async settleSourceAuthority(
|
|
2552
|
+
envelope: ServiceRelocationEnvelope,
|
|
2553
|
+
primary: ZLinkAuthoritySnapshot,
|
|
2554
|
+
target: ServiceWireRelocationTarget,
|
|
2555
|
+
restoreDeadlineReached: () => boolean,
|
|
2556
|
+
window: SourceCutoverWindow
|
|
2557
|
+
): Promise<'target' | 'source' | 'lost'> {
|
|
2558
|
+
const owner = { ownerId: primary.ownerId, leaseGeneration: primary.ownerLeaseGeneration };
|
|
2559
|
+
const store = this.requireLocationStore();
|
|
2560
|
+
for (;;) {
|
|
2561
|
+
if (this.disposed) throw new Error('Relocation runtime stopped.');
|
|
2562
|
+
await this.resendSourceCutover(window);
|
|
2563
|
+
const observed = await this.readSourceSettlement(envelope, primary, target).catch(
|
|
2564
|
+
() => undefined
|
|
2565
|
+
);
|
|
2566
|
+
if (observed?.kind === 'target') return 'target';
|
|
2567
|
+
if (observed?.kind === 'other') return 'lost';
|
|
2568
|
+
if (restoreDeadlineReached() && observed?.kind === 'source') {
|
|
2569
|
+
// An unreadable lease is an uncertain result, not an expiry.
|
|
2570
|
+
if (await this.exactSourceLeaseExpired(owner).catch(() => false)) return 'lost';
|
|
2571
|
+
const publication = this.codec.read(observed.current.payload);
|
|
2572
|
+
const preserved = await store
|
|
2573
|
+
.compareExchangeAuthority(
|
|
2574
|
+
{ value: primaryKey(envelope) } as ZLinkAuthorityKey,
|
|
2575
|
+
observed.current.storeVersion,
|
|
2576
|
+
{
|
|
2577
|
+
kind: 'put',
|
|
2578
|
+
generationTransition: 'preserve',
|
|
2579
|
+
payload:
|
|
2580
|
+
publication === undefined
|
|
2581
|
+
? observed.current.payload
|
|
2582
|
+
: this.codec.clear(observed.current.payload, publication.reference)
|
|
2583
|
+
}
|
|
2584
|
+
)
|
|
2585
|
+
.catch(() => undefined);
|
|
2586
|
+
if (preserved?.kind === 'stored') return 'source';
|
|
2587
|
+
}
|
|
2588
|
+
// The caller awaits this settlement, so the retry timer keeps the loop alive.
|
|
2589
|
+
await new Promise<void>((resolve) => setTimeout(resolve, 10));
|
|
2590
|
+
}
|
|
2591
|
+
}
|
|
2592
|
+
|
|
2593
|
+
private async readSourceSettlement(
|
|
2594
|
+
envelope: ServiceRelocationEnvelope,
|
|
2595
|
+
primary: ZLinkAuthoritySnapshot,
|
|
2596
|
+
target: ServiceWireRelocationTarget
|
|
2597
|
+
): Promise<
|
|
2598
|
+
| { readonly kind: 'target' }
|
|
2599
|
+
| { readonly kind: 'source'; readonly current: ZLinkAuthoritySnapshot }
|
|
2600
|
+
| { readonly kind: 'other' }
|
|
2601
|
+
> {
|
|
2602
|
+
const primaryAuthorityKey = primaryKey(envelope);
|
|
2603
|
+
let committed = 0;
|
|
2604
|
+
let primaryCurrent: ZLinkAuthoritySnapshot | undefined;
|
|
2605
|
+
for (const participant of envelope.participants) {
|
|
2606
|
+
const current = await this.requireLocationStore().readAuthority({
|
|
2607
|
+
value: participant.key
|
|
2608
|
+
} as ZLinkAuthorityKey);
|
|
2609
|
+
if (current.kind !== 'snapshot') return { kind: 'other' };
|
|
2610
|
+
if (
|
|
2611
|
+
current.objectGeneration === participant.objectGeneration &&
|
|
2612
|
+
current.ownerId === target.ownerId &&
|
|
2613
|
+
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
2614
|
+
current.authorityOwnerGeneration > participant.authorityOwnerGeneration &&
|
|
2615
|
+
String(current.allocation.descriptor.rid) === target.nodeRid &&
|
|
2616
|
+
current.allocation.descriptorLifecycleGeneration === target.nodeGeneration
|
|
2617
|
+
) {
|
|
2618
|
+
committed += 1;
|
|
2619
|
+
}
|
|
2620
|
+
if (participant.key === primaryAuthorityKey) primaryCurrent = current;
|
|
2621
|
+
}
|
|
2622
|
+
if (committed === envelope.participants.length) return { kind: 'target' };
|
|
2623
|
+
if (primaryCurrent !== undefined && isSourceOwner(primaryCurrent, primary)) {
|
|
2624
|
+
return { kind: 'source', current: primaryCurrent };
|
|
2625
|
+
}
|
|
2626
|
+
return { kind: 'other' };
|
|
2415
2627
|
}
|
|
2416
2628
|
|
|
2417
2629
|
/**
|
|
@@ -2525,6 +2737,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2525
2737
|
request.nodeInternalBoundSessions
|
|
2526
2738
|
);
|
|
2527
2739
|
validatePrepareEnvelope(request, inventoryEnvelope);
|
|
2740
|
+
const releaseActivation =
|
|
2741
|
+
(await this.options.activationAdmission?.acquire(meshName, signal)) ?? (() => undefined);
|
|
2528
2742
|
let materialized:
|
|
2529
2743
|
(Pick<LocalStage, 'owner' | 'staging'> & { readonly target: LocalTargetPort }) | undefined;
|
|
2530
2744
|
let reservation: TargetRelocationReservation | undefined;
|
|
@@ -2539,7 +2753,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2539
2753
|
meshName,
|
|
2540
2754
|
request,
|
|
2541
2755
|
inventoryEnvelope,
|
|
2542
|
-
materialized.target,
|
|
2543
2756
|
signal
|
|
2544
2757
|
);
|
|
2545
2758
|
const offer: TargetRelocationOffer = {
|
|
@@ -2547,7 +2760,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2547
2760
|
prepareFingerprint: fingerprint,
|
|
2548
2761
|
authenticatedSourceNodeRid: String(sourceNodeRid),
|
|
2549
2762
|
envelope: inventoryEnvelope,
|
|
2550
|
-
restoreDeadlineAtMs: performance.now() + RELOCATION_OPERATION_RETENTION_MS,
|
|
2551
2763
|
reservation
|
|
2552
2764
|
};
|
|
2553
2765
|
const stage: LocalStage = {
|
|
@@ -2556,10 +2768,12 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2556
2768
|
phase: 'ready',
|
|
2557
2769
|
lane: Promise.resolve(),
|
|
2558
2770
|
cutoverReceived: false,
|
|
2559
|
-
boundaryRelay: []
|
|
2771
|
+
boundaryRelay: [],
|
|
2772
|
+
releaseActivation
|
|
2560
2773
|
};
|
|
2561
2774
|
this.targetStages.set(stagingId, stage);
|
|
2562
2775
|
} catch (error) {
|
|
2776
|
+
releaseActivation();
|
|
2563
2777
|
if (reservation !== undefined) {
|
|
2564
2778
|
await this.abortTargetReservation(reservation, signal).catch(() => undefined);
|
|
2565
2779
|
}
|
|
@@ -2571,11 +2785,23 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2571
2785
|
return relocationReady(request);
|
|
2572
2786
|
}
|
|
2573
2787
|
|
|
2788
|
+
/**
|
|
2789
|
+
* The cutover wait of a Ready stage (spec 28 §4.4) is a Warning only: it
|
|
2790
|
+
* records `cutover_timeout` and never submits the target CAS or opens
|
|
2791
|
+
* dispatch. From then on the target settlement reads the Location Store so
|
|
2792
|
+
* a source Preserve fence or a lost target lease ends the staging; only a
|
|
2793
|
+
* verified cutover lets that settlement submit the CAS.
|
|
2794
|
+
*/
|
|
2574
2795
|
private armTargetCutoverFallback(meshName: string, stagingId: string): void {
|
|
2575
2796
|
const stage = this.targetStages.get(stagingId);
|
|
2576
2797
|
if (stage === undefined || stage.fallback !== undefined || stage.finalize !== undefined) return;
|
|
2577
2798
|
stage.fallback = setTimeout(() => {
|
|
2578
|
-
|
|
2799
|
+
if (stage.finalize !== undefined) return;
|
|
2800
|
+
console.warn('[zlink.runtime.relocation.cutover_timeout]', stagingId);
|
|
2801
|
+
this.options.metrics?.count('zlink.relocation.cutover_timeout', 1, {
|
|
2802
|
+
mesh_name: meshName
|
|
2803
|
+
});
|
|
2804
|
+
void this.beginTargetFinalize(meshName, stagingId, stage).catch((error) =>
|
|
2579
2805
|
console.error('[zlink.runtime.relocation.location_update_failed]', error)
|
|
2580
2806
|
);
|
|
2581
2807
|
}, this.relocationLimits().relocationCutoverWaitTimeoutMs);
|
|
@@ -2603,19 +2829,26 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2603
2829
|
throw new Error(`Relocation staging '${stagingId}' is missing.`);
|
|
2604
2830
|
}
|
|
2605
2831
|
validateTargetOneWayControl(stage, request, sourceNodeRid);
|
|
2832
|
+
// A verified cutover closes the boundary batch. A later record or cutover
|
|
2833
|
+
// is a duplicate retransmission and never changes the staged batch.
|
|
2834
|
+
if (stage.cutoverReceived || (stage.phase !== 'ready' && stage.phase !== 'finalizing')) {
|
|
2835
|
+
if (request.kind === 'cutover') {
|
|
2836
|
+
console.warn('[zlink.runtime.relocation.late_cutover]', stagingId);
|
|
2837
|
+
}
|
|
2838
|
+
return;
|
|
2839
|
+
}
|
|
2606
2840
|
if (request.kind === 'data') {
|
|
2607
2841
|
stage.lane = stage.lane.then(() => this.stageBoundaryRelay(stage, request));
|
|
2608
2842
|
await stage.lane;
|
|
2609
2843
|
return;
|
|
2610
2844
|
}
|
|
2611
|
-
if (stage.cutoverReceived || stage.phase !== 'ready') {
|
|
2612
|
-
console.warn('[zlink.runtime.relocation.late_cutover]', stagingId);
|
|
2613
|
-
return;
|
|
2614
|
-
}
|
|
2615
2845
|
this.reconcileBoundaryRelay(stage, request, stagingId);
|
|
2616
2846
|
stage.cutoverReceived = true;
|
|
2847
|
+
// A settlement started by the cutover wait Warning is still reading the
|
|
2848
|
+
// Store; it submits the CAS on its next round now that cutover verified.
|
|
2849
|
+
if (stage.finalize !== undefined) return;
|
|
2617
2850
|
if (stage.fallback !== undefined) clearTimeout(stage.fallback);
|
|
2618
|
-
await this.beginTargetFinalize(meshName, stagingId, stage,
|
|
2851
|
+
await this.beginTargetFinalize(meshName, stagingId, stage, signal);
|
|
2619
2852
|
}
|
|
2620
2853
|
|
|
2621
2854
|
/** Validates one boundary relay record and buffers it until cutover applies it. */
|
|
@@ -2643,15 +2876,15 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2643
2876
|
}
|
|
2644
2877
|
const expectedCount = Number(request.boundaryRecordCount);
|
|
2645
2878
|
const staged = stage.boundaryRelay;
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2879
|
+
// The batch is the last `boundaryRecordCount` records: the source sends
|
|
2880
|
+
// the whole batch right before this cutover on the ordered connection,
|
|
2881
|
+
// so a retransmission replaces an earlier partial span as a whole.
|
|
2882
|
+
if (staged.length >= expectedCount) {
|
|
2883
|
+
const batch = staged.slice(staged.length - expectedCount);
|
|
2884
|
+
if (
|
|
2885
|
+
crc32c(Buffer.concat(batch.map((value) => value.frozenRecord.canonicalBytes))) ===
|
|
2886
|
+
request.boundaryChecksumCrc32c
|
|
2887
|
+
) {
|
|
2655
2888
|
staged.splice(0, staged.length - expectedCount);
|
|
2656
2889
|
return;
|
|
2657
2890
|
}
|
|
@@ -2668,21 +2901,16 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2668
2901
|
meshName: string,
|
|
2669
2902
|
stagingId: string,
|
|
2670
2903
|
stage: LocalStage,
|
|
2671
|
-
fallback: boolean,
|
|
2672
2904
|
signal?: AbortSignal
|
|
2673
2905
|
): Promise<void> {
|
|
2674
2906
|
if (stage.finalize !== undefined) return stage.finalize;
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
mesh_name: meshName
|
|
2679
|
-
});
|
|
2680
|
-
}
|
|
2907
|
+
// Records staged before this point finish first. The settlement is not
|
|
2908
|
+
// put on the lane, so a cutover that arrives while it waits for the
|
|
2909
|
+
// authority result is still reconciled.
|
|
2681
2910
|
const finalize = stage.lane.then(() =>
|
|
2682
2911
|
this.finalizeTargetStage(meshName, stagingId, stage, signal)
|
|
2683
2912
|
);
|
|
2684
2913
|
stage.finalize = finalize;
|
|
2685
|
-
stage.lane = finalize;
|
|
2686
2914
|
return finalize;
|
|
2687
2915
|
}
|
|
2688
2916
|
|
|
@@ -2698,14 +2926,15 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2698
2926
|
let authority: ZLinkAuthoritySnapshot | undefined;
|
|
2699
2927
|
let resumeStartedAt: number | undefined;
|
|
2700
2928
|
try {
|
|
2701
|
-
// The boundary relay span is applied to the temporary queue before the
|
|
2702
|
-
// owner CAS so the merged order is saved work, pre-boundary relay, then
|
|
2703
|
-
// post-boundary temporary records (spec 28 §4.6).
|
|
2704
|
-
for (const relay of stage.boundaryRelay.splice(0)) {
|
|
2705
|
-
await this.applyRelocationData(stage, relay, signal);
|
|
2706
|
-
}
|
|
2707
2929
|
authority = await this.commitTargetReservation(stage, stage.offer.reservation, signal);
|
|
2930
|
+
if (authority === undefined) {
|
|
2931
|
+
// Source Preserve, a definitive conflict or the target lease ended
|
|
2932
|
+
// this staging before any target commit (Location runtime §10).
|
|
2933
|
+
console.warn('[zlink.runtime.relocation.target_staging_discarded]', stagingId);
|
|
2934
|
+
throw new Error(`Relocation '${stagingId}' target staging lost the authority settlement.`);
|
|
2935
|
+
}
|
|
2708
2936
|
authorityCommitted = true;
|
|
2937
|
+
stage.releaseActivation();
|
|
2709
2938
|
resumeStartedAt = performance.now();
|
|
2710
2939
|
stage.phase = 'committed';
|
|
2711
2940
|
await stage.owner.normalize(stage.staging, authority, signal);
|
|
@@ -2720,13 +2949,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2720
2949
|
}
|
|
2721
2950
|
}
|
|
2722
2951
|
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
2723
|
-
//
|
|
2724
|
-
//
|
|
2725
|
-
//
|
|
2726
|
-
//
|
|
2727
|
-
// roll back already committed target ownership or strand its staged
|
|
2728
|
-
// object: the next reservation recognizes the retained journal root and
|
|
2729
|
-
// continues its existing cleanup/retry path.
|
|
2952
|
+
// Clear the final relocation publication before admission becomes
|
|
2953
|
+
// externally visible so a later Join cannot reserve against the
|
|
2954
|
+
// previous root. A bounded CAS conflict must not roll back already
|
|
2955
|
+
// committed target ownership or strand its staged object.
|
|
2730
2956
|
try {
|
|
2731
2957
|
await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
2732
2958
|
} catch (error) {
|
|
@@ -2941,41 +3167,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2941
3167
|
}
|
|
2942
3168
|
}
|
|
2943
3169
|
|
|
2944
|
-
private async waitForTargetAuthorities(
|
|
2945
|
-
envelope: ServiceRelocationEnvelope,
|
|
2946
|
-
target: ServiceWireRelocationTarget,
|
|
2947
|
-
deadlineAtMs: number,
|
|
2948
|
-
signal?: AbortSignal
|
|
2949
|
-
): Promise<void> {
|
|
2950
|
-
for (;;) {
|
|
2951
|
-
signal?.throwIfAborted();
|
|
2952
|
-
let exact = true;
|
|
2953
|
-
for (const participant of envelope.participants) {
|
|
2954
|
-
const current = await this.requireLocationStore().readAuthority(
|
|
2955
|
-
{ value: participant.key } as ZLinkAuthorityKey,
|
|
2956
|
-
signal
|
|
2957
|
-
);
|
|
2958
|
-
if (
|
|
2959
|
-
current.kind !== 'snapshot' ||
|
|
2960
|
-
current.objectGeneration !== participant.objectGeneration ||
|
|
2961
|
-
current.ownerId !== target.ownerId ||
|
|
2962
|
-
current.ownerLeaseGeneration !== target.ownerLeaseGeneration ||
|
|
2963
|
-
current.authorityOwnerGeneration <= participant.authorityOwnerGeneration ||
|
|
2964
|
-
String(current.allocation.descriptor.rid) !== target.nodeRid ||
|
|
2965
|
-
current.allocation.descriptorLifecycleGeneration !== target.nodeGeneration
|
|
2966
|
-
) {
|
|
2967
|
-
exact = false;
|
|
2968
|
-
break;
|
|
2969
|
-
}
|
|
2970
|
-
}
|
|
2971
|
-
if (exact) return;
|
|
2972
|
-
if (performance.now() >= deadlineAtMs) {
|
|
2973
|
-
throw new Error('Relocation target owner was not confirmed before its original deadline.');
|
|
2974
|
-
}
|
|
2975
|
-
await new Promise<void>((resolve) => setTimeout(resolve, 10));
|
|
2976
|
-
}
|
|
2977
|
-
}
|
|
2978
|
-
|
|
2979
3170
|
private async clearTargetRelocationPublication(
|
|
2980
3171
|
stage: LocalStage,
|
|
2981
3172
|
primary: ZLinkAuthoritySnapshot,
|
|
@@ -2986,36 +3177,25 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2986
3177
|
stage.staging.primaryAuthorityKey.value,
|
|
2987
3178
|
signal
|
|
2988
3179
|
);
|
|
2989
|
-
// The
|
|
2990
|
-
//
|
|
2991
|
-
//
|
|
2992
|
-
//
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
},
|
|
3008
|
-
signal
|
|
3009
|
-
);
|
|
3010
|
-
if (result.kind === 'stored') return;
|
|
3011
|
-
const read = await this.requireLocationStore().readAuthority(
|
|
3012
|
-
stage.staging.primaryAuthorityKey,
|
|
3013
|
-
signal
|
|
3014
|
-
);
|
|
3015
|
-
if (read.kind !== 'snapshot') break;
|
|
3016
|
-
current = read;
|
|
3180
|
+
// The target owner is the only writer of the committed primary
|
|
3181
|
+
// authority between its commit and this clear, so one CAS against the
|
|
3182
|
+
// committed snapshot decides. A conflict means another owner changed the
|
|
3183
|
+
// row; the caller records it and leaves the committed ownership intact.
|
|
3184
|
+
const publication = this.codec.read(primary.payload);
|
|
3185
|
+
if (publication === undefined) return;
|
|
3186
|
+
const result = await this.requireLocationStore().compareExchangeAuthority(
|
|
3187
|
+
stage.staging.primaryAuthorityKey,
|
|
3188
|
+
primary.storeVersion,
|
|
3189
|
+
{
|
|
3190
|
+
kind: 'put',
|
|
3191
|
+
generationTransition: 'preserve',
|
|
3192
|
+
payload: this.codec.clear(primary.payload, publication.reference)
|
|
3193
|
+
},
|
|
3194
|
+
signal
|
|
3195
|
+
);
|
|
3196
|
+
if (result.kind !== 'stored') {
|
|
3197
|
+
throw new Error('Relocation target authority normalization CAS failed.');
|
|
3017
3198
|
}
|
|
3018
|
-
throw new Error('Relocation target authority normalization CAS failed.');
|
|
3019
3199
|
}
|
|
3020
3200
|
|
|
3021
3201
|
private async handleReplyRelay(
|
|
@@ -3269,7 +3449,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3269
3449
|
}
|
|
3270
3450
|
|
|
3271
3451
|
private async exactSourceLeaseExpired(
|
|
3272
|
-
source: ServiceWireRequestSourceFence,
|
|
3452
|
+
source: Pick<ServiceWireRequestSourceFence, 'ownerId' | 'leaseGeneration'>,
|
|
3273
3453
|
signal?: AbortSignal
|
|
3274
3454
|
): Promise<boolean> {
|
|
3275
3455
|
const lease = await this.requireLocationStore().readOwnerLease(source.ownerId, signal);
|
|
@@ -3436,51 +3616,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3436
3616
|
meshName: string,
|
|
3437
3617
|
prepare: ServiceMaintenanceRelocationPrepare,
|
|
3438
3618
|
envelope: ServiceRelocationEnvelope,
|
|
3439
|
-
targetPort: LocalTargetPort,
|
|
3440
3619
|
signal?: AbortSignal
|
|
3441
3620
|
): Promise<TargetRelocationReservation> {
|
|
3442
3621
|
const authorities = await this.readTargetParticipantAuthorities(envelope, signal);
|
|
3443
3622
|
const publication = relocationPublication(prepare, envelope);
|
|
3444
|
-
// Carries the exact reference + CRC of a deferred-Join publication the
|
|
3445
|
-
// target reservation must keep connected through the CAS (spec 28 —
|
|
3446
|
-
// clearing/replacing it here would strand the journal's own
|
|
3447
|
-
// prepared->committed transition with nothing left for it to find).
|
|
3448
|
-
const retainedDeferredJoinRoots = new Map<
|
|
3449
|
-
string,
|
|
3450
|
-
{ readonly reference: string; readonly checksumCrc32c: number }
|
|
3451
|
-
>();
|
|
3452
|
-
for (const participant of envelope.participants) {
|
|
3453
|
-
const staged = targetPort.deferredJoinRoot(participant.key);
|
|
3454
|
-
if (staged !== undefined) {
|
|
3455
|
-
retainedDeferredJoinRoots.set(participant.key, {
|
|
3456
|
-
reference: staged.reference.value,
|
|
3457
|
-
checksumCrc32c: staged.checksumCrc32c
|
|
3458
|
-
});
|
|
3459
|
-
continue;
|
|
3460
|
-
}
|
|
3461
|
-
const current = this.codec.read(authorities.get(participant.key)!.payload);
|
|
3462
|
-
if (
|
|
3463
|
-
participant.objectKind === 'actor' &&
|
|
3464
|
-
current?.canonical === true &&
|
|
3465
|
-
(await this.options.actorTransfer.isDeferredJoinAcceptedRootPublication(
|
|
3466
|
-
current.reference,
|
|
3467
|
-
current.checksumCrc32c,
|
|
3468
|
-
{
|
|
3469
|
-
authorityKey: participant.key,
|
|
3470
|
-
objectKind: 'actor',
|
|
3471
|
-
objectGeneration: participant.objectGeneration,
|
|
3472
|
-
aggregateId: current.aggregateId,
|
|
3473
|
-
aggregateGeneration: current.aggregateGeneration
|
|
3474
|
-
},
|
|
3475
|
-
signal
|
|
3476
|
-
))
|
|
3477
|
-
) {
|
|
3478
|
-
retainedDeferredJoinRoots.set(participant.key, {
|
|
3479
|
-
reference: current.reference,
|
|
3480
|
-
checksumCrc32c: current.checksumCrc32c
|
|
3481
|
-
});
|
|
3482
|
-
}
|
|
3483
|
-
}
|
|
3484
3623
|
const participants = envelope.participants.map((participant) => {
|
|
3485
3624
|
const expected = authorities.get(participant.key)!;
|
|
3486
3625
|
const membership = actorMembershipTarget(envelope, participant.key);
|
|
@@ -3495,48 +3634,34 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3495
3634
|
) {
|
|
3496
3635
|
throw new Error(`Relocation target RouteMesh '${meshName}' has no Entry Spot.`);
|
|
3497
3636
|
}
|
|
3498
|
-
const retainedDeferredJoinRoot = retainedDeferredJoinRoots.get(participant.key);
|
|
3499
|
-
const participantPublication =
|
|
3500
|
-
retainedDeferredJoinRoot === undefined
|
|
3501
|
-
? publication
|
|
3502
|
-
: {
|
|
3503
|
-
...publication,
|
|
3504
|
-
reference: retainedDeferredJoinRoot.reference,
|
|
3505
|
-
checksumCrc32c: retainedDeferredJoinRoot.checksumCrc32c
|
|
3506
|
-
};
|
|
3507
3637
|
return {
|
|
3508
3638
|
key: { value: participant.key } as ZLinkAuthorityKey,
|
|
3509
3639
|
expected,
|
|
3510
3640
|
ownerTransition: 'newOwner' as const,
|
|
3511
|
-
authorityPayload: this.authorityPayloadForPublication(
|
|
3512
|
-
|
|
3513
|
-
|
|
3514
|
-
|
|
3515
|
-
owner: {
|
|
3516
|
-
ownerId: prepare.target.ownerId,
|
|
3517
|
-
leaseGeneration: prepare.target.ownerLeaseGeneration
|
|
3518
|
-
},
|
|
3519
|
-
meshName,
|
|
3520
|
-
nodeRid: prepare.target.nodeRid,
|
|
3521
|
-
nodeGeneration: prepare.target.nodeGeneration,
|
|
3522
|
-
objectGeneration: expected.objectGeneration,
|
|
3523
|
-
targetAttemptGeneration: prepare.targetAttemptGeneration,
|
|
3524
|
-
coordinatorExpectedStoreVersion: expected.storeVersion.value,
|
|
3525
|
-
...(membership === undefined
|
|
3526
|
-
? expected.allocation.objectKind === 'actor'
|
|
3527
|
-
? {
|
|
3528
|
-
actorSpotId: entrySpotId!,
|
|
3529
|
-
actorSpotGeneration: prepare.target.nodeGeneration,
|
|
3530
|
-
actorSpotKind: ZLinkSpotKind.Entry as const
|
|
3531
|
-
}
|
|
3532
|
-
: {}
|
|
3533
|
-
: {
|
|
3534
|
-
...membership,
|
|
3535
|
-
actorSpotKind: ZLinkSpotKind.User as const
|
|
3536
|
-
})
|
|
3641
|
+
authorityPayload: this.authorityPayloadForPublication(expected.payload, publication, {
|
|
3642
|
+
owner: {
|
|
3643
|
+
ownerId: prepare.target.ownerId,
|
|
3644
|
+
leaseGeneration: prepare.target.ownerLeaseGeneration
|
|
3537
3645
|
},
|
|
3538
|
-
|
|
3539
|
-
|
|
3646
|
+
meshName,
|
|
3647
|
+
nodeRid: prepare.target.nodeRid,
|
|
3648
|
+
nodeGeneration: prepare.target.nodeGeneration,
|
|
3649
|
+
objectGeneration: expected.objectGeneration,
|
|
3650
|
+
targetAttemptGeneration: prepare.targetAttemptGeneration,
|
|
3651
|
+
coordinatorExpectedStoreVersion: expected.storeVersion.value,
|
|
3652
|
+
...(membership === undefined
|
|
3653
|
+
? expected.allocation.objectKind === 'actor'
|
|
3654
|
+
? {
|
|
3655
|
+
actorSpotId: entrySpotId!,
|
|
3656
|
+
actorSpotGeneration: prepare.target.nodeGeneration,
|
|
3657
|
+
actorSpotKind: ZLinkSpotKind.Entry as const
|
|
3658
|
+
}
|
|
3659
|
+
: {}
|
|
3660
|
+
: {
|
|
3661
|
+
...membership,
|
|
3662
|
+
actorSpotKind: ZLinkSpotKind.User as const
|
|
3663
|
+
})
|
|
3664
|
+
}),
|
|
3540
3665
|
membershipMutation: encodeMembershipMutation(envelope.memberships, participant.key)
|
|
3541
3666
|
};
|
|
3542
3667
|
});
|
|
@@ -3576,24 +3701,16 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3576
3701
|
readonly actorSpotId?: string;
|
|
3577
3702
|
readonly actorSpotGeneration?: bigint;
|
|
3578
3703
|
readonly actorSpotKind?: ZLinkSpotKind.Entry | ZLinkSpotKind.User;
|
|
3579
|
-
}
|
|
3580
|
-
replacesDeferredJoinRoot = false
|
|
3704
|
+
}
|
|
3581
3705
|
): Uint8Array {
|
|
3582
3706
|
const existing = this.codec.read(payload);
|
|
3583
3707
|
const canonicalIdentity = serviceRelocationAuthoritySlotIdentity(payload);
|
|
3584
|
-
const replacesCanonicalJournal = replacesDeferredJoinRoot && existing?.canonical === true;
|
|
3585
3708
|
if (
|
|
3586
3709
|
canonicalIdentity !== undefined &&
|
|
3587
3710
|
(existing === undefined ||
|
|
3588
|
-
(existing.canonical === true &&
|
|
3589
|
-
(existing.aggregateId === publication.aggregateId || replacesCanonicalJournal)))
|
|
3711
|
+
(existing.canonical === true && existing.aggregateId === publication.aggregateId))
|
|
3590
3712
|
) {
|
|
3591
|
-
const published =
|
|
3592
|
-
existing === undefined
|
|
3593
|
-
? this.codec.publish(payload, publication)
|
|
3594
|
-
: replacesCanonicalJournal && existing.aggregateId !== publication.aggregateId
|
|
3595
|
-
? this.codec.publish(this.codec.clear(payload, existing.reference), publication)
|
|
3596
|
-
: payload;
|
|
3713
|
+
const published = existing === undefined ? this.codec.publish(payload, publication) : payload;
|
|
3597
3714
|
if (target === undefined) return published;
|
|
3598
3715
|
return projectServiceRelocationAuthorityTargetReady(
|
|
3599
3716
|
replaceServiceRelocationAuthorityApplicationPayload(
|
|
@@ -3665,6 +3782,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3665
3782
|
|
|
3666
3783
|
private async abortTargetStage(stage: LocalStage, signal?: AbortSignal): Promise<void> {
|
|
3667
3784
|
if (stage.fallback !== undefined) clearTimeout(stage.fallback);
|
|
3785
|
+
stage.releaseActivation();
|
|
3668
3786
|
const failures: unknown[] = [];
|
|
3669
3787
|
try {
|
|
3670
3788
|
await this.abortTargetReservation(stage.offer.reservation, signal);
|
|
@@ -3687,29 +3805,46 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3687
3805
|
stage: LocalStage,
|
|
3688
3806
|
reservation: TargetRelocationReservation,
|
|
3689
3807
|
signal?: AbortSignal
|
|
3690
|
-
): Promise<ZLinkAuthoritySnapshot> {
|
|
3808
|
+
): Promise<ZLinkAuthoritySnapshot | undefined> {
|
|
3691
3809
|
return await this.commitTargetAggregate(stage, reservation.prepared, signal);
|
|
3692
3810
|
}
|
|
3693
3811
|
|
|
3812
|
+
/**
|
|
3813
|
+
* The one target authority settlement (Location runtime §10, spec 28
|
|
3814
|
+
* §4.4-§4.5). Only a verified cutover lets the target submit its NewOwner
|
|
3815
|
+
* CAS; a transient or indeterminate result resubmits the same RelocationId
|
|
3816
|
+
* and expected StoreVersion while the target owner lease is valid (no
|
|
3817
|
+
* separate timeout, no fixed count). Before a verified cutover the target
|
|
3818
|
+
* only reads. It returns the committed authority, or undefined when a
|
|
3819
|
+
* definitive Conflict excludes this commit after reconciliation, a source
|
|
3820
|
+
* Preserve fence moved the source row, or the target lease ended; the
|
|
3821
|
+
* caller then discards the staging.
|
|
3822
|
+
*/
|
|
3694
3823
|
private async commitTargetAggregate(
|
|
3695
3824
|
stage: LocalStage,
|
|
3696
3825
|
prepared: ServicePreparedRelocationAggregate,
|
|
3697
3826
|
signal?: AbortSignal
|
|
3698
|
-
): Promise<ZLinkAuthoritySnapshot> {
|
|
3699
|
-
|
|
3827
|
+
): Promise<ZLinkAuthoritySnapshot | undefined> {
|
|
3828
|
+
const target = stage.offer.prepare.target;
|
|
3700
3829
|
for (;;) {
|
|
3701
3830
|
signal?.throwIfAborted();
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
)
|
|
3831
|
+
if (this.disposed) throw new Error('Relocation runtime stopped.');
|
|
3832
|
+
let conflict = false;
|
|
3833
|
+
if (stage.cutoverReceived) {
|
|
3834
|
+
// The boundary relay span is applied to the temporary queue before
|
|
3835
|
+
// the owner CAS so the merged order is saved work, pre-boundary relay,
|
|
3836
|
+
// then post-boundary temporary records (spec 28 §4.6).
|
|
3837
|
+
for (const relay of stage.boundaryRelay.splice(0)) {
|
|
3838
|
+
await this.applyRelocationData(stage, relay, signal);
|
|
3839
|
+
}
|
|
3840
|
+
// A thrown provider call has an unknown result; the read below decides.
|
|
3841
|
+
const result = await this.requireLocationStore()
|
|
3842
|
+
.commitAggregate(prepared.fence, signal)
|
|
3843
|
+
.catch(() => undefined);
|
|
3844
|
+
if (result?.kind === 'generationExhausted') {
|
|
3845
|
+
throw new Error('location_update_failed: relocation aggregate commit exhausted.');
|
|
3846
|
+
}
|
|
3847
|
+
conflict = result?.kind === 'stale';
|
|
3713
3848
|
}
|
|
3714
3849
|
const observed = await this.readAggregateForCommitRetry(prepared, signal);
|
|
3715
3850
|
if (observed.kind === 'committed') {
|
|
@@ -3719,15 +3854,17 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3719
3854
|
}
|
|
3720
3855
|
return primary;
|
|
3721
3856
|
}
|
|
3722
|
-
if (observed.kind === 'stale')
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
)
|
|
3857
|
+
if (observed.kind === 'stale' || (conflict && observed.kind === 'source')) return undefined;
|
|
3858
|
+
// An unreadable lease is an uncertain result, not an expiry.
|
|
3859
|
+
if (
|
|
3860
|
+
await this.exactSourceLeaseExpired(
|
|
3861
|
+
{ ownerId: target.ownerId, leaseGeneration: target.ownerLeaseGeneration },
|
|
3862
|
+
signal
|
|
3863
|
+
).catch(() => false)
|
|
3864
|
+
) {
|
|
3865
|
+
return undefined;
|
|
3729
3866
|
}
|
|
3730
|
-
await
|
|
3867
|
+
await waitForRelocationRetry(25, signal);
|
|
3731
3868
|
}
|
|
3732
3869
|
}
|
|
3733
3870
|
|
|
@@ -4095,6 +4232,26 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4095
4232
|
return value;
|
|
4096
4233
|
}
|
|
4097
4234
|
|
|
4235
|
+
private afterSpotCloseDecision<T>(
|
|
4236
|
+
meshName: string,
|
|
4237
|
+
spotId: RoutingId,
|
|
4238
|
+
seal: () => T
|
|
4239
|
+
): T | Promise<T> {
|
|
4240
|
+
const manager = this.requireSpotManager();
|
|
4241
|
+
const sealAfterDecision = (): T => {
|
|
4242
|
+
if (manager.isSpotClosing(meshName, spotId)) {
|
|
4243
|
+
throw createInternalFrameworkException(
|
|
4244
|
+
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
4245
|
+
`Spot '${String(spotId)}' is closing.`,
|
|
4246
|
+
true
|
|
4247
|
+
);
|
|
4248
|
+
}
|
|
4249
|
+
return seal();
|
|
4250
|
+
};
|
|
4251
|
+
const decision = manager.pendingSpotCloseDecision(meshName, spotId);
|
|
4252
|
+
return decision === undefined ? sealAfterDecision() : decision.then(sealAfterDecision);
|
|
4253
|
+
}
|
|
4254
|
+
|
|
4098
4255
|
private requireActorManager(): DefaultZLinkActorManager {
|
|
4099
4256
|
const value = this.options.actorManager();
|
|
4100
4257
|
if (value === undefined) throw new Error('Host relocation requires the Actor manager.');
|
|
@@ -4237,8 +4394,6 @@ function relocationDebug(marker: string, detail: Record<string, unknown>): void
|
|
|
4237
4394
|
}
|
|
4238
4395
|
|
|
4239
4396
|
class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden> {
|
|
4240
|
-
private readonly deferredJoinRoots = new Map<string, ZLinkDeferredJoinAcceptedRoot>();
|
|
4241
|
-
|
|
4242
4397
|
constructor(
|
|
4243
4398
|
private readonly options: ZLinkHostRelocationOptions,
|
|
4244
4399
|
private readonly meshName: string,
|
|
@@ -4464,14 +4619,7 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4464
4619
|
if (recovery.request.actorId !== hidden.actor.context.actorId) {
|
|
4465
4620
|
throw new Error('Canonical Actor Join recovery names a different staged Actor.');
|
|
4466
4621
|
}
|
|
4467
|
-
|
|
4468
|
-
recovery,
|
|
4469
|
-
undefined,
|
|
4470
|
-
inventoryDigest(this.envelope.participants, this.envelope.memberships)
|
|
4471
|
-
);
|
|
4472
|
-
if (deferred !== undefined) {
|
|
4473
|
-
this.deferredJoinRoots.set(hidden.authorityKey, deferred);
|
|
4474
|
-
}
|
|
4622
|
+
await this.requireSpotManager().restoreCanonicalActorJoinRecovery(recovery);
|
|
4475
4623
|
return;
|
|
4476
4624
|
}
|
|
4477
4625
|
const packet = decodeQueuedHandoffPacket(message);
|
|
@@ -4480,10 +4628,6 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4480
4628
|
hidden.replayPackets.push(packet);
|
|
4481
4629
|
}
|
|
4482
4630
|
|
|
4483
|
-
deferredJoinRoot(authorityKey: string): ZLinkDeferredJoinAcceptedRoot | undefined {
|
|
4484
|
-
return this.deferredJoinRoots.get(authorityKey);
|
|
4485
|
-
}
|
|
4486
|
-
|
|
4487
4631
|
async openAdmission(hidden: LocalHidden): Promise<void> {
|
|
4488
4632
|
if (hidden.actor === undefined) return;
|
|
4489
4633
|
this.requireActorManager().publishRelocationActor(hidden.actor.context.actorId);
|
|
@@ -4971,7 +5115,13 @@ function isExactSourceAuthority(
|
|
|
4971
5115
|
expected: ZLinkAuthoritySnapshot
|
|
4972
5116
|
): boolean {
|
|
4973
5117
|
return (
|
|
4974
|
-
current.storeVersion.value === expected.storeVersion.value &&
|
|
5118
|
+
current.storeVersion.value === expected.storeVersion.value && isSourceOwner(current, expected)
|
|
5119
|
+
);
|
|
5120
|
+
}
|
|
5121
|
+
|
|
5122
|
+
/** The same source tenure owns the row, at any StoreVersion. */
|
|
5123
|
+
function isSourceOwner(current: ZLinkAuthoritySnapshot, expected: ZLinkAuthoritySnapshot): boolean {
|
|
5124
|
+
return (
|
|
4975
5125
|
current.objectGeneration === expected.objectGeneration &&
|
|
4976
5126
|
current.authorityOwnerGeneration === expected.authorityOwnerGeneration &&
|
|
4977
5127
|
current.ownerId === expected.ownerId &&
|
|
@@ -5002,36 +5152,6 @@ function isExactCommittedTarget(
|
|
|
5002
5152
|
);
|
|
5003
5153
|
}
|
|
5004
5154
|
|
|
5005
|
-
async function waitForLocationRetry(
|
|
5006
|
-
deadlineAtMs: number,
|
|
5007
|
-
signal: AbortSignal | undefined,
|
|
5008
|
-
cause: unknown
|
|
5009
|
-
): Promise<void> {
|
|
5010
|
-
signal?.throwIfAborted();
|
|
5011
|
-
if (performance.now() >= deadlineAtMs) {
|
|
5012
|
-
throw new Error(
|
|
5013
|
-
'location_update_failed: relocation Restore validity expired before exact target owner confirmation.',
|
|
5014
|
-
{ cause }
|
|
5015
|
-
);
|
|
5016
|
-
}
|
|
5017
|
-
await new Promise<void>((resolve, reject) => {
|
|
5018
|
-
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
5019
|
-
const aborted = () => {
|
|
5020
|
-
if (timer !== undefined) clearTimeout(timer);
|
|
5021
|
-
reject(signal?.reason);
|
|
5022
|
-
};
|
|
5023
|
-
signal?.addEventListener('abort', aborted, { once: true });
|
|
5024
|
-
timer = setTimeout(
|
|
5025
|
-
() => {
|
|
5026
|
-
signal?.removeEventListener('abort', aborted);
|
|
5027
|
-
resolve();
|
|
5028
|
-
},
|
|
5029
|
-
Math.min(25, Math.max(1, deadlineAtMs - performance.now()))
|
|
5030
|
-
);
|
|
5031
|
-
timer.unref();
|
|
5032
|
-
});
|
|
5033
|
-
}
|
|
5034
|
-
|
|
5035
5155
|
async function waitForRelocationRetry(delayMs: number, signal?: AbortSignal): Promise<void> {
|
|
5036
5156
|
signal?.throwIfAborted();
|
|
5037
5157
|
await new Promise<void>((resolve, reject) => {
|
|
@@ -5045,7 +5165,6 @@ async function waitForRelocationRetry(delayMs: number, signal?: AbortSignal): Pr
|
|
|
5045
5165
|
signal?.removeEventListener('abort', aborted);
|
|
5046
5166
|
resolve();
|
|
5047
5167
|
}, delayMs);
|
|
5048
|
-
timer.unref();
|
|
5049
5168
|
});
|
|
5050
5169
|
}
|
|
5051
5170
|
|