@zlink-systems/framework 0.30.1 → 0.30.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/contracts/Common/ZLinkEncodedPayload.d.ts +6 -0
- package/dist/contracts/Common/ZLinkEncodedPayload.js +18 -3
- package/dist/contracts/Common/encoded-payload-storage.d.ts +1 -1
- package/dist/contracts/Common/encoded-payload-storage.js +5 -2
- package/dist/contracts/Configuration/Builders.d.ts +1 -1
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.d.ts +2 -1
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.js +5 -1
- package/dist/contracts/Configuration/RegistrationBuilders.js +2 -2
- package/dist/contracts/Configuration/RegistrationTypes.d.ts +1 -0
- package/dist/contracts/Dispatch/ZLinkDispatchOptions.d.ts +2 -2
- package/dist/contracts/Locations/Rows.d.ts +2 -0
- package/dist/contracts/Spots/Contracts.d.ts +4 -0
- package/dist/contracts/Spots/ZLinkSpot.d.ts +3 -3
- package/dist/runtime/abort.d.ts +5 -0
- package/dist/runtime/abort.js +10 -0
- package/dist/runtime/actors/actor-authority-publication.js +3 -2
- package/dist/runtime/actors/actor-client.d.ts +0 -1
- package/dist/runtime/actors/actor-client.js +38 -45
- package/dist/runtime/actors/actor-context.d.ts +2 -0
- package/dist/runtime/actors/actor-context.js +46 -9
- package/dist/runtime/actors/actor-handoff.d.ts +7 -3
- package/dist/runtime/actors/actor-handoff.js +212 -88
- package/dist/runtime/actors/actor-join-relocation.d.ts +2 -0
- package/dist/runtime/actors/actor-local-native-join.js +4 -0
- package/dist/runtime/actors/actor-mailbox.js +27 -1
- package/dist/runtime/actors/actor-remote-joiner.js +3 -1
- package/dist/runtime/actors/actor-reply.d.ts +4 -0
- package/dist/runtime/actors/actor-reply.js +29 -0
- package/dist/runtime/actors/actor-runtime-contracts.d.ts +2 -0
- package/dist/runtime/actors/actor-runtime-state.js +2 -20
- package/dist/runtime/actors/actor-source-transfer.d.ts +1 -17
- package/dist/runtime/actors/index.d.ts +6 -2
- package/dist/runtime/actors/index.js +37 -21
- package/dist/runtime/actors/session-disconnected-wire.d.ts +8 -0
- package/dist/runtime/actors/session-disconnected-wire.js +48 -0
- package/dist/runtime/admission.d.ts +9 -1
- package/dist/runtime/admission.js +55 -32
- package/dist/runtime/application-jobs/application-job-queue-scope.d.ts +13 -0
- package/dist/runtime/application-jobs/application-job-queue-scope.js +27 -1
- package/dist/runtime/application-jobs/contracts.d.ts +2 -0
- package/dist/runtime/backend/contracts/index.d.ts +19 -3
- package/dist/runtime/backend/mesh-actor-session-node-adapter.js +2 -2
- package/dist/runtime/backend/mesh-completion-table.js +5 -0
- package/dist/runtime/backend/mesh-dispatch-pump.d.ts +26 -2
- package/dist/runtime/backend/mesh-dispatch-pump.js +193 -107
- package/dist/runtime/backend/node/node-raw-binding-port.js +90 -58
- package/dist/runtime/backend/node/node-raw-mesh-backend.d.ts +33 -8
- package/dist/runtime/backend/node/node-raw-mesh-backend.js +142 -143
- package/dist/runtime/backend/node/node-socket-backend-adapter.d.ts +1 -1
- package/dist/runtime/backend/node/node-socket-backend-adapter.js +5 -5
- package/dist/runtime/channels/channel-clients.d.ts +1 -0
- package/dist/runtime/channels/channel-clients.js +20 -5
- package/dist/runtime/channels/channel-dispatch-pipeline.d.ts +22 -3
- package/dist/runtime/channels/channel-dispatch-pipeline.js +103 -57
- package/dist/runtime/channels/channel-dispatchers.d.ts +3 -2
- package/dist/runtime/channels/channel-dispatchers.js +44 -37
- package/dist/runtime/channels/channel-envelope.d.ts +1 -0
- package/dist/runtime/channels/channel-envelope.js +74 -31
- package/dist/runtime/channels/channel-runtime-lifecycle.d.ts +4 -3
- package/dist/runtime/channels/channel-runtime-lifecycle.js +19 -24
- package/dist/runtime/channels/channel-runtime-manager.d.ts +3 -2
- package/dist/runtime/channels/channel-runtime-manager.js +4 -4
- package/dist/runtime/channels/channel-transports.d.ts +0 -1
- package/dist/runtime/channels/channel-transports.js +35 -33
- package/dist/runtime/channels/client-server-location-runtime.js +5 -4
- package/dist/runtime/channels/dispatch-error-reporter.d.ts +2 -1
- package/dist/runtime/channels/dispatch-error-reporter.js +2 -1
- package/dist/runtime/channels/fanout-location-runtime.d.ts +1 -1
- package/dist/runtime/channels/fanout-location-runtime.js +3 -3
- package/dist/runtime/diagnostics/flow-context.js +19 -7
- package/dist/runtime/diagnostics/index.js +4 -9
- package/dist/runtime/diagnostics/message-flow.d.ts +8 -0
- package/dist/runtime/diagnostics/message-flow.js +35 -5
- package/dist/runtime/diagnostics/runtime-metrics.d.ts +2 -2
- package/dist/runtime/diagnostics/runtime-metrics.js +10 -8
- package/dist/runtime/execution/index.d.ts +4 -2
- package/dist/runtime/execution/index.js +33 -10
- package/dist/runtime/execution/serial-execution-queue.d.ts +4 -0
- package/dist/runtime/execution/serial-execution-queue.js +21 -2
- package/dist/runtime/execution/state-lane.d.ts +2 -0
- package/dist/runtime/execution/state-lane.js +5 -0
- package/dist/runtime/foundation/operation-registry.d.ts +25 -1
- package/dist/runtime/foundation/operation-registry.js +86 -18
- package/dist/runtime/foundation/raw-service-mesh-runtime.d.ts +39 -3
- package/dist/runtime/foundation/raw-service-mesh-runtime.js +151 -55
- package/dist/runtime/foundation/service-liveness-registry.d.ts +12 -1
- package/dist/runtime/foundation/service-liveness-registry.js +38 -0
- package/dist/runtime/foundation/service-mailbox.d.ts +7 -0
- package/dist/runtime/foundation/service-maintenance-runtime.js +1 -6
- package/dist/runtime/foundation/service-relocation-object-owner.d.ts +2 -3
- package/dist/runtime/foundation/service-relocation-object-owner.js +48 -35
- package/dist/runtime/foundation/service-relocation-runtime.js +7 -3
- package/dist/runtime/foundation/service-runtime-contracts.d.ts +8 -1
- package/dist/runtime/foundation/service-session-binding-ingress-port.d.ts +2 -1
- package/dist/runtime/foundation/service-stateful-registry.d.ts +5 -4
- package/dist/runtime/foundation/service-stateful-registry.js +24 -26
- package/dist/runtime/foundation/service-stateful-runtime.d.ts +72 -6
- package/dist/runtime/foundation/service-stateful-runtime.js +444 -117
- package/dist/runtime/foundation/service-stateful-wire-codec.d.ts +14 -1
- package/dist/runtime/foundation/service-stateful-wire-codec.js +48 -27
- package/dist/runtime/foundation/service-wire-binary-primitives.d.ts +6 -1
- package/dist/runtime/foundation/service-wire-binary-primitives.js +27 -11
- package/dist/runtime/foundation/service-wire-constants.generated.d.ts +1 -0
- package/dist/runtime/foundation/service-wire-constants.generated.js +1 -0
- package/dist/runtime/foundation/service-wire-m6a-codec.d.ts +10 -0
- package/dist/runtime/foundation/service-wire-m6a-codec.js +166 -53
- package/dist/runtime/host/actor-packet-relay.d.ts +6 -4
- package/dist/runtime/host/actor-packet-relay.js +59 -72
- package/dist/runtime/host/actor-placement-coordinator.js +25 -8
- package/dist/runtime/host/actor-transfer-runtime.d.ts +7 -31
- package/dist/runtime/host/actor-transfer-runtime.js +85 -331
- package/dist/runtime/host/application-job-queue.d.ts +24 -3
- package/dist/runtime/host/application-job-queue.js +28 -41
- package/dist/runtime/host/index.d.ts +5 -1
- package/dist/runtime/host/index.js +171 -97
- package/dist/runtime/host/location-runtime-owner.js +2 -6
- package/dist/runtime/host/mesh-router-resolver.d.ts +4 -0
- package/dist/runtime/host/mesh-router-resolver.js +14 -6
- package/dist/runtime/host/remote-bound-session-relay.d.ts +2 -4
- package/dist/runtime/host/remote-bound-session-relay.js +71 -190
- package/dist/runtime/host/route-mesh-runtime.d.ts +2 -2
- package/dist/runtime/host/route-mesh-runtime.js +5 -3
- package/dist/runtime/host/runtime-shutdown.d.ts +1 -0
- package/dist/runtime/host/runtime-shutdown.js +3 -2
- package/dist/runtime/host/service-relocation-host-runtime.d.ts +17 -7
- package/dist/runtime/host/service-relocation-host-runtime.js +550 -367
- package/dist/runtime/host/spot-runtime-options-factory.js +15 -7
- package/dist/runtime/host/user-spot-creation-coordinator.js +33 -84
- package/dist/runtime/locations/auto-connect-loop.d.ts +2 -0
- package/dist/runtime/locations/auto-connect-loop.js +22 -14
- package/dist/runtime/locations/auto-connect-reconciler.js +0 -6
- package/dist/runtime/locations/auto-connect-types.d.ts +0 -1
- package/dist/runtime/locations/descriptor-publication.d.ts +4 -0
- package/dist/runtime/locations/descriptor-publication.js +11 -0
- package/dist/runtime/locations/domain-store-contract.d.ts +1 -0
- package/dist/runtime/locations/in-memory-authority-store.d.ts +7 -1
- package/dist/runtime/locations/in-memory-authority-store.js +56 -19
- package/dist/runtime/locations/in-memory-location-store.js +1 -10
- package/dist/runtime/locations/index.d.ts +1 -0
- package/dist/runtime/locations/index.js +1 -0
- package/dist/runtime/locations/lease-tracker.js +1 -6
- package/dist/runtime/locations/location-store-repository.d.ts +51 -1
- package/dist/runtime/locations/location-store-repository.js +363 -139
- package/dist/runtime/locations/relocation-blob.d.ts +18 -1
- package/dist/runtime/locations/relocation-blob.js +52 -1
- package/dist/runtime/locations/request-source-resolver.d.ts +37 -0
- package/dist/runtime/locations/request-source-resolver.js +80 -0
- package/dist/runtime/locations/resolvers.js +1 -1
- package/dist/runtime/locations/runtime.d.ts +6 -2
- package/dist/runtime/locations/runtime.js +37 -30
- package/dist/runtime/messaging/payload-codec.d.ts +7 -1
- package/dist/runtime/messaging/payload-codec.js +7 -6
- package/dist/runtime/messaging/request-result.js +29 -26
- package/dist/runtime/protocol/service_wire_codec.generated.d.ts +19 -1
- package/dist/runtime/protocol/service_wire_codec.generated.js +85 -0
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.d.ts +7 -1
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.js +23 -7
- package/dist/runtime/spots/formal-remote-actor-admission-registry.js +3 -2
- package/dist/runtime/spots/index.d.ts +18 -11
- package/dist/runtime/spots/index.js +234 -176
- package/dist/runtime/spots/spot-activation-registry.d.ts +4 -2
- package/dist/runtime/spots/spot-activation-registry.js +21 -11
- package/dist/runtime/spots/spot-activation-state.d.ts +1 -1
- package/dist/runtime/spots/spot-activation-state.js +7 -7
- package/dist/runtime/spots/spot-activation.d.ts +6 -3
- package/dist/runtime/spots/spot-activation.js +123 -44
- package/dist/runtime/spots/spot-actor-admission-coordinator.d.ts +1 -0
- package/dist/runtime/spots/spot-actor-admission-coordinator.js +13 -22
- package/dist/runtime/spots/spot-actor-membership.d.ts +0 -1
- package/dist/runtime/spots/spot-actor-membership.js +0 -8
- package/dist/runtime/spots/spot-actor-packet-dispatch.js +33 -8
- package/dist/runtime/spots/spot-actor-packet-drain.js +5 -33
- package/dist/runtime/spots/spot-entry-activation.d.ts +2 -0
- package/dist/runtime/spots/spot-entry-activation.js +14 -9
- package/dist/runtime/spots/spot-handler-registry.d.ts +4 -3
- package/dist/runtime/spots/spot-handler-registry.js +6 -2
- package/dist/runtime/spots/spot-manager-internal-contracts.d.ts +2 -2
- package/dist/runtime/spots/spot-manager-public.js +2 -2
- package/dist/runtime/spots/spot-node-autoconnect.d.ts +2 -1
- package/dist/runtime/spots/spot-node-autoconnect.js +18 -39
- package/dist/runtime/spots/spot-node-runtime-manager.d.ts +19 -2
- package/dist/runtime/spots/spot-node-runtime-manager.js +115 -50
- package/dist/runtime/spots/spot-outbound.js +3 -3
- package/dist/runtime/spots/spot-remote-codec.js +5 -1
- package/dist/runtime/spots/spot-runtime-ports.d.ts +0 -1
- package/dist/runtime/spots/spot-serial-executor.d.ts +10 -0
- package/dist/runtime/spots/spot-serial-executor.js +61 -8
- package/dist/runtime/spots/spot-serial-turn-executor.js +19 -2
- package/dist/runtime/spots/spot-subscription-dispatch.js +1 -1
- package/dist/runtime/spots/spot-timer.d.ts +57 -5
- package/dist/runtime/spots/spot-timer.js +287 -103
- package/dist/runtime/streams/actor-session-binding-registry.d.ts +42 -11
- package/dist/runtime/streams/actor-session-binding-registry.js +165 -48
- package/dist/runtime/streams/actor-session-binding-runtime-owner.d.ts +8 -4
- package/dist/runtime/streams/bound-actor-relay-sender.d.ts +2 -1
- package/dist/runtime/streams/bound-actor-relay-sender.js +15 -8
- package/dist/runtime/streams/bound-session-service.d.ts +1 -1
- package/dist/runtime/streams/bound-session-service.js +6 -11
- package/dist/runtime/streams/index.d.ts +3 -2
- package/dist/runtime/streams/index.js +6 -4
- package/dist/runtime/streams/managed-stream.js +3 -2
- package/dist/runtime/streams/native-fallback-bound-session.js +8 -73
- package/dist/runtime/streams/session-actor-coordinator.d.ts +2 -8
- package/dist/runtime/streams/session-actor-coordinator.js +15 -32
- package/dist/runtime/streams/stream-binding-runtime-ports.d.ts +3 -10
- package/dist/runtime/streams/stream-session-runtime.js +1 -4
- package/dist/runtime/workers/index.js +1 -1
- package/package.json +3 -3
- package/src/contracts/Common/ZLinkEncodedPayload.ts +24 -4
- package/src/contracts/Common/encoded-payload-storage.ts +6 -3
- package/src/contracts/Configuration/Builders.ts +4 -1
- package/src/contracts/Configuration/RegistrationBuilderPolicy.ts +7 -2
- package/src/contracts/Configuration/RegistrationBuilders.ts +6 -2
- package/src/contracts/Configuration/RegistrationTypes.ts +1 -0
- package/src/contracts/Dispatch/ZLinkDispatchOptions.ts +2 -1
- package/src/contracts/Locations/Rows.ts +2 -0
- package/src/contracts/Spots/Contracts.ts +8 -0
- package/src/contracts/Spots/ZLinkSpot.ts +3 -3
- package/src/runtime/abort.ts +12 -0
- package/src/runtime/actors/actor-authority-publication.ts +3 -2
- package/src/runtime/actors/actor-client.ts +61 -70
- package/src/runtime/actors/actor-context.ts +49 -9
- package/src/runtime/actors/actor-handoff.ts +282 -106
- package/src/runtime/actors/actor-join-relocation.ts +2 -0
- package/src/runtime/actors/actor-local-native-join.ts +4 -0
- package/src/runtime/actors/actor-mailbox.ts +40 -2
- package/src/runtime/actors/actor-remote-joiner.ts +6 -1
- package/src/runtime/actors/actor-reply.ts +42 -0
- package/src/runtime/actors/actor-runtime-contracts.ts +2 -0
- package/src/runtime/actors/actor-runtime-state.ts +2 -20
- package/src/runtime/actors/actor-source-transfer.ts +1 -34
- package/src/runtime/actors/index.ts +41 -27
- package/src/runtime/actors/session-disconnected-wire.ts +66 -0
- package/src/runtime/admission.ts +69 -34
- package/src/runtime/application-jobs/application-job-queue-scope.ts +33 -1
- package/src/runtime/application-jobs/contracts.ts +2 -0
- package/src/runtime/backend/contracts/index.ts +39 -4
- package/src/runtime/backend/mesh-actor-session-node-adapter.ts +5 -3
- package/src/runtime/backend/mesh-completion-table.ts +4 -0
- package/src/runtime/backend/mesh-dispatch-pump.ts +223 -124
- package/src/runtime/backend/node/node-raw-binding-port.ts +106 -70
- package/src/runtime/backend/node/node-raw-mesh-backend.ts +221 -170
- package/src/runtime/backend/node/node-socket-backend-adapter.ts +5 -5
- package/src/runtime/channels/channel-clients.ts +49 -24
- package/src/runtime/channels/channel-dispatch-pipeline.ts +173 -124
- package/src/runtime/channels/channel-dispatchers.ts +116 -72
- package/src/runtime/channels/channel-envelope.ts +105 -36
- package/src/runtime/channels/channel-runtime-lifecycle.ts +13 -13
- package/src/runtime/channels/channel-runtime-manager.ts +12 -5
- package/src/runtime/channels/channel-transports.ts +49 -58
- package/src/runtime/channels/client-server-location-runtime.ts +11 -10
- package/src/runtime/channels/dispatch-error-reporter.ts +4 -2
- package/src/runtime/channels/fanout-location-runtime.ts +17 -9
- package/src/runtime/diagnostics/flow-context.ts +20 -7
- package/src/runtime/diagnostics/index.ts +1 -7
- package/src/runtime/diagnostics/message-flow.ts +47 -9
- package/src/runtime/diagnostics/runtime-metrics.ts +12 -10
- package/src/runtime/execution/index.ts +40 -14
- package/src/runtime/execution/serial-execution-queue.ts +31 -4
- package/src/runtime/execution/state-lane.ts +7 -0
- package/src/runtime/foundation/operation-registry.ts +98 -20
- package/src/runtime/foundation/raw-service-mesh-runtime.ts +244 -67
- package/src/runtime/foundation/service-liveness-registry.ts +39 -0
- package/src/runtime/foundation/service-mailbox.ts +8 -0
- package/src/runtime/foundation/service-maintenance-runtime.ts +1 -8
- package/src/runtime/foundation/service-relocation-object-owner.ts +54 -37
- package/src/runtime/foundation/service-relocation-runtime.ts +7 -3
- package/src/runtime/foundation/service-runtime-contracts.ts +12 -1
- package/src/runtime/foundation/service-session-binding-ingress-port.ts +2 -1
- package/src/runtime/foundation/service-stateful-registry.ts +38 -35
- package/src/runtime/foundation/service-stateful-runtime.ts +667 -152
- package/src/runtime/foundation/service-stateful-wire-codec.ts +49 -11
- package/src/runtime/foundation/service-wire-binary-primitives.ts +35 -12
- package/src/runtime/foundation/service-wire-constants.generated.ts +1 -0
- package/src/runtime/foundation/service-wire-m6a-codec.ts +187 -51
- package/src/runtime/host/actor-packet-relay.ts +113 -94
- package/src/runtime/host/actor-placement-coordinator.ts +31 -8
- package/src/runtime/host/actor-transfer-runtime.ts +104 -430
- package/src/runtime/host/application-job-queue.ts +29 -44
- package/src/runtime/host/index.ts +286 -129
- package/src/runtime/host/location-runtime-owner.ts +2 -6
- package/src/runtime/host/mesh-router-resolver.ts +13 -5
- package/src/runtime/host/remote-bound-session-relay.ts +86 -248
- package/src/runtime/host/route-mesh-runtime.ts +11 -5
- package/src/runtime/host/runtime-shutdown.ts +15 -4
- package/src/runtime/host/service-relocation-host-runtime.ts +784 -475
- package/src/runtime/host/spot-runtime-options-factory.ts +18 -7
- package/src/runtime/host/user-spot-creation-coordinator.ts +41 -100
- package/src/runtime/locations/auto-connect-loop.ts +22 -14
- package/src/runtime/locations/auto-connect-reconciler.ts +0 -6
- package/src/runtime/locations/auto-connect-types.ts +0 -1
- package/src/runtime/locations/descriptor-publication.ts +7 -0
- package/src/runtime/locations/domain-store-contract.ts +5 -0
- package/src/runtime/locations/in-memory-authority-store.ts +85 -23
- package/src/runtime/locations/in-memory-location-store.ts +1 -12
- package/src/runtime/locations/index.ts +1 -0
- package/src/runtime/locations/lease-tracker.ts +2 -9
- package/src/runtime/locations/location-store-repository.ts +628 -152
- package/src/runtime/locations/relocation-blob.ts +75 -1
- package/src/runtime/locations/request-source-resolver.ts +138 -0
- package/src/runtime/locations/resolvers.ts +2 -2
- package/src/runtime/locations/runtime.ts +70 -40
- package/src/runtime/messaging/payload-codec.ts +8 -11
- package/src/runtime/messaging/request-result.ts +33 -29
- package/src/runtime/protocol/service_wire_codec.generated.ts +87 -4
- package/src/runtime/protocol/service_wire_pilot_codec.generated.ts +13 -10
- package/src/runtime/spots/formal-remote-actor-admission-registry.ts +3 -2
- package/src/runtime/spots/index.ts +375 -281
- package/src/runtime/spots/spot-activation-registry.ts +25 -13
- package/src/runtime/spots/spot-activation-state.ts +12 -19
- package/src/runtime/spots/spot-activation.ts +170 -86
- package/src/runtime/spots/spot-actor-admission-coordinator.ts +25 -55
- package/src/runtime/spots/spot-actor-membership.ts +0 -15
- package/src/runtime/spots/spot-actor-packet-dispatch.ts +56 -11
- package/src/runtime/spots/spot-actor-packet-drain.ts +9 -52
- package/src/runtime/spots/spot-entry-activation.ts +24 -19
- package/src/runtime/spots/spot-handler-registry.ts +15 -4
- package/src/runtime/spots/spot-manager-internal-contracts.ts +2 -2
- package/src/runtime/spots/spot-manager-public.ts +2 -2
- package/src/runtime/spots/spot-node-autoconnect.ts +22 -45
- package/src/runtime/spots/spot-node-runtime-manager.ts +197 -67
- package/src/runtime/spots/spot-outbound.ts +3 -3
- package/src/runtime/spots/spot-remote-codec.ts +9 -2
- package/src/runtime/spots/spot-runtime-ports.ts +0 -1
- package/src/runtime/spots/spot-serial-executor.ts +97 -25
- package/src/runtime/spots/spot-serial-turn-executor.ts +22 -3
- package/src/runtime/spots/spot-subscription-dispatch.ts +1 -1
- package/src/runtime/spots/spot-timer.ts +413 -157
- package/src/runtime/streams/actor-session-binding-registry.ts +236 -64
- package/src/runtime/streams/actor-session-binding-runtime-owner.ts +16 -8
- package/src/runtime/streams/bound-actor-relay-sender.ts +23 -9
- package/src/runtime/streams/bound-session-service.ts +7 -10
- package/src/runtime/streams/index.ts +12 -6
- package/src/runtime/streams/managed-stream.ts +4 -2
- package/src/runtime/streams/native-fallback-bound-session.ts +14 -103
- package/src/runtime/streams/session-actor-coordinator.ts +39 -88
- package/src/runtime/streams/stream-binding-runtime-ports.ts +3 -9
- package/src/runtime/streams/stream-session-runtime.ts +1 -5
- package/src/runtime/workers/index.ts +1 -1
|
@@ -5,19 +5,26 @@ exports.decodeQueuedHandoffPacket = decodeQueuedHandoffPacket;
|
|
|
5
5
|
exports.createServiceRelocationId = createServiceRelocationId;
|
|
6
6
|
exports.relocationFailedFailureCode = relocationFailedFailureCode;
|
|
7
7
|
exports.encodeHandoffQueuedMessages = encodeHandoffQueuedMessages;
|
|
8
|
+
const actor_reply_1 = require("../actors/actor-reply");
|
|
9
|
+
const protocol_1 = require("../streams/protocol");
|
|
10
|
+
const service_wire_m6a_codec_1 = require("../foundation/service-wire-m6a-codec");
|
|
11
|
+
const service_wire_constants_generated_1 = require("../foundation/service-wire-constants.generated");
|
|
12
|
+
const operation_registry_1 = require("../foundation/operation-registry");
|
|
8
13
|
const durable_service_request_1 = require("../foundation/durable-service-request");
|
|
9
14
|
const stream_wire_1 = require("@zlink-systems/stream-wire");
|
|
10
15
|
const node_crypto_1 = require("node:crypto");
|
|
11
16
|
const contracts_1 = require("../../contracts");
|
|
17
|
+
const ZLinkDispatchOptions_1 = require("../../contracts/Dispatch/ZLinkDispatchOptions");
|
|
12
18
|
const Options_1 = require("../../contracts/Locations/Options");
|
|
13
19
|
const Stores_1 = require("../../contracts/Locations/Stores");
|
|
14
20
|
const actor_authority_publication_1 = require("../actors/actor-authority-publication");
|
|
15
|
-
const actor_handoff_1 = require("../actors/actor-handoff");
|
|
16
21
|
const actor_remote_wire_1 = require("../actors/actor-remote-wire");
|
|
17
22
|
const actor_runtime_state_1 = require("../actors/actor-runtime-state");
|
|
18
23
|
const runtime_values_1 = require("../backend/runtime-values");
|
|
19
24
|
const diagnostics_1 = require("../diagnostics");
|
|
20
25
|
const runtime_metrics_1 = require("../diagnostics/runtime-metrics");
|
|
26
|
+
const message_flow_1 = require("../diagnostics/message-flow");
|
|
27
|
+
const dispatch_error_details_1 = require("../diagnostics/dispatch-error-details");
|
|
21
28
|
const state_lane_1 = require("../execution/state-lane");
|
|
22
29
|
const actor_join_recovery_codec_1 = require("../foundation/actor-join-recovery-codec");
|
|
23
30
|
const operation_identity_1 = require("../foundation/operation-identity");
|
|
@@ -29,8 +36,8 @@ const service_relocation_object_owner_1 = require("../foundation/service-relocat
|
|
|
29
36
|
const service_relocation_runtime_1 = require("../foundation/service-relocation-runtime");
|
|
30
37
|
const service_runtime_contracts_1 = require("../foundation/service-runtime-contracts");
|
|
31
38
|
const service_stateful_wire_codec_1 = require("../foundation/service-stateful-wire-codec");
|
|
32
|
-
const
|
|
33
|
-
const
|
|
39
|
+
const service_wire_constants_generated_2 = require("../foundation/service-wire-constants.generated");
|
|
40
|
+
const service_wire_m6a_codec_2 = require("../foundation/service-wire-m6a-codec");
|
|
34
41
|
const framework_errors_internal_1 = require("../framework-errors-internal");
|
|
35
42
|
const authority_key_codec_1 = require("../locations/authority-key-codec");
|
|
36
43
|
const location_store_failure_1 = require("../locations/location-store-failure");
|
|
@@ -66,7 +73,6 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
66
73
|
options;
|
|
67
74
|
targetStages = new Map();
|
|
68
75
|
terminalTargets = new bounded_replay_map_1.BoundedReplayMap(RELOCATION_TARGET_TOMBSTONE_LIMIT);
|
|
69
|
-
relocationAuthorityKeys = new Map();
|
|
70
76
|
targetPrepareOperations = new Map();
|
|
71
77
|
pendingReplyRelays = new Map();
|
|
72
78
|
pendingSessionRelocations = new Map();
|
|
@@ -104,6 +110,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
104
110
|
* checksum integrity failure — it must not be reported as DataLost(35).
|
|
105
111
|
*/
|
|
106
112
|
disposed = false;
|
|
113
|
+
targetPreparationStop = new AbortController();
|
|
107
114
|
constructor(options) {
|
|
108
115
|
this.options = options;
|
|
109
116
|
}
|
|
@@ -131,6 +138,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
131
138
|
reject(failure.cause);
|
|
132
139
|
};
|
|
133
140
|
});
|
|
141
|
+
let acceptRelay;
|
|
142
|
+
const relayReady = new Promise((resolve) => {
|
|
143
|
+
acceptRelay = resolve;
|
|
144
|
+
});
|
|
134
145
|
this.sourceActorJoinProfiles.set(input.relocationId, {
|
|
135
146
|
actor: input.actor,
|
|
136
147
|
state: input.state,
|
|
@@ -144,32 +155,43 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
144
155
|
: actor_join_recovery_codec_1.RECREATE_RELOCATION_CONTENT_TYPE,
|
|
145
156
|
completionOperationId: input.completionOperationId,
|
|
146
157
|
canonicalRecovery: input.canonicalRecovery,
|
|
158
|
+
relayReady: acceptRelay,
|
|
147
159
|
ready,
|
|
148
160
|
resolveReady
|
|
149
161
|
});
|
|
150
162
|
const membershipEpoch = input.state.spotMembershipEpoch > 0n ? input.state.spotMembershipEpoch : 1n;
|
|
151
163
|
const spotGeneration = input.target.targetSpotGeneration ?? target.lifecycleGeneration;
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
catch (error) {
|
|
163
|
-
if (error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) {
|
|
164
|
-
resolveReady(error);
|
|
165
|
-
this.completeActorJoinSourceCleanup(input.state.actorId);
|
|
164
|
+
const cleanup = (async () => {
|
|
165
|
+
try {
|
|
166
|
+
await this.relocateStandaloneActor(input.meshName, input.state, target, target.applicationVersion, input.signal, {
|
|
167
|
+
...input.target,
|
|
168
|
+
spotKind: input.target.spotKind === contracts_1.ZLinkSpotKind.Entry
|
|
169
|
+
? contracts_1.ZLinkSpotKind.Entry
|
|
170
|
+
: contracts_1.ZLinkSpotKind.User,
|
|
171
|
+
spotObjectGeneration: spotGeneration
|
|
172
|
+
}, input.relocationId, true, input.advertisedReceiveChunkLimitBytes, input.deadlineAtMs);
|
|
173
|
+
resolveReady();
|
|
166
174
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
175
|
+
catch (error) {
|
|
176
|
+
if (error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) {
|
|
177
|
+
resolveReady(error);
|
|
178
|
+
this.completeActorJoinSourceCleanup(input.state.actorId);
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
this.sourceActorJoinProfiles.delete(input.relocationId);
|
|
182
|
+
resolveReady();
|
|
183
|
+
throw error;
|
|
184
|
+
}
|
|
170
185
|
}
|
|
171
|
-
}
|
|
186
|
+
})();
|
|
187
|
+
const settledCleanup = Promise.allSettled([cleanup]);
|
|
188
|
+
await Promise.race([relayReady, cleanup]);
|
|
172
189
|
return {
|
|
190
|
+
sourceCleanup: async () => {
|
|
191
|
+
const [outcome] = await settledCleanup;
|
|
192
|
+
if (outcome.status === 'rejected')
|
|
193
|
+
throw outcome.reason;
|
|
194
|
+
},
|
|
173
195
|
actorRef: {
|
|
174
196
|
actorId: sourceRef.actorId,
|
|
175
197
|
generation: sourceRef.generation,
|
|
@@ -204,8 +226,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
204
226
|
this.sourceCutoverWindows.clear();
|
|
205
227
|
if (hadSourceWindows)
|
|
206
228
|
this.options.sourceQuiescenceChanged?.();
|
|
207
|
-
const stopped = new
|
|
208
|
-
stopped.name = 'AbortError';
|
|
229
|
+
const stopped = new ServiceRelocationRuntimeStoppedError();
|
|
209
230
|
this.targetPrepareOperations.clear();
|
|
210
231
|
for (const pending of this.pendingReplyRelays.values()) {
|
|
211
232
|
if (pending.timer !== undefined)
|
|
@@ -223,10 +244,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
223
244
|
this.pendingSessionRelocations.clear();
|
|
224
245
|
this.terminalSessionRelocations.clear();
|
|
225
246
|
this.failedSessionRouteSubmits.clear();
|
|
226
|
-
this.
|
|
247
|
+
await this.drainActorJoinSources();
|
|
227
248
|
this.terminalActorJoinSourceLeaves.clear();
|
|
228
|
-
this.options.boundSessionRelocation?.clear?.();
|
|
229
|
-
this.relocationAuthorityKeys.clear();
|
|
249
|
+
await this.options.boundSessionRelocation?.clear?.();
|
|
230
250
|
if (errors.length === 1)
|
|
231
251
|
throw errors[0];
|
|
232
252
|
if (errors.length > 1)
|
|
@@ -234,6 +254,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
234
254
|
}
|
|
235
255
|
/** Host shutdown waits for the existing attempt terminal before releasing its lease. */
|
|
236
256
|
async drainTargetAttempts() {
|
|
257
|
+
this.targetPreparationStop.abort(new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.ShuttingDown, 'Relocation target is shutting down.'));
|
|
258
|
+
for (const pending of this.targetAssemblies.values()) {
|
|
259
|
+
pending.assembly.fail('Relocation target is shutting down.');
|
|
260
|
+
}
|
|
261
|
+
await Promise.allSettled([...this.targetPrepareOperations.values()].map((value) => value.promise));
|
|
237
262
|
const results = await Promise.allSettled([...this.targetStages.entries()].map(([stagingId, stage]) => {
|
|
238
263
|
if (stage.fallback !== undefined)
|
|
239
264
|
clearTimeout(stage.fallback);
|
|
@@ -256,18 +281,34 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
256
281
|
}
|
|
257
282
|
/** Spec 28 §4.4: the target owner alone orders shutdown seal and verified cutover. */
|
|
258
283
|
targetStageSealed(stage) {
|
|
259
|
-
return
|
|
284
|
+
return !stage.cutoverReceived && this.targetAdmissionSealed();
|
|
285
|
+
}
|
|
286
|
+
/** The Shutdown admission seal: closed once the target owner starts shutdown. */
|
|
287
|
+
targetAdmissionSealed() {
|
|
288
|
+
return this.disposed || this.options.targetAdmissionSealed?.() === true;
|
|
260
289
|
}
|
|
261
290
|
completeActorJoinSourceCleanup(actorId) {
|
|
262
291
|
for (const [relocationId, profile] of this.sourceActorJoinProfiles) {
|
|
263
292
|
if (profile.state.actorId !== actorId)
|
|
264
293
|
continue;
|
|
265
|
-
this.
|
|
266
|
-
this.terminalActorJoinSourceLeaves.remember(relocationId, actorId);
|
|
267
|
-
void this.finalizeActorJoinSource(profile);
|
|
294
|
+
void this.retireActorJoinSource(relocationId, profile);
|
|
268
295
|
}
|
|
269
296
|
}
|
|
270
|
-
|
|
297
|
+
/** Drains the exact source scopes before their Spot lifecycle owners stop. */
|
|
298
|
+
async drainActorJoinSources(meshName) {
|
|
299
|
+
await Promise.all([...this.sourceActorJoinProfiles].map(([relocationId, profile]) => meshName === undefined || profile.state.meshName === meshName
|
|
300
|
+
? this.retireActorJoinSource(relocationId, profile)
|
|
301
|
+
: undefined));
|
|
302
|
+
}
|
|
303
|
+
retireActorJoinSource(relocationId, profile) {
|
|
304
|
+
return (profile.retirement ??= this.finalizeActorJoinSource(profile, relocationId).finally(() => {
|
|
305
|
+
if (this.sourceActorJoinProfiles.get(relocationId) === profile) {
|
|
306
|
+
this.sourceActorJoinProfiles.delete(relocationId);
|
|
307
|
+
this.terminalActorJoinSourceLeaves.remember(relocationId, profile.state.actorId);
|
|
308
|
+
}
|
|
309
|
+
}));
|
|
310
|
+
}
|
|
311
|
+
async finalizeActorJoinSource(profile, relocationId) {
|
|
271
312
|
const failures = [];
|
|
272
313
|
try {
|
|
273
314
|
await profile.ready;
|
|
@@ -275,6 +316,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
275
316
|
catch (error) {
|
|
276
317
|
failures.push(error);
|
|
277
318
|
}
|
|
319
|
+
// A pre-commit failure removed this attempt and restored the source.
|
|
320
|
+
if (this.sourceActorJoinProfiles.get(relocationId) !== profile)
|
|
321
|
+
return;
|
|
278
322
|
try {
|
|
279
323
|
await this.options.actorTransfer.completeRelocationSourceLeave(profile.actor, profile.sourceSpotId);
|
|
280
324
|
}
|
|
@@ -284,7 +328,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
284
328
|
try {
|
|
285
329
|
await this.options
|
|
286
330
|
.actorManager()
|
|
287
|
-
?.
|
|
331
|
+
?.completeCoreRelocationSource(profile.state.actorId, profile.state);
|
|
288
332
|
}
|
|
289
333
|
catch (error) {
|
|
290
334
|
failures.push(error);
|
|
@@ -330,10 +374,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
330
374
|
id: `actor:${state.actorId}`,
|
|
331
375
|
relocate: async (relocationSignal) => {
|
|
332
376
|
await relocateShell(relocationSignal);
|
|
377
|
+
const spotAuthority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(activation.spotId)), relocationSignal);
|
|
333
378
|
await this.relocateStandaloneActor(meshName, state, undefined, targetApplicationVersion, relocationSignal, {
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
spotObjectGeneration: (await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(activation.spotId)), relocationSignal)).objectGeneration
|
|
379
|
+
...committedSpotRoute(meshName, activation.spotId, contracts_1.ZLinkSpotKind.User, spotAuthority.allocation.descriptor.rid, spotAuthority),
|
|
380
|
+
spotObjectGeneration: spotAuthority.objectGeneration
|
|
337
381
|
});
|
|
338
382
|
}
|
|
339
383
|
});
|
|
@@ -412,13 +456,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
412
456
|
const payload = record.parts[0].data();
|
|
413
457
|
const command = (0, service_relocation_control_1.serviceRelocationControlCommand)(payload);
|
|
414
458
|
if (command === undefined) {
|
|
415
|
-
if (record.kind !== service_runtime_contracts_1.ReceiveKind.NodeSend)
|
|
459
|
+
if (record.kind !== service_runtime_contracts_1.ReceiveKind.NodeSend || record.parts.length !== 1)
|
|
416
460
|
return false;
|
|
417
461
|
const sourceLeave = (0, actor_remote_wire_1.decodeRemoteActorSourceLeaveTerminal)(payload);
|
|
418
462
|
if (sourceLeave === undefined)
|
|
419
463
|
return false;
|
|
420
|
-
if (record.parts.length !== 1)
|
|
421
|
-
return false;
|
|
422
464
|
return this.acceptActorJoinSourceLeave(sourceLeave);
|
|
423
465
|
}
|
|
424
466
|
return this.handleControl(meshName, record, payload, command, signal);
|
|
@@ -441,9 +483,38 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
441
483
|
return true;
|
|
442
484
|
}
|
|
443
485
|
if (command === service_stateful_wire_codec_1.M6bServiceWireCommand.sessionRelocationSeal) {
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
486
|
+
let request;
|
|
487
|
+
try {
|
|
488
|
+
request = (0, service_stateful_wire_codec_1.decodeSessionRelocationSeal)(payload);
|
|
489
|
+
}
|
|
490
|
+
catch (error) {
|
|
491
|
+
const outcome = ZLinkDispatchOptions_1.ZLinkRuntimeMessageFlowOutcome.Error;
|
|
492
|
+
(0, message_flow_1.flowIfEnabled)(this.options.messageFlow?.(), outcome)?.trace({
|
|
493
|
+
outcome,
|
|
494
|
+
surface: ZLinkDispatchOptions_1.ZLinkDispatchErrorSurface.StreamSession,
|
|
495
|
+
messageKind: ZLinkDispatchOptions_1.ZLinkDispatchMessageKind.Control,
|
|
496
|
+
packetName: 'sessionRelocationSeal',
|
|
497
|
+
sourceRid: record.sourceNodeRid ?? undefined,
|
|
498
|
+
errorReason: 'invalid_frame',
|
|
499
|
+
errorAction: ZLinkDispatchOptions_1.ZLinkRuntimeDispatchErrorAction.Drop,
|
|
500
|
+
...(0, dispatch_error_details_1.dispatchErrorDetails)(error)
|
|
501
|
+
});
|
|
502
|
+
throw error;
|
|
503
|
+
}
|
|
504
|
+
let response;
|
|
505
|
+
try {
|
|
506
|
+
response = await this.handleSessionRelocationSeal(meshName, request, record.sourceNodeRid, signal);
|
|
507
|
+
}
|
|
508
|
+
catch (error) {
|
|
509
|
+
(0, message_flow_1.traceSessionBindingControlDrop)(this.options.messageFlow?.(), record.sourceNodeRid ?? '', request.actor.actor.actorId, 'sessionRelocationSeal', error instanceof contracts_1.ZLinkFrameworkException &&
|
|
510
|
+
error.kind === contracts_1.ZLinkFrameworkErrorKind.Unavailable
|
|
511
|
+
? 'location_unavailable'
|
|
512
|
+
: 'stale_target', error);
|
|
513
|
+
throw error;
|
|
514
|
+
}
|
|
515
|
+
if (response !== undefined) {
|
|
516
|
+
await this.sendSessionRelocationResponse(meshName, record.sourceNodeRid, (0, service_stateful_wire_codec_1.encodeSessionRelocationSealed)(response));
|
|
517
|
+
}
|
|
447
518
|
return true;
|
|
448
519
|
}
|
|
449
520
|
if (command === service_stateful_wire_codec_1.M6bServiceWireCommand.sessionRelocationRoute) {
|
|
@@ -471,7 +542,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
471
542
|
throw new Error('Relocation prepare has no authenticated source node.');
|
|
472
543
|
}
|
|
473
544
|
if (record.kind !== service_runtime_contracts_1.ReceiveKind.NodeRequest) {
|
|
474
|
-
throw new
|
|
545
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Relocation prepare requires a Core request leg.');
|
|
475
546
|
}
|
|
476
547
|
const operationKey = relocationStagingId(request);
|
|
477
548
|
const fingerprint = stringifyWire(request);
|
|
@@ -484,12 +555,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
484
555
|
this.targetAssemblies.get(operationKey)?.fingerprint ??
|
|
485
556
|
this.targetStages.get(operationKey)?.offer.prepareFingerprint ??
|
|
486
557
|
this.terminalTargets.get(operationKey);
|
|
487
|
-
if (request.sourceNodeRid
|
|
558
|
+
if (!(0, routing_id_1.routingIdsEqual)(request.sourceNodeRid, record.sourceNodeRid))
|
|
488
559
|
return true;
|
|
489
560
|
if (acceptedFingerprint !== undefined && acceptedFingerprint !== fingerprint) {
|
|
490
561
|
if (!samePrepareIdentity(acceptedFingerprint, request))
|
|
491
562
|
return true;
|
|
492
|
-
this.replyPrepare(record, relocationFailed(request,
|
|
563
|
+
this.replyPrepare(record, relocationFailed(request, service_wire_constants_generated_2.ServiceWireFrameworkErrorCode.relocationDataLost));
|
|
493
564
|
return true;
|
|
494
565
|
}
|
|
495
566
|
// A restore that failed outright is terminal: the source observes the
|
|
@@ -513,7 +584,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
513
584
|
// finish the restore and the Ready reply asynchronously.
|
|
514
585
|
let promise;
|
|
515
586
|
try {
|
|
516
|
-
this.registerTargetAssembly(operationKey, request,
|
|
587
|
+
this.registerTargetAssembly(operationKey, request, record.sourceNodeRid);
|
|
517
588
|
promise = this.handlePrepareControl(meshName, operationKey, request, record.sourceNodeRid, signal);
|
|
518
589
|
}
|
|
519
590
|
catch (error) {
|
|
@@ -624,7 +695,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
624
695
|
const pending = this.targetAssemblies.get(operationKey);
|
|
625
696
|
if (pending === undefined ||
|
|
626
697
|
sourceNodeRid === null ||
|
|
627
|
-
|
|
698
|
+
!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, pending.authenticatedSourceNodeRid) ||
|
|
628
699
|
request.senderRole !== 'source' ||
|
|
629
700
|
!sameCoordinator(request.coordinator, pending.prepare.coordinator) ||
|
|
630
701
|
!sameRelocationObject(request.object, pending.prepare.object)) {
|
|
@@ -676,7 +747,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
676
747
|
// to itself; RouteMesh has no self connection (NotConnected), so the
|
|
677
748
|
// control dispatches through the same inbound handler locally.
|
|
678
749
|
const localRid = this.requireMeshNode(meshName).status().routingId;
|
|
679
|
-
if (
|
|
750
|
+
if ((0, routing_id_1.routingIdsEqual)(localRid, targetNodeRid)) {
|
|
680
751
|
await this.handleSessionRelocationRoute(meshName, request, localRid, signal);
|
|
681
752
|
return;
|
|
682
753
|
}
|
|
@@ -696,20 +767,21 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
696
767
|
const key = sessionRelocationPendingKey(request);
|
|
697
768
|
const bytes = (0, service_stateful_wire_codec_1.encodeSessionRelocationSeal)(request);
|
|
698
769
|
const requestFingerprint = bytes.toString('base64');
|
|
699
|
-
const expectedTarget =
|
|
770
|
+
const expectedTarget = (0, routing_id_1.encodeRoutingIdStorageHex)(targetNodeRid);
|
|
700
771
|
const terminal = this.terminalSessionRelocations.get(key);
|
|
701
772
|
if (terminal !== undefined) {
|
|
702
773
|
if (terminal.targetNodeRid !== expectedTarget ||
|
|
703
774
|
terminal.requestFingerprint !== requestFingerprint) {
|
|
704
|
-
return Promise.reject(new
|
|
775
|
+
return Promise.reject(new service_wire_m6a_codec_2.ServiceWireProtocolError(`Session relocation control '${key}' repeated with different bytes or target.`));
|
|
705
776
|
}
|
|
706
777
|
this.terminalSessionRelocations.touch(key);
|
|
778
|
+
return Promise.resolve(terminal.response);
|
|
707
779
|
}
|
|
708
780
|
const existing = this.pendingSessionRelocations.get(key);
|
|
709
781
|
if (existing !== undefined) {
|
|
710
782
|
if (existing.targetNodeRid !== expectedTarget ||
|
|
711
783
|
existing.requestFingerprint !== requestFingerprint) {
|
|
712
|
-
return Promise.reject(new
|
|
784
|
+
return Promise.reject(new service_wire_m6a_codec_2.ServiceWireProtocolError(`Session relocation control '${key}' repeated with different bytes or target.`));
|
|
713
785
|
}
|
|
714
786
|
return existing.promise;
|
|
715
787
|
}
|
|
@@ -762,10 +834,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
762
834
|
// connection (NotConnected); dispatch through the inbound handler
|
|
763
835
|
// locally instead of the wire.
|
|
764
836
|
const localRid = this.requireMeshNode(meshName).status().routingId;
|
|
765
|
-
if (
|
|
837
|
+
if ((0, routing_id_1.encodeRoutingIdStorageHex)(localRid) === expectedTarget) {
|
|
766
838
|
try {
|
|
767
839
|
const response = await this.handleSessionRelocationSeal(meshName, request, localRid, signal);
|
|
768
|
-
|
|
840
|
+
if (response !== undefined)
|
|
841
|
+
pending.resolve(response);
|
|
769
842
|
}
|
|
770
843
|
catch (error) {
|
|
771
844
|
pending.reject(error);
|
|
@@ -798,9 +871,8 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
798
871
|
resolve: (response) => finish(() => {
|
|
799
872
|
this.terminalSessionRelocations.remember(key, {
|
|
800
873
|
targetNodeRid: expectedTarget,
|
|
801
|
-
request,
|
|
802
874
|
requestFingerprint,
|
|
803
|
-
|
|
875
|
+
response
|
|
804
876
|
});
|
|
805
877
|
resolvePromise(response);
|
|
806
878
|
}),
|
|
@@ -826,18 +898,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
826
898
|
const terminal = this.terminalSessionRelocations.get(key);
|
|
827
899
|
if (terminal === undefined)
|
|
828
900
|
return;
|
|
829
|
-
|
|
830
|
-
throw new service_wire_m6a_codec_1.ServiceWireProtocolError(`Session relocation ACK '${key}' source node changed.`);
|
|
831
|
-
}
|
|
832
|
-
validateSessionRelocationResponse(terminal.request, response);
|
|
833
|
-
if (sessionRelocationResponseFingerprint(response) !== terminal.responseFingerprint) {
|
|
834
|
-
throw new service_wire_m6a_codec_1.ServiceWireProtocolError(`Session relocation ACK '${key}' repeated with different bytes.`);
|
|
835
|
-
}
|
|
901
|
+
// Session–Actor binding §8.3: the first command 43 is final for this participant.
|
|
836
902
|
this.terminalSessionRelocations.touch(key);
|
|
837
903
|
return;
|
|
838
904
|
}
|
|
839
|
-
if (sourceNodeRid === null ||
|
|
840
|
-
|
|
905
|
+
if (sourceNodeRid === null ||
|
|
906
|
+
(0, routing_id_1.encodeRoutingIdStorageHex)(sourceNodeRid) !== pending.targetNodeRid) {
|
|
907
|
+
pending.reject(new service_wire_m6a_codec_2.ServiceWireProtocolError(`Session relocation ACK '${key}' source node changed.`));
|
|
841
908
|
return;
|
|
842
909
|
}
|
|
843
910
|
try {
|
|
@@ -856,44 +923,52 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
856
923
|
return undefined;
|
|
857
924
|
const node = this.requireMeshNode(meshName);
|
|
858
925
|
const status = node.status();
|
|
859
|
-
if (
|
|
926
|
+
if ((0, routing_id_1.routingIdsEqual)(status.routingId, sourceNodeRid)) {
|
|
860
927
|
return status.lifecycleGeneration;
|
|
861
928
|
}
|
|
862
929
|
return node
|
|
863
930
|
.peers()
|
|
864
|
-
.find((peer) => peer.routingId !== null &&
|
|
931
|
+
.find((peer) => peer.routingId !== null && (0, routing_id_1.routingIdsEqual)(peer.routingId, sourceNodeRid))
|
|
865
932
|
?.lifecycleGeneration;
|
|
866
933
|
}
|
|
867
934
|
async handleSessionRelocationSeal(meshName, request, sourceNodeRid, signal) {
|
|
868
935
|
if (sourceNodeRid === null ||
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
throw new
|
|
936
|
+
!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, request.coordinator.nodeRid) ||
|
|
937
|
+
!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, request.actor.actor.nodeRid)) {
|
|
938
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation seal source does not match its coordinator and Actor fence.');
|
|
872
939
|
}
|
|
873
940
|
this.validateSessionOwnerFence(meshName, request.session);
|
|
874
941
|
const sealSourceGeneration = this.sourceLifecycleGeneration(meshName, sourceNodeRid);
|
|
875
942
|
if (sealSourceGeneration !== request.actor.targetNodeGeneration ||
|
|
876
943
|
sealSourceGeneration !== request.coordinator.nodeGeneration) {
|
|
877
|
-
throw new
|
|
944
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation seal source lifecycle generation is stale.');
|
|
878
945
|
}
|
|
879
946
|
const handler = this.options.boundSessionRelocation?.receiveSeal;
|
|
880
947
|
if (handler === undefined) {
|
|
881
948
|
throw new Error('Session relocation seal ingress is not configured.');
|
|
882
949
|
}
|
|
883
|
-
return await handler(request, signal)
|
|
950
|
+
return await handler(request, signal, () => {
|
|
951
|
+
const node = this.requireMeshNode(meshName);
|
|
952
|
+
if ((0, routing_id_1.routingIdsEqual)(node.status().routingId, request.targetNodeRid) ||
|
|
953
|
+
node.isPeerRouteReady?.(request.targetNodeRid, request.targetNodeGeneration) === true)
|
|
954
|
+
return;
|
|
955
|
+
void this.options
|
|
956
|
+
.requestPeerReconcile?.(meshName, request.targetNodeRid)
|
|
957
|
+
.catch((error) => console.error('[zlink.runtime.relocation.session_route_reconcile_failed]', error));
|
|
958
|
+
});
|
|
884
959
|
}
|
|
885
960
|
async handleSessionRelocationRoute(meshName, request, sourceNodeRid, _signal) {
|
|
886
961
|
this.validateSessionOwnerFence(meshName, request.session);
|
|
887
962
|
const expectedSource = request.route.action === 'commit' ? request.route.targetNodeRid : request.coordinator.nodeRid;
|
|
888
|
-
if (sourceNodeRid === null ||
|
|
889
|
-
throw new
|
|
963
|
+
if (sourceNodeRid === null || !(0, routing_id_1.routingIdsEqual)(sourceNodeRid, expectedSource)) {
|
|
964
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation route source does not match its action fence.');
|
|
890
965
|
}
|
|
891
966
|
const routeSourceGeneration = this.sourceLifecycleGeneration(meshName, sourceNodeRid);
|
|
892
967
|
const expectedSourceGeneration = request.route.action === 'commit'
|
|
893
968
|
? request.route.targetNodeGeneration
|
|
894
969
|
: request.coordinator.nodeGeneration;
|
|
895
970
|
if (routeSourceGeneration !== expectedSourceGeneration) {
|
|
896
|
-
throw new
|
|
971
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation route source lifecycle generation is stale.');
|
|
897
972
|
}
|
|
898
973
|
const handler = this.options.boundSessionRelocation?.receiveRoute;
|
|
899
974
|
if (handler === undefined) {
|
|
@@ -904,16 +979,20 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
904
979
|
validateSessionOwnerFence(meshName, session) {
|
|
905
980
|
const status = this.requireMeshNode(meshName).status();
|
|
906
981
|
const owner = this.options.currentOwner();
|
|
982
|
+
// Session-Actor binding §4: without this host's own descriptor the command is Unavailable.
|
|
983
|
+
if (owner === undefined) {
|
|
984
|
+
throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected, 'Session owner has no live MeshNode descriptor.');
|
|
985
|
+
}
|
|
907
986
|
if (!(0, routing_id_1.routingIdsEqual)(status.routingId, session.sessionOwnerNodeRid) ||
|
|
908
987
|
status.lifecycleGeneration !== session.sessionOwnerNodeGeneration ||
|
|
909
|
-
owner
|
|
988
|
+
owner.ownerId !== session.sessionOwnerId ||
|
|
910
989
|
owner.leaseGeneration !== session.sessionOwnerLeaseGeneration) {
|
|
911
|
-
throw new
|
|
990
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation command does not match the local Session owner fence.');
|
|
912
991
|
}
|
|
913
992
|
}
|
|
914
993
|
async sendSessionRelocationResponse(meshName, targetNodeRid, bytes) {
|
|
915
994
|
if (targetNodeRid === null) {
|
|
916
|
-
throw new
|
|
995
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation command has no authenticated source.');
|
|
917
996
|
}
|
|
918
997
|
const submitted = await this.sendInfrastructureControl(meshName, targetNodeRid, bytes);
|
|
919
998
|
if (submitted !== runtime_values_1.SubmitResult.Ok) {
|
|
@@ -938,7 +1017,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
938
1017
|
let readinessBoundaryConsumed = false;
|
|
939
1018
|
let interruptionStartedAt;
|
|
940
1019
|
const spotRegistration = this.spotRegistration(meshName, kind, spotAuthority.allocation.stableType);
|
|
941
|
-
target ??= await this.selectTarget(meshName,
|
|
1020
|
+
target ??= await this.selectTarget(meshName, this.requireMeshNode(meshName).status().routingId, targetApplicationVersion, relocationTargetRequirements([
|
|
942
1021
|
{
|
|
943
1022
|
objectKind: kind,
|
|
944
1023
|
stableType: spotAuthority.allocation.stableType,
|
|
@@ -963,23 +1042,32 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
963
1042
|
objectGeneration: spotAuthority.objectGeneration,
|
|
964
1043
|
authorityOwnerGeneration: spotAuthority.authorityOwnerGeneration,
|
|
965
1044
|
seal: async (captureSignal) => {
|
|
1045
|
+
// Relocation 04 §4.1: every bound member's Session is sealed before
|
|
1046
|
+
// the Spot capture stops member dispatch.
|
|
1047
|
+
const preparations = await Promise.allSettled(actorStates.map(async (state) => ({
|
|
1048
|
+
state,
|
|
1049
|
+
actor: state.actor,
|
|
1050
|
+
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, captureSignal, false, relocationWireId(aggregateId), target)
|
|
1051
|
+
})));
|
|
1052
|
+
const failures = [];
|
|
1053
|
+
for (const preparation of preparations) {
|
|
1054
|
+
if (preparation.status === 'fulfilled')
|
|
1055
|
+
sessions.push(preparation.value);
|
|
1056
|
+
else
|
|
1057
|
+
failures.push(preparation.reason);
|
|
1058
|
+
}
|
|
1059
|
+
if (failures.length === 1)
|
|
1060
|
+
throw failures[0];
|
|
1061
|
+
if (failures.length > 1) {
|
|
1062
|
+
throw new AggregateError(failures, 'Spot relocation source preparation failed.');
|
|
1063
|
+
}
|
|
966
1064
|
const { spotCaptureOperation } = await this.afterSpotCloseDecision(meshName, activation.spotId, () => {
|
|
967
1065
|
interruptionStartedAt = performance.now();
|
|
968
1066
|
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
969
|
-
// Both seals run in the same event-loop turn after the Close decision.
|
|
1067
|
+
// Both Spot seals run in the same event-loop turn after the Close decision.
|
|
970
1068
|
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
971
1069
|
});
|
|
972
|
-
const preparedSessions = Promise.all(actorStates.map(async (state) => ({
|
|
973
|
-
state,
|
|
974
|
-
actor: state.actor,
|
|
975
|
-
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, captureSignal, false, relocationWireId(aggregateId))
|
|
976
|
-
})));
|
|
977
|
-
const preparedSessionsOutcome = preparedSessions.then((value) => ({ value }), (error) => ({ error }));
|
|
978
1070
|
spotCapture = await spotCaptureOperation;
|
|
979
|
-
const preparation = await preparedSessionsOutcome;
|
|
980
|
-
if ('error' in preparation)
|
|
981
|
-
throw preparation.error;
|
|
982
|
-
sessions.push(...preparation.value);
|
|
983
1071
|
return {
|
|
984
1072
|
boundSessionState: Buffer.alloc(0),
|
|
985
1073
|
queuedMessages: [],
|
|
@@ -987,7 +1075,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
987
1075
|
};
|
|
988
1076
|
},
|
|
989
1077
|
captureApplicationState: (captureSignal) => this.captureApplication(spotRegistration.relocation, activation.spot, captureSignal),
|
|
990
|
-
commitSeal: async () => {
|
|
1078
|
+
commitSeal: async (followAuthorities) => {
|
|
991
1079
|
const failures = [];
|
|
992
1080
|
try {
|
|
993
1081
|
if (!spotSealCommitted) {
|
|
@@ -997,7 +1085,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
997
1085
|
spotSealCommitted = true;
|
|
998
1086
|
}
|
|
999
1087
|
if (spotMessageFollowSeal !== undefined) {
|
|
1000
|
-
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, spotKey, activation
|
|
1088
|
+
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, followAuthorities.get(spotKey.value), activation);
|
|
1001
1089
|
}
|
|
1002
1090
|
}
|
|
1003
1091
|
catch (error) {
|
|
@@ -1005,20 +1093,17 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1005
1093
|
}
|
|
1006
1094
|
for (const session of sessions) {
|
|
1007
1095
|
try {
|
|
1008
|
-
const committedAuthority =
|
|
1096
|
+
const committedAuthority = followAuthorities.get((0, authority_key_codec_1.encodeAuthorityKey)('actor', session.state.actorId).value);
|
|
1097
|
+
const committedSpot = followAuthorities.get(spotKey.value);
|
|
1009
1098
|
const targetActorRef = {
|
|
1010
1099
|
actorId: session.state.actorId,
|
|
1011
1100
|
objectGeneration: actorAuthorities.get(session.state.actorId).objectGeneration,
|
|
1012
1101
|
meshName,
|
|
1013
1102
|
nodeRid: target.rid
|
|
1014
1103
|
};
|
|
1015
|
-
await session.prepared.commit(
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
spotId: activation.spotId,
|
|
1019
|
-
spotKind: kind === 'user_spot' ? contracts_1.ZLinkSpotKind.User : contracts_1.ZLinkSpotKind.Instance,
|
|
1020
|
-
authorityOwnerGeneration: committedAuthority.authorityOwnerGeneration
|
|
1021
|
-
}, targetActorRef, (0, actor_transfer_runtime_1.committedActorOwnerFence)(session.state.actorId, targetActorRef, committedAuthority));
|
|
1104
|
+
await session.prepared.commit(committedSpotRoute(meshName, activation.spotId, kind === 'user_spot' ? contracts_1.ZLinkSpotKind.User : contracts_1.ZLinkSpotKind.Instance, target.rid, committedSpot), targetActorRef, committedAuthority === undefined || committedSpot === undefined
|
|
1105
|
+
? null
|
|
1106
|
+
: (0, actor_transfer_runtime_1.committedActorOwnerFence)(session.state.actorId, targetActorRef, committedAuthority));
|
|
1022
1107
|
}
|
|
1023
1108
|
catch (error) {
|
|
1024
1109
|
failures.push(error);
|
|
@@ -1050,6 +1135,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1050
1135
|
abortSeal: async () => {
|
|
1051
1136
|
const failures = [];
|
|
1052
1137
|
if (spotCapture !== undefined) {
|
|
1138
|
+
// Spot model §5.1: Continued completes while the source seal still holds jobs.
|
|
1139
|
+
if (readinessBoundaryConsumed) {
|
|
1140
|
+
try {
|
|
1141
|
+
await activation.completeConsumedRelocationBoundary(contracts_1.ZLinkSpotRelocationReadyOutcome.Continued);
|
|
1142
|
+
}
|
|
1143
|
+
catch (error) {
|
|
1144
|
+
failures.push(error);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1053
1147
|
try {
|
|
1054
1148
|
activation.abortRelocation(spotCapture);
|
|
1055
1149
|
}
|
|
@@ -1089,7 +1183,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1089
1183
|
spotObjectGeneration: spotAuthority.objectGeneration,
|
|
1090
1184
|
membershipEpoch: state.spotMembershipEpoch > 0n ? state.spotMembershipEpoch : 1n
|
|
1091
1185
|
}));
|
|
1092
|
-
|
|
1186
|
+
const relocationMetric = spotRegistration.relocation.kind === 'disabled'
|
|
1187
|
+
? undefined
|
|
1188
|
+
: this.options.metrics?.startRelocation(meshName, kind, spotRegistration.relocation.kind);
|
|
1093
1189
|
try {
|
|
1094
1190
|
const waitForRelocationBoundary = activation.waitForRelocationBoundary;
|
|
1095
1191
|
readinessBoundaryConsumed =
|
|
@@ -1109,10 +1205,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1109
1205
|
(0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId).value,
|
|
1110
1206
|
actorAuthorities.get(state.actorId)
|
|
1111
1207
|
])
|
|
1112
|
-
]), signal, sessions);
|
|
1208
|
+
]), signal, sessions, undefined, undefined, relocationMetric);
|
|
1113
1209
|
}
|
|
1114
1210
|
catch (error) {
|
|
1115
|
-
outcome = 'failed';
|
|
1116
1211
|
if (readinessBoundaryConsumed &&
|
|
1117
1212
|
!(error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) &&
|
|
1118
1213
|
!(error instanceof service_relocation_coordinator_1.ServiceRelocationAuthorityError)) {
|
|
@@ -1123,9 +1218,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1123
1218
|
throw error;
|
|
1124
1219
|
}
|
|
1125
1220
|
finally {
|
|
1221
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1126
1222
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1127
1223
|
if (interruptionStartedAt !== undefined) {
|
|
1128
|
-
this.recordRelocationInterruption(meshName, kind, kind === 'user_spot' ? 'spot_wide' : undefined, interruptionStartedAt
|
|
1224
|
+
this.recordRelocationInterruption(meshName, kind, kind === 'user_spot' ? 'spot_wide' : undefined, interruptionStartedAt);
|
|
1129
1225
|
}
|
|
1130
1226
|
}
|
|
1131
1227
|
}
|
|
@@ -1136,21 +1232,19 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1136
1232
|
(0, diagnostics_1.relocationDebug)('standalone_actor.begin', { actorId: state.actorId });
|
|
1137
1233
|
}
|
|
1138
1234
|
const authority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId), signal);
|
|
1139
|
-
|
|
1235
|
+
const actorPolicy = this.actorRegistration(state.meshName ?? meshName, authority.allocation.stableType).relocation.kind;
|
|
1236
|
+
target ??= await this.selectTarget(meshName, this.requireMeshNode(meshName).status().routingId, targetApplicationVersion, [
|
|
1140
1237
|
{
|
|
1141
1238
|
objectKind: 'actor',
|
|
1142
1239
|
stableType: authority.allocation.stableType,
|
|
1143
|
-
policy:
|
|
1240
|
+
policy: actorPolicy,
|
|
1144
1241
|
count: 1
|
|
1145
1242
|
}
|
|
1146
1243
|
], signal);
|
|
1147
1244
|
const sessions = [];
|
|
1148
1245
|
const membershipSpotId = targetMembership?.spotId ?? (target.entrySpotId ?? String(target.rid));
|
|
1149
1246
|
const aggregateId = aggregateIdOverride ?? this.reserveRelocationId();
|
|
1150
|
-
const unit = this.actorCaptureUnit(state, authority, sessions, true, target, meshName,
|
|
1151
|
-
spotId: membershipSpotId,
|
|
1152
|
-
spotKind: targetMembership?.spotKind ?? contracts_1.ZLinkSpotKind.Entry
|
|
1153
|
-
}, relocationWireId(aggregateId), retainSourceForActorJoin);
|
|
1247
|
+
const unit = this.actorCaptureUnit(state, authority, sessions, true, target, meshName, targetMembership, relocationWireId(aggregateId), retainSourceForActorJoin);
|
|
1154
1248
|
const membership = {
|
|
1155
1249
|
actorKey: (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId).value,
|
|
1156
1250
|
spotKey: (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(membershipSpotId)).value,
|
|
@@ -1165,7 +1259,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1165
1259
|
return unit.seal(captureSignal);
|
|
1166
1260
|
}
|
|
1167
1261
|
};
|
|
1168
|
-
|
|
1262
|
+
const relocationMetric = actorPolicy === 'disabled'
|
|
1263
|
+
? undefined
|
|
1264
|
+
: this.options.metrics?.startRelocation(meshName, 'actor', actorPolicy);
|
|
1169
1265
|
try {
|
|
1170
1266
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1171
1267
|
(0, diagnostics_1.relocationDebug)('standalone_actor.wait_budget', {
|
|
@@ -1181,16 +1277,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1181
1277
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1182
1278
|
(0, diagnostics_1.relocationDebug)('standalone_actor.capture_complete', { actorId: state.actorId });
|
|
1183
1279
|
}
|
|
1184
|
-
await this.runCoordinator(meshName, target, authority, captured, new Map([[measuredUnit.authorityKey, authority]]), signal, sessions, advertisedReceiveChunkLimitBytes, deadlineAtMs);
|
|
1185
|
-
}
|
|
1186
|
-
catch (error) {
|
|
1187
|
-
outcome = 'failed';
|
|
1188
|
-
throw error;
|
|
1280
|
+
await this.runCoordinator(meshName, target, authority, captured, new Map([[measuredUnit.authorityKey, authority]]), signal, sessions, advertisedReceiveChunkLimitBytes, deadlineAtMs, relocationMetric);
|
|
1189
1281
|
}
|
|
1190
1282
|
finally {
|
|
1283
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1191
1284
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1192
1285
|
if (interruptionStartedAt !== undefined) {
|
|
1193
|
-
this.recordRelocationInterruption(meshName, 'actor', undefined, interruptionStartedAt
|
|
1286
|
+
this.recordRelocationInterruption(meshName, 'actor', undefined, interruptionStartedAt);
|
|
1194
1287
|
}
|
|
1195
1288
|
}
|
|
1196
1289
|
}
|
|
@@ -1201,7 +1294,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1201
1294
|
const spotKey = (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(activation.spotId));
|
|
1202
1295
|
const spotAuthority = await requireAuthority(this.requireLocationStore(), spotKey, signal);
|
|
1203
1296
|
const spotRegistration = this.spotRegistration(meshName, 'user_spot', spotAuthority.allocation.stableType);
|
|
1204
|
-
target ??= await this.selectTarget(meshName,
|
|
1297
|
+
target ??= await this.selectTarget(meshName, this.requireMeshNode(meshName).status().routingId, targetApplicationVersion, [
|
|
1205
1298
|
{
|
|
1206
1299
|
objectKind: 'user_spot',
|
|
1207
1300
|
stableType: spotAuthority.allocation.stableType,
|
|
@@ -1235,7 +1328,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1235
1328
|
// PerActor User Spots are stateless shells. Application state and the
|
|
1236
1329
|
// Spot relocation adapter are intentionally not part of this unit.
|
|
1237
1330
|
captureApplicationState: async () => Buffer.alloc(0),
|
|
1238
|
-
commitSeal: async () => {
|
|
1331
|
+
commitSeal: async (followAuthorities) => {
|
|
1239
1332
|
if (!sealCommitted) {
|
|
1240
1333
|
if (spotCapture === undefined || !(await activation.commitRelocation(spotCapture))) {
|
|
1241
1334
|
throw new Error(`PerActor Spot '${String(activation.spotId)}' relocation seal became stale.`);
|
|
@@ -1243,7 +1336,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1243
1336
|
sealCommitted = true;
|
|
1244
1337
|
}
|
|
1245
1338
|
if (spotMessageFollowSeal !== undefined) {
|
|
1246
|
-
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, spotKey, activation
|
|
1339
|
+
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, followAuthorities.get(spotKey.value), activation);
|
|
1247
1340
|
}
|
|
1248
1341
|
await this.requireSpotManager().completeRelocationSource(activation);
|
|
1249
1342
|
},
|
|
@@ -1260,20 +1353,19 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1260
1353
|
}
|
|
1261
1354
|
};
|
|
1262
1355
|
const aggregateId = this.reserveRelocationId();
|
|
1263
|
-
|
|
1356
|
+
const relocationMetric = spotRegistration.relocation.kind === 'disabled'
|
|
1357
|
+
? undefined
|
|
1358
|
+
: this.options.metrics?.startRelocation(meshName, 'user_spot', spotRegistration.relocation.kind);
|
|
1264
1359
|
try {
|
|
1265
1360
|
await this.waitForRelocationBudgetHeadroom(String(target.rid), signal);
|
|
1266
1361
|
const captured = await new service_relocation_object_owner_1.ServiceRelocationObjectCaptureOwner().captureUserSpotAggregate(aggregateId, 1n, spotUnit, [], [], signal);
|
|
1267
|
-
await this.runCoordinator(meshName, target, spotAuthority, captured, new Map([[spotKey.value, spotAuthority]]), signal, []);
|
|
1268
|
-
}
|
|
1269
|
-
catch (error) {
|
|
1270
|
-
outcome = 'failed';
|
|
1271
|
-
throw error;
|
|
1362
|
+
await this.runCoordinator(meshName, target, spotAuthority, captured, new Map([[spotKey.value, spotAuthority]]), signal, [], undefined, undefined, relocationMetric);
|
|
1272
1363
|
}
|
|
1273
1364
|
finally {
|
|
1365
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1274
1366
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1275
1367
|
if (interruptionStartedAt !== undefined) {
|
|
1276
|
-
this.recordRelocationInterruption(meshName, 'user_spot', 'per_actor', interruptionStartedAt
|
|
1368
|
+
this.recordRelocationInterruption(meshName, 'user_spot', 'per_actor', interruptionStartedAt);
|
|
1277
1369
|
}
|
|
1278
1370
|
}
|
|
1279
1371
|
}
|
|
@@ -1291,7 +1383,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1291
1383
|
ownSession = {
|
|
1292
1384
|
state,
|
|
1293
1385
|
actor: state.actor,
|
|
1294
|
-
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, signal, true, sessionRelocation)
|
|
1386
|
+
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, signal, true, sessionRelocation, target)
|
|
1295
1387
|
};
|
|
1296
1388
|
sessions.push(ownSession);
|
|
1297
1389
|
}
|
|
@@ -1319,7 +1411,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1319
1411
|
},
|
|
1320
1412
|
// The sealed Session identity is part of the canonical participant vector.
|
|
1321
1413
|
captureApplicationState: (signal) => this.captureApplication(registration.relocation, state.actor, signal),
|
|
1322
|
-
commitSeal: async () => {
|
|
1414
|
+
commitSeal: async (followAuthorities) => {
|
|
1323
1415
|
if (!standalone ||
|
|
1324
1416
|
ownSession === undefined ||
|
|
1325
1417
|
target === undefined ||
|
|
@@ -1327,33 +1419,36 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1327
1419
|
return;
|
|
1328
1420
|
const failures = [];
|
|
1329
1421
|
try {
|
|
1330
|
-
const
|
|
1331
|
-
const committedAuthority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId));
|
|
1422
|
+
const committedAuthority = followAuthorities.get((0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId).value);
|
|
1332
1423
|
const targetActorRef = {
|
|
1333
1424
|
actorId: state.actorId,
|
|
1334
1425
|
objectGeneration: authority.objectGeneration,
|
|
1335
1426
|
meshName,
|
|
1336
1427
|
nodeRid: target.rid
|
|
1337
1428
|
};
|
|
1338
|
-
await ownSession.prepared.commit({
|
|
1429
|
+
await ownSession.prepared.commit(targetMembership ?? {
|
|
1339
1430
|
routerChannelId: meshName,
|
|
1340
1431
|
targetNodeRid: target.rid,
|
|
1341
|
-
spotId:
|
|
1342
|
-
spotKind:
|
|
1343
|
-
|
|
1344
|
-
|
|
1432
|
+
spotId: (target.entrySpotId ?? String(target.rid)),
|
|
1433
|
+
spotKind: contracts_1.ZLinkSpotKind.Entry
|
|
1434
|
+
}, targetActorRef, committedAuthority === undefined
|
|
1435
|
+
? null
|
|
1436
|
+
: (0, actor_transfer_runtime_1.committedActorOwnerFence)(state.actorId, targetActorRef, committedAuthority));
|
|
1345
1437
|
}
|
|
1346
1438
|
catch (error) {
|
|
1347
1439
|
failures.push(error);
|
|
1348
1440
|
}
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1441
|
+
try {
|
|
1442
|
+
if (retainSourceForActorJoin) {
|
|
1443
|
+
this.requireActorManager().detachRelocationSource(state);
|
|
1352
1444
|
}
|
|
1353
|
-
|
|
1354
|
-
|
|
1445
|
+
else {
|
|
1446
|
+
await this.requireActorManager().completeRelocationSource(state.actorId);
|
|
1355
1447
|
}
|
|
1356
1448
|
}
|
|
1449
|
+
catch (error) {
|
|
1450
|
+
failures.push(error);
|
|
1451
|
+
}
|
|
1357
1452
|
if (failures.length > 0)
|
|
1358
1453
|
throw failures.length === 1
|
|
1359
1454
|
? failures[0]
|
|
@@ -1390,7 +1485,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1390
1485
|
}
|
|
1391
1486
|
async runCoordinator(meshName, target, primary, captured, _authorities, signal, sessions = [],
|
|
1392
1487
|
/** Target's advertised relocation state chunk cap (Actor Join only; spec 15 §4.2). */
|
|
1393
|
-
advertisedReceiveChunkLimitBytes, controlDeadlineAtMs = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS) {
|
|
1488
|
+
advertisedReceiveChunkLimitBytes, controlDeadlineAtMs = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS, relocationMetric) {
|
|
1394
1489
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1395
1490
|
(0, diagnostics_1.relocationDebug)('coordinator.begin', {
|
|
1396
1491
|
aggregateId: captured.envelope.aggregateId,
|
|
@@ -1401,12 +1496,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1401
1496
|
const coordinator = {
|
|
1402
1497
|
ownerId: primary.ownerId,
|
|
1403
1498
|
leaseGeneration: primary.ownerLeaseGeneration,
|
|
1404
|
-
nodeRid:
|
|
1499
|
+
nodeRid: localStatus.routingId,
|
|
1405
1500
|
nodeGeneration: localStatus.lifecycleGeneration,
|
|
1406
1501
|
expectedAuthorityStoreVersion: primary.storeVersion.value
|
|
1407
1502
|
};
|
|
1408
1503
|
const targetFence = {
|
|
1409
|
-
nodeRid:
|
|
1504
|
+
nodeRid: target.rid,
|
|
1410
1505
|
nodeGeneration: target.lifecycleGeneration,
|
|
1411
1506
|
ownerId: target.ownerId,
|
|
1412
1507
|
ownerLeaseGeneration: target.leaseGeneration
|
|
@@ -1419,11 +1514,6 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1419
1514
|
// attempt even when it reuses the same routing id.
|
|
1420
1515
|
const targetAttemptGeneration = target.lifecycleGeneration;
|
|
1421
1516
|
const relayAuthorityId = wireIdText(relocationWireId(captured.envelope.aggregateId));
|
|
1422
|
-
const relayAuthorityKey = primaryKey(captured.envelope);
|
|
1423
|
-
if (this.relocationAuthorityKeys.has(relayAuthorityId)) {
|
|
1424
|
-
throw new Error(`Relocation reply relay authority '${relayAuthorityId}' is already registered.`);
|
|
1425
|
-
}
|
|
1426
|
-
this.relocationAuthorityKeys.set(relayAuthorityId, relayAuthorityKey);
|
|
1427
1517
|
// The captured payload stays in source memory: this copy is the only
|
|
1428
1518
|
// handoff original until the cutover submit terminal and the
|
|
1429
1519
|
// retransmission window end (spec 28 §4.2).
|
|
@@ -1433,6 +1523,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1433
1523
|
? canonicalEnvelope
|
|
1434
1524
|
: appendCanonicalActorJoinRecovery(canonicalEnvelope, actorJoinProfile, coordinator, primary);
|
|
1435
1525
|
const encoded = (0, service_relocation_runtime_1.encodeServiceRelocationEnvelope)(transportEnvelope, target.applicationVersion);
|
|
1526
|
+
relocationMetric?.recordBytes(encoded.byteLength);
|
|
1436
1527
|
const limits = this.relocationLimits();
|
|
1437
1528
|
// Actor Join threads the target's admission-time advertised cap in;
|
|
1438
1529
|
// other relocation paths have no such negotiation and fall back to the
|
|
@@ -1460,7 +1551,8 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1460
1551
|
payloadTotalLength: BigInt(plan.totalLength),
|
|
1461
1552
|
payloadChunkCount: plan.chunkCount,
|
|
1462
1553
|
payloadChecksumCrc32c: plan.checksumCrc32c,
|
|
1463
|
-
applicationVersion: target.applicationVersion
|
|
1554
|
+
applicationVersion: target.applicationVersion,
|
|
1555
|
+
deadlineUnixMs: BigInt(Date.now() + Math.max(0, Math.ceil(controlDeadlineAtMs - performance.now())))
|
|
1464
1556
|
};
|
|
1465
1557
|
// Start the Restore request first (it also registers the temporary
|
|
1466
1558
|
// queue and the assembly on the target), then stream the payload chunks
|
|
@@ -1487,6 +1579,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1487
1579
|
// forward the source can no longer reopen dispatch without risking two
|
|
1488
1580
|
// owners.
|
|
1489
1581
|
readyReceived = true;
|
|
1582
|
+
actorJoinProfile?.relayReady();
|
|
1490
1583
|
const boundaryBatch = [];
|
|
1491
1584
|
for (const session of sessions) {
|
|
1492
1585
|
const participant = captured.envelope.participants.find((value) => value.objectKind === 'actor' &&
|
|
@@ -1539,7 +1632,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1539
1632
|
}
|
|
1540
1633
|
const window = this.beginSourceCutoverWindow(meshName, target.rid, relayAuthorityId, [...boundaryBatch.map((value) => value.frame), cutoverFrame], submittedRouteGeneration, limits);
|
|
1541
1634
|
try {
|
|
1542
|
-
settlement = await this.settleSourceAuthority(captured.envelope, primary, targetFence, restoreDeadlineReached, window);
|
|
1635
|
+
settlement = await this.settleSourceAuthority(captured.envelope, primary, targetFence, restoreDeadlineReached, window, relocationMetric);
|
|
1543
1636
|
}
|
|
1544
1637
|
finally {
|
|
1545
1638
|
this.closeSourceCutoverWindow(relayAuthorityId, window);
|
|
@@ -1554,22 +1647,31 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1554
1647
|
// Preserve won: the captured queue, timers and held records return to
|
|
1555
1648
|
// the source queue in their accepted order, and dispatch reopens.
|
|
1556
1649
|
await captured.abortSource();
|
|
1650
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
1557
1651
|
throw new Error(`Relocation '${relayAuthorityId}' ended by the source Preserve fence; the source kept the authority.`);
|
|
1558
1652
|
}
|
|
1559
1653
|
if (settlement === 'lost') {
|
|
1654
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
1560
1655
|
// Expired owner: dispatch and Store changes stop for good and every
|
|
1561
1656
|
// pending request without a terminal gets one Unavailable.
|
|
1562
1657
|
const unavailable = (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected, `Relocation '${relayAuthorityId}' source authority ended (source owner lease expired, authority Missing, or owner changed).`, true);
|
|
1563
1658
|
await captured.discardSource(unavailable);
|
|
1564
1659
|
throw new service_relocation_coordinator_1.ServiceRelocationAuthorityError(unavailable.message, { cause: unavailable });
|
|
1565
1660
|
}
|
|
1661
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
1566
1662
|
for (const session of sessions)
|
|
1567
1663
|
session.prepared.setReplayResults([]);
|
|
1568
|
-
await captured.commitSource();
|
|
1664
|
+
await captured.commitSource(settlement.followAuthorities);
|
|
1569
1665
|
await this.options.reconcileStatefulAuthorityRoutes?.(signal);
|
|
1570
1666
|
}
|
|
1571
1667
|
catch (error) {
|
|
1572
|
-
if (
|
|
1668
|
+
if (this.disposed &&
|
|
1669
|
+
(error instanceof ServiceRelocationRuntimeStoppedError ||
|
|
1670
|
+
(error instanceof runtime_values_1.ZLinkBackendResultError &&
|
|
1671
|
+
error.result ===
|
|
1672
|
+
(error.operation === 'request' ? runtime_values_1.RequestResult.Terminated : runtime_values_1.SubmitResult.Terminated))))
|
|
1673
|
+
this.completeSourceRelocationMetric(relocationMetric, undefined, true);
|
|
1674
|
+
if (settlement === undefined || typeof settlement === 'string')
|
|
1573
1675
|
throw error;
|
|
1574
1676
|
// Target commit is confirmed; later source cleanup failures cannot move
|
|
1575
1677
|
// the authority back (spec 28 §10).
|
|
@@ -1577,16 +1679,25 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1577
1679
|
throw new service_relocation_coordinator_1.ServiceRelocationPostCommitError(authority, { id: `${captured.envelope.aggregateId}:${captured.envelope.aggregateGeneration}` }, error);
|
|
1578
1680
|
}
|
|
1579
1681
|
finally {
|
|
1682
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1580
1683
|
if (!readyReceived) {
|
|
1581
1684
|
// Only an explicit failure before the Ready reply becomes accepted
|
|
1582
1685
|
// restores the source from the memory payload (spec 28 §9).
|
|
1583
1686
|
await captured.abortSource().catch(() => undefined);
|
|
1584
1687
|
}
|
|
1585
|
-
if (this.relocationAuthorityKeys.get(relayAuthorityId) === relayAuthorityKey) {
|
|
1586
|
-
this.relocationAuthorityKeys.delete(relayAuthorityId);
|
|
1587
|
-
}
|
|
1588
1688
|
}
|
|
1589
1689
|
}
|
|
1690
|
+
completeSourceRelocationMetric(metric, settlement, sourceStopped = false) {
|
|
1691
|
+
metric?.complete(settlement === 'source'
|
|
1692
|
+
? 'aborted'
|
|
1693
|
+
: settlement === 'lost'
|
|
1694
|
+
? 'failed'
|
|
1695
|
+
: settlement !== undefined
|
|
1696
|
+
? 'completed'
|
|
1697
|
+
: sourceStopped
|
|
1698
|
+
? 'shutdown'
|
|
1699
|
+
: 'failed');
|
|
1700
|
+
}
|
|
1590
1701
|
/** Streams the payload chunks of one stage under the effective in-flight budgets. */
|
|
1591
1702
|
async sendRelocationStateChunks(meshName, target, identity, encoded, plan, signal) {
|
|
1592
1703
|
for (let ordinal = 0; ordinal < plan.chunkCount; ordinal += 1) {
|
|
@@ -1721,15 +1832,18 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1721
1832
|
* excludes a late target commit. `target`: the
|
|
1722
1833
|
* target committed. `source`: Preserve fenced the target out while the
|
|
1723
1834
|
* source owner lease was valid. `lost`: the source owner lease ended first
|
|
1724
|
-
* or authority is Missing or
|
|
1835
|
+
* or authority is Missing or fails the same-object, advanced-generation fence.
|
|
1836
|
+
* A committed later owner has no Follow snapshot (Location runtime §10). An uncertain
|
|
1725
1837
|
* Store or lease result keeps the work retained and repeats the same decision.
|
|
1726
1838
|
*/
|
|
1727
|
-
async settleSourceAuthority(envelope, primary, target, restoreDeadlineReached, window) {
|
|
1839
|
+
async settleSourceAuthority(envelope, primary, target, restoreDeadlineReached, window, relocationMetric) {
|
|
1728
1840
|
const owner = { ownerId: primary.ownerId, leaseGeneration: primary.ownerLeaseGeneration };
|
|
1729
1841
|
const store = this.requireLocationStore();
|
|
1730
1842
|
for (;;) {
|
|
1731
|
-
if (this.disposed)
|
|
1732
|
-
|
|
1843
|
+
if (this.disposed) {
|
|
1844
|
+
this.completeSourceRelocationMetric(relocationMetric, undefined, true);
|
|
1845
|
+
throw new ServiceRelocationRuntimeStoppedError();
|
|
1846
|
+
}
|
|
1733
1847
|
await this.resendSourceCutover(window);
|
|
1734
1848
|
const observed = await this.readSourceSettlement(envelope, primary, target).catch((error) => {
|
|
1735
1849
|
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error))
|
|
@@ -1737,7 +1851,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1737
1851
|
return undefined;
|
|
1738
1852
|
});
|
|
1739
1853
|
if (observed?.kind === 'target')
|
|
1740
|
-
return
|
|
1854
|
+
return observed;
|
|
1741
1855
|
if (observed?.kind === 'other')
|
|
1742
1856
|
return 'lost';
|
|
1743
1857
|
if (restoreDeadlineReached() && observed?.kind === 'source') {
|
|
@@ -1772,6 +1886,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1772
1886
|
async readSourceSettlement(envelope, primary, target) {
|
|
1773
1887
|
const primaryAuthorityKey = primaryKey(envelope);
|
|
1774
1888
|
let committed = 0;
|
|
1889
|
+
const followAuthorities = new Map();
|
|
1775
1890
|
let primaryCurrent;
|
|
1776
1891
|
for (const participant of envelope.participants) {
|
|
1777
1892
|
const current = await this.requireLocationStore().readAuthority({
|
|
@@ -1780,18 +1895,22 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1780
1895
|
if (current.kind !== 'snapshot')
|
|
1781
1896
|
return { kind: 'other' };
|
|
1782
1897
|
if (current.objectGeneration === participant.objectGeneration &&
|
|
1783
|
-
current.ownerId === target.ownerId &&
|
|
1784
|
-
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
1785
1898
|
current.authorityOwnerGeneration > participant.authorityOwnerGeneration &&
|
|
1786
|
-
|
|
1787
|
-
|
|
1899
|
+
// Location runtime §6: an owner is the (OwnerId, OwnerLeaseGeneration) pair;
|
|
1900
|
+
// a restarted node keeps its OwnerId under a new lease generation.
|
|
1901
|
+
(current.ownerId !== primary.ownerId ||
|
|
1902
|
+
current.ownerLeaseGeneration !== primary.ownerLeaseGeneration)) {
|
|
1788
1903
|
committed += 1;
|
|
1904
|
+
// Location runtime §10: only the observed first target can receive Follow.
|
|
1905
|
+
if (current.ownerId === target.ownerId &&
|
|
1906
|
+
current.ownerLeaseGeneration === target.ownerLeaseGeneration)
|
|
1907
|
+
followAuthorities.set(participant.key, current);
|
|
1789
1908
|
}
|
|
1790
1909
|
if (participant.key === primaryAuthorityKey)
|
|
1791
1910
|
primaryCurrent = current;
|
|
1792
1911
|
}
|
|
1793
1912
|
if (committed === envelope.participants.length)
|
|
1794
|
-
return { kind: 'target' };
|
|
1913
|
+
return { kind: 'target', followAuthorities };
|
|
1795
1914
|
if (primaryCurrent !== undefined && isSourceOwner(primaryCurrent, primary)) {
|
|
1796
1915
|
return { kind: 'source', current: primaryCurrent };
|
|
1797
1916
|
}
|
|
@@ -1826,11 +1945,21 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1826
1945
|
}
|
|
1827
1946
|
/** Restores the hidden object and its saved-work prefix before Ready. */
|
|
1828
1947
|
async handlePrepareControl(meshName, stagingId, request, sourceNodeRid, signal) {
|
|
1948
|
+
// Host relocation flow 05: a sealed target refuses a new PREPARE before any
|
|
1949
|
+
// staging. The typed ShuttingDown reaches the source as relocationFailed.
|
|
1950
|
+
if (this.targetAdmissionSealed()) {
|
|
1951
|
+
throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.ShuttingDown, 'Relocation target admission is sealed for shutdown.');
|
|
1952
|
+
}
|
|
1953
|
+
signal =
|
|
1954
|
+
signal === undefined
|
|
1955
|
+
? this.targetPreparationStop.signal
|
|
1956
|
+
: AbortSignal.any([signal, this.targetPreparationStop.signal]);
|
|
1957
|
+
signal.throwIfAborted();
|
|
1829
1958
|
const targetStatus = this.requireMeshNode(meshName).status();
|
|
1830
1959
|
const targetOwner = this.options.currentOwner();
|
|
1831
1960
|
const targetDescriptor = this.options.localDescriptor?.(meshName);
|
|
1832
1961
|
if (targetOwner === undefined ||
|
|
1833
|
-
|
|
1962
|
+
!(0, routing_id_1.routingIdsEqual)(targetStatus.routingId, request.target.nodeRid) ||
|
|
1834
1963
|
targetStatus.lifecycleGeneration !== request.target.nodeGeneration ||
|
|
1835
1964
|
targetOwner.ownerId !== request.target.ownerId ||
|
|
1836
1965
|
targetOwner.leaseGeneration !== request.target.ownerLeaseGeneration ||
|
|
@@ -1860,11 +1989,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1860
1989
|
payload = await pendingAssembly.assembly.payload();
|
|
1861
1990
|
}
|
|
1862
1991
|
catch (error) {
|
|
1863
|
-
//
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
// the generic InternalFailure code (spec 15 §4).
|
|
1992
|
+
// Target shutdown preserves its lifecycle failure instead of reporting payload corruption.
|
|
1993
|
+
this.targetAssemblies.delete(stagingId);
|
|
1994
|
+
if (this.targetPreparationStop.signal.aborted)
|
|
1995
|
+
throw this.targetPreparationStop.signal.reason;
|
|
1868
1996
|
if (this.disposed)
|
|
1869
1997
|
throw error;
|
|
1870
1998
|
const message = error !== null && typeof error === 'object' && 'message' in error
|
|
@@ -1891,11 +2019,43 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1891
2019
|
// rejects unrelated authority CAS until commit/abort.
|
|
1892
2020
|
materialized = await this.materializeTargetEnvelope(meshName, inventoryEnvelope, signal);
|
|
1893
2021
|
await materialized.owner.restoreSavedWork(materialized.staging, signal);
|
|
2022
|
+
for (const participant of inventoryEnvelope.participants) {
|
|
2023
|
+
const binding = participant.sessionBinding;
|
|
2024
|
+
if (binding === undefined)
|
|
2025
|
+
continue;
|
|
2026
|
+
const peerRid = (0, routing_id_1.decodeRoutingId)('', Buffer.from(binding.sessionOwnerNodeRid).toString('hex'));
|
|
2027
|
+
const node = this.requireMeshNode(meshName);
|
|
2028
|
+
if ((0, routing_id_1.routingIdsEqual)(node.status().routingId, peerRid))
|
|
2029
|
+
continue;
|
|
2030
|
+
if (node.isPeerRouteReady?.(peerRid, binding.sessionOwnerNodeGeneration) !== true) {
|
|
2031
|
+
// Subscribe before requesting reconciliation so a synchronous admission cannot be lost.
|
|
2032
|
+
const wait = node.waitForPeerRoute;
|
|
2033
|
+
if (wait === undefined)
|
|
2034
|
+
throw new Error('Mesh peer route observation is unavailable.');
|
|
2035
|
+
const ready = wait.call(node, peerRid, binding.sessionOwnerNodeGeneration, request.deadlineUnixMs, signal);
|
|
2036
|
+
void this.options
|
|
2037
|
+
.requestPeerReconcile?.(meshName, String(peerRid))
|
|
2038
|
+
.catch((error) => console.error('[zlink.runtime.relocation.session_route_reconcile_failed]', error));
|
|
2039
|
+
try {
|
|
2040
|
+
await ready;
|
|
2041
|
+
}
|
|
2042
|
+
catch (error) {
|
|
2043
|
+
if (error instanceof operation_registry_1.OperationTimeoutError) {
|
|
2044
|
+
throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.DeadlineExceeded, 'Relocation source operation deadline exceeded waiting for Session owner route.');
|
|
2045
|
+
}
|
|
2046
|
+
throw error;
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
if (request.deadlineUnixMs <= BigInt(Date.now())) {
|
|
2051
|
+
throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.DeadlineExceeded, 'Relocation source operation deadline exceeded before relay-ready.');
|
|
2052
|
+
}
|
|
2053
|
+
signal.throwIfAborted();
|
|
1894
2054
|
reservation = await this.reserveTargetForPrepare(meshName, request, inventoryEnvelope, signal);
|
|
1895
2055
|
const offer = {
|
|
1896
2056
|
prepare: request,
|
|
1897
2057
|
prepareFingerprint: fingerprint,
|
|
1898
|
-
authenticatedSourceNodeRid:
|
|
2058
|
+
authenticatedSourceNodeRid: sourceNodeRid,
|
|
1899
2059
|
envelope: inventoryEnvelope,
|
|
1900
2060
|
reservation
|
|
1901
2061
|
};
|
|
@@ -1912,6 +2072,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1912
2072
|
this.targetStages.set(stagingId, stage);
|
|
1913
2073
|
}
|
|
1914
2074
|
catch (error) {
|
|
2075
|
+
this.targetAssemblies.delete(stagingId);
|
|
1915
2076
|
releaseJoinAdmission();
|
|
1916
2077
|
releaseActivation();
|
|
1917
2078
|
if (reservation !== undefined) {
|
|
@@ -2052,24 +2213,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2052
2213
|
stage.releaseActivation();
|
|
2053
2214
|
resumeStartedAt = performance.now();
|
|
2054
2215
|
stage.phase = 'committed';
|
|
2055
|
-
await stage.owner.normalize(stage.staging, authority, signal);
|
|
2056
2216
|
await stage.owner.publish(stage.staging, authority, signal);
|
|
2057
|
-
// The target may have visited this Actor before and still retain that
|
|
2058
|
-
// tenure in its private Actor-client resolver. The target-only CAS is
|
|
2059
|
-
// now authoritative, so expire the old projection before OnJoinedActor
|
|
2060
|
-
// can issue an Actor request and deadlock behind source leave.
|
|
2061
|
-
for (const hidden of stage.staging.hidden.values()) {
|
|
2062
|
-
if (hidden.actor !== undefined) {
|
|
2063
|
-
this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
|
|
2064
|
-
}
|
|
2065
|
-
}
|
|
2066
|
-
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
2067
2217
|
// Clear the final relocation publication before admission becomes
|
|
2068
2218
|
// externally visible so a later Join cannot reserve against the
|
|
2069
2219
|
// previous root. A bounded CAS conflict must not roll back already
|
|
2070
2220
|
// committed target ownership or strand its staged object.
|
|
2071
2221
|
try {
|
|
2072
|
-
await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
2222
|
+
authority = await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
2073
2223
|
}
|
|
2074
2224
|
catch (error) {
|
|
2075
2225
|
console.warn('[zlink.runtime.relocation.publication_clear_failed]', stagingId, error);
|
|
@@ -2077,15 +2227,33 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2077
2227
|
mesh_name: meshName
|
|
2078
2228
|
});
|
|
2079
2229
|
}
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2230
|
+
for (const [index, participant] of stage.staging.envelope.participants.entries()) {
|
|
2231
|
+
const hidden = stage.staging.hidden.get(participant.key);
|
|
2232
|
+
if (hidden?.actor === undefined || hidden.replayPackets.length === 0)
|
|
2233
|
+
continue;
|
|
2234
|
+
const coordinator = this.targetReplyRelayCoordinator(meshName, stage, authority);
|
|
2235
|
+
hidden.replayStart = stage.finalize;
|
|
2236
|
+
hidden.onReplayResult = (packet, result) => this.relayTerminalReply(meshName, stage, coordinator, BigInt(index + 1), packet, result, signal);
|
|
2237
|
+
}
|
|
2238
|
+
await stage.owner.normalize(stage.staging, authority, signal);
|
|
2239
|
+
// The target may have visited this Actor before and still retain that
|
|
2240
|
+
// tenure in its private Actor-client resolver. The target-only CAS is
|
|
2241
|
+
// now authoritative, so expire the old projection before OnJoinedActor
|
|
2242
|
+
// can issue an Actor request and deadlock behind source leave.
|
|
2243
|
+
for (const hidden of stage.staging.hidden.values()) {
|
|
2244
|
+
if (hidden.actor !== undefined) {
|
|
2245
|
+
this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
2083
2249
|
await this.publishSessionRoutes(stage.staging);
|
|
2084
|
-
await this.relayTerminalReplies(meshName, stage, this.targetReplyRelayCoordinator(meshName, stage, authority), signal);
|
|
2085
2250
|
stage.phase = 'open';
|
|
2086
2251
|
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
2087
2252
|
this.targetStages.delete(stagingId);
|
|
2088
2253
|
this.targetAssemblies.delete(stagingId);
|
|
2254
|
+
await stage.owner.openAdmission(stage.staging, signal);
|
|
2255
|
+
// S2→S3: owner confirmation to application dispatch opening.
|
|
2256
|
+
this.options.metrics?.duration(runtime_metrics_1.METRIC_NAMES.RelocationTargetResume, Math.max(0, performance.now() - resumeStartedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND, { mesh_name: meshName });
|
|
2089
2257
|
}
|
|
2090
2258
|
catch (error) {
|
|
2091
2259
|
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
@@ -2182,9 +2350,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2182
2350
|
if (profile.state.actorId !== input.actorId || !input.succeeded) {
|
|
2183
2351
|
throw new Error(`Actor Join relocation '${input.transferId}' source leave identity changed.`);
|
|
2184
2352
|
}
|
|
2185
|
-
this.
|
|
2186
|
-
this.terminalActorJoinSourceLeaves.remember(input.transferId, input.actorId);
|
|
2187
|
-
void this.finalizeActorJoinSource(profile);
|
|
2353
|
+
void this.retireActorJoinSource(input.transferId, profile);
|
|
2188
2354
|
return true;
|
|
2189
2355
|
}
|
|
2190
2356
|
async sendControlOneWay(meshName, targetNodeRid, request) {
|
|
@@ -2206,7 +2372,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2206
2372
|
// row; the caller records it and leaves the committed ownership intact.
|
|
2207
2373
|
const publication = this.codec.read(primary.payload);
|
|
2208
2374
|
if (publication === undefined)
|
|
2209
|
-
return;
|
|
2375
|
+
return primary;
|
|
2210
2376
|
const result = await this.requireLocationStore().compareExchangeAuthority(stage.staging.primaryAuthorityKey, primary.storeVersion, {
|
|
2211
2377
|
kind: 'put',
|
|
2212
2378
|
generationTransition: 'preserve',
|
|
@@ -2215,39 +2381,52 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2215
2381
|
if (result.kind !== 'stored') {
|
|
2216
2382
|
throw new Error('Relocation target authority normalization CAS failed.');
|
|
2217
2383
|
}
|
|
2384
|
+
return { ...result, kind: 'snapshot' };
|
|
2218
2385
|
}
|
|
2219
2386
|
async handleReplyRelay(meshName, relay, sourceNodeRid, signal) {
|
|
2220
2387
|
if (sourceNodeRid === null) {
|
|
2221
2388
|
throw new Error('Relocation reply relay has no authenticated target node.');
|
|
2222
2389
|
}
|
|
2223
|
-
await this.requireAdmittedPeer(meshName,
|
|
2224
|
-
const
|
|
2225
|
-
const
|
|
2226
|
-
if (authorityKey === undefined) {
|
|
2227
|
-
throw new Error('Relocation reply relay has no registered durable authority.');
|
|
2228
|
-
}
|
|
2229
|
-
const authority = await requireAuthority(this.requireLocationStore(), { value: authorityKey }, signal);
|
|
2230
|
-
if (authority.storeVersion.value !== relay.coordinator.expectedAuthorityStoreVersion ||
|
|
2231
|
-
authority.ownerId !== relay.coordinator.ownerId ||
|
|
2232
|
-
authority.ownerLeaseGeneration !== relay.coordinator.leaseGeneration ||
|
|
2233
|
-
String(authority.allocation.descriptor.rid) !== relay.coordinator.nodeRid ||
|
|
2234
|
-
authority.allocation.descriptorLifecycleGeneration !== relay.coordinator.nodeGeneration) {
|
|
2235
|
-
throw new Error('Relocation reply relay coordinator authority fence is stale.');
|
|
2236
|
-
}
|
|
2237
|
-
const accepted = this.options.actorTransfer.relayCanonicalMaintenanceTerminal(wireIdText(relay.operation), relay.replyRouteId.toString(), handoffResultFromRelay(relay), sourceNodeRid);
|
|
2238
|
-
if ((accepted.status !== 'terminalReceived' && accepted.status !== 'alreadyTerminal') ||
|
|
2239
|
-
accepted.source === undefined) {
|
|
2240
|
-
throw new Error('Canonical relocation reply relay collided with another source route.');
|
|
2241
|
-
}
|
|
2242
|
-
const requestSource = handoffSourceFence(accepted.source);
|
|
2243
|
-
const localStatus = this.requireMeshNode(meshName).status();
|
|
2390
|
+
await this.requireAdmittedPeer(meshName, sourceNodeRid, relay.coordinator, signal);
|
|
2391
|
+
const node = this.requireMeshNode(meshName);
|
|
2392
|
+
const localStatus = node.status();
|
|
2244
2393
|
const localOwner = this.options.currentOwner();
|
|
2245
|
-
if (localOwner === undefined
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2394
|
+
if (localOwner === undefined)
|
|
2395
|
+
throw new Error('Relocation requester owner lease is unavailable.');
|
|
2396
|
+
let status = node.acceptRelocatedActorTerminal?.(relay.operation, relay.replyRouteId, {
|
|
2397
|
+
terminalResult: relay.terminalResult,
|
|
2398
|
+
failureCode: relay.failureCode,
|
|
2399
|
+
...(relay.payload === undefined
|
|
2400
|
+
? {}
|
|
2401
|
+
: {
|
|
2402
|
+
payload: {
|
|
2403
|
+
packetName: relay.payload.packetName,
|
|
2404
|
+
contentType: relay.payload.contentType,
|
|
2405
|
+
payload: Buffer.from(relay.payload.bytes)
|
|
2406
|
+
}
|
|
2407
|
+
})
|
|
2408
|
+
});
|
|
2409
|
+
let requestSource = {
|
|
2410
|
+
ownerId: localOwner.ownerId,
|
|
2411
|
+
leaseGeneration: localOwner.leaseGeneration,
|
|
2412
|
+
nodeRid: localStatus.routingId,
|
|
2413
|
+
nodeGeneration: localStatus.lifecycleGeneration
|
|
2414
|
+
};
|
|
2415
|
+
if (status === undefined) {
|
|
2416
|
+
const accepted = this.options.actorTransfer.relayCanonicalMaintenanceTerminal(relay.operation.high.toString(16).padStart(16, '0') +
|
|
2417
|
+
relay.operation.low.toString(16).padStart(16, '0'), relay.replyRouteId.toString(), handoffResultFromRelay(relay), sourceNodeRid);
|
|
2418
|
+
if (accepted.status === 'notAcknowledged' || accepted.source === undefined) {
|
|
2419
|
+
throw new Error('Canonical relocation reply relay collided with another source route.');
|
|
2420
|
+
}
|
|
2421
|
+
requestSource = handoffSourceFence(accepted.source);
|
|
2422
|
+
if (!sameRequestSource(requestSource, {
|
|
2423
|
+
ownerId: localOwner.ownerId,
|
|
2424
|
+
leaseGeneration: localOwner.leaseGeneration,
|
|
2425
|
+
nodeRid: localStatus.routingId,
|
|
2426
|
+
nodeGeneration: localStatus.lifecycleGeneration
|
|
2427
|
+
}))
|
|
2428
|
+
throw new Error('Relocation reply relay request-source fence changed after capture.');
|
|
2429
|
+
status = accepted.status;
|
|
2251
2430
|
}
|
|
2252
2431
|
const ack = {
|
|
2253
2432
|
relocation: relay.relocation,
|
|
@@ -2255,16 +2434,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2255
2434
|
operation: relay.operation,
|
|
2256
2435
|
replyRouteId: relay.replyRouteId,
|
|
2257
2436
|
requestSource,
|
|
2258
|
-
status
|
|
2437
|
+
status
|
|
2259
2438
|
};
|
|
2260
2439
|
if ((await this.sendInfrastructureControl(meshName, sourceNodeRid, (0, service_stateful_wire_codec_1.encodeMaintenanceReplyRelayAck)(ack))) !== runtime_values_1.SubmitResult.Ok) {
|
|
2261
2440
|
throw new Error('Relocation reply relay ACK send was rejected.');
|
|
2262
2441
|
}
|
|
2263
2442
|
}
|
|
2264
2443
|
async sendReplyRelay(meshName, ackTargetNodeRid, request, expectedRequestSource, signal) {
|
|
2265
|
-
const targetRid = String(ackTargetNodeRid);
|
|
2266
2444
|
const identityKey = replyRelayIdentityKey(request);
|
|
2267
|
-
const key = replyRelayPendingKey(
|
|
2445
|
+
const key = replyRelayPendingKey(ackTargetNodeRid, request);
|
|
2268
2446
|
if (this.pendingReplyRelays.has(key)) {
|
|
2269
2447
|
throw new Error(`Relocation reply relay ACK '${key}' is already pending.`);
|
|
2270
2448
|
}
|
|
@@ -2278,7 +2456,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2278
2456
|
action();
|
|
2279
2457
|
};
|
|
2280
2458
|
const pending = {
|
|
2281
|
-
ackTargetNodeRid
|
|
2459
|
+
ackTargetNodeRid,
|
|
2282
2460
|
identityKey,
|
|
2283
2461
|
request,
|
|
2284
2462
|
expectedRequestSource,
|
|
@@ -2314,7 +2492,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2314
2492
|
}
|
|
2315
2493
|
async acceptReplyRelayAck(meshName, ack, sourceNodeRid, signal) {
|
|
2316
2494
|
const identityKey = replyRelayIdentityKey(ack);
|
|
2317
|
-
const exactKey = sourceNodeRid === null ? undefined : replyRelayPendingKey(
|
|
2495
|
+
const exactKey = sourceNodeRid === null ? undefined : replyRelayPendingKey(sourceNodeRid, ack);
|
|
2318
2496
|
const pending = exactKey === undefined ? undefined : this.pendingReplyRelays.get(exactKey);
|
|
2319
2497
|
if (pending === undefined) {
|
|
2320
2498
|
const collision = [...this.pendingReplyRelays.values()].find((value) => value.identityKey === identityKey);
|
|
@@ -2337,12 +2515,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2337
2515
|
}
|
|
2338
2516
|
}
|
|
2339
2517
|
async requireAdmittedPeer(meshName, authenticatedRid, expected, signal) {
|
|
2340
|
-
const descriptor = (await this.options.liveDescriptors(meshName, signal)).find((value) =>
|
|
2518
|
+
const descriptor = (await this.options.liveDescriptors(meshName, signal)).find((value) => (0, routing_id_1.routingIdsEqual)(value.rid, authenticatedRid) && value.state === 1);
|
|
2341
2519
|
if (descriptor === undefined ||
|
|
2342
2520
|
(expected !== undefined &&
|
|
2343
2521
|
(descriptor.ownerId !== expected.ownerId ||
|
|
2344
2522
|
descriptor.leaseGeneration !== expected.leaseGeneration ||
|
|
2345
|
-
|
|
2523
|
+
!(0, routing_id_1.routingIdsEqual)(descriptor.rid, expected.nodeRid) ||
|
|
2346
2524
|
descriptor.lifecycleGeneration !== expected.nodeGeneration))) {
|
|
2347
2525
|
throw new Error('Relocation reply relay peer is not the exact admitted source fence.');
|
|
2348
2526
|
}
|
|
@@ -2354,39 +2532,32 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2354
2532
|
if (owner === undefined ||
|
|
2355
2533
|
owner.ownerId !== target.ownerId ||
|
|
2356
2534
|
owner.leaseGeneration !== target.ownerLeaseGeneration ||
|
|
2357
|
-
|
|
2535
|
+
!(0, routing_id_1.routingIdsEqual)(status.routingId, target.nodeRid) ||
|
|
2358
2536
|
status.lifecycleGeneration !== target.nodeGeneration) {
|
|
2359
2537
|
throw new Error('Relocation reply relay target coordinator fence is stale.');
|
|
2360
2538
|
}
|
|
2361
2539
|
return {
|
|
2362
2540
|
ownerId: owner.ownerId,
|
|
2363
2541
|
leaseGeneration: owner.leaseGeneration,
|
|
2364
|
-
nodeRid:
|
|
2542
|
+
nodeRid: status.routingId,
|
|
2365
2543
|
nodeGeneration: status.lifecycleGeneration,
|
|
2366
2544
|
expectedAuthorityStoreVersion: authority.storeVersion.value
|
|
2367
2545
|
};
|
|
2368
2546
|
}
|
|
2369
|
-
async
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
operationId: packet.messageFollowContext.operationId,
|
|
2384
|
-
source: packet.source,
|
|
2385
|
-
result
|
|
2386
|
-
};
|
|
2387
|
-
await this.sendReplyRelay(meshName, packet.source.nodeRid, maintenanceReplyRelay(stage, coordinator, BigInt(participantIndex + 1), completion), handoffSourceFence(packet.source), signal);
|
|
2388
|
-
}
|
|
2389
|
-
}
|
|
2547
|
+
async relayTerminalReply(meshName, stage, coordinator, participantId, packet, result, signal) {
|
|
2548
|
+
if (!packet.returnResponse || packet.source === undefined)
|
|
2549
|
+
return;
|
|
2550
|
+
if (result.index !== packet.index) {
|
|
2551
|
+
throw new Error('Relocation request result changed after target dispatch.');
|
|
2552
|
+
}
|
|
2553
|
+
const completion = {
|
|
2554
|
+
index: packet.index,
|
|
2555
|
+
operationId: packet.messageFollowContext.operationId,
|
|
2556
|
+
source: packet.source,
|
|
2557
|
+
result
|
|
2558
|
+
};
|
|
2559
|
+
const source = handoffSourceFence(packet.source);
|
|
2560
|
+
await this.sendReplyRelay(meshName, source.nodeRid, maintenanceReplyRelay(stage, coordinator, participantId, completion, packet, this.options.registration.messageSerializers, this.options.flowCreationEnabled?.() ?? false), source, signal);
|
|
2390
2561
|
}
|
|
2391
2562
|
async exactSourceLeaseExpired(source, signal) {
|
|
2392
2563
|
const lease = await this.requireLocationStore().readOwnerLease(source.ownerId, signal);
|
|
@@ -2414,7 +2585,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2414
2585
|
}
|
|
2415
2586
|
const replyFrames = await this.requestInfrastructureControl(meshName, targetNodeRid, frames, remainingMs, signal, onSubmitted);
|
|
2416
2587
|
if (replyFrames.length !== 1) {
|
|
2417
|
-
throw new
|
|
2588
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError(`Relocation control '${key}' reply must contain exactly one frame.`);
|
|
2418
2589
|
}
|
|
2419
2590
|
const response = (0, service_relocation_control_1.decodeServiceRelocationControlResponse)(Buffer.from(replyFrames[0]));
|
|
2420
2591
|
if (response.kind === 'failed') {
|
|
@@ -2427,11 +2598,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2427
2598
|
async selectTarget(meshName, localRid, targetApplicationVersion, requirements = [], signal) {
|
|
2428
2599
|
const descriptors = await this.options.liveDescriptors(meshName, signal);
|
|
2429
2600
|
const local = this.options.localDescriptor?.(meshName) ??
|
|
2430
|
-
descriptors.find((descriptor) =>
|
|
2601
|
+
descriptors.find((descriptor) => (0, routing_id_1.routingIdsEqual)(descriptor.rid, localRid));
|
|
2431
2602
|
const requiredVersion = targetApplicationVersion ?? local?.applicationVersion;
|
|
2432
2603
|
const peers = this.requireMeshNode(meshName).peers();
|
|
2433
2604
|
const candidates = descriptors
|
|
2434
|
-
.filter((descriptor) =>
|
|
2605
|
+
.filter((descriptor) => !(0, routing_id_1.routingIdsEqual)(descriptor.rid, localRid) &&
|
|
2435
2606
|
descriptor.state === contracts_1.ZLinkFrameworkRuntimeState.Serving &&
|
|
2436
2607
|
descriptor.objectRole === contracts_1.ZLinkObjectRole.Server &&
|
|
2437
2608
|
descriptor.placementWeight > 0 &&
|
|
@@ -2439,7 +2610,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2439
2610
|
(local?.maintenanceWave === undefined ||
|
|
2440
2611
|
descriptor.maintenanceWave !== local.maintenanceWave) &&
|
|
2441
2612
|
peers.some((peer) => peer.routingId !== null &&
|
|
2442
|
-
|
|
2613
|
+
(0, routing_id_1.routingIdsEqual)(peer.routingId, descriptor.rid) &&
|
|
2443
2614
|
peer.lifecycleGeneration === descriptor.lifecycleGeneration &&
|
|
2444
2615
|
peer.state === service_runtime_contracts_1.MeshPeerRuntimeState.Serving) &&
|
|
2445
2616
|
relocationTargetSupports(descriptor, requirements))
|
|
@@ -2852,19 +3023,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2852
3023
|
async createRelocationAdapter(adapterType) {
|
|
2853
3024
|
return (await (0, spot_provider_1.createProviderInstance)(adapterType, this.options.providerResolver));
|
|
2854
3025
|
}
|
|
2855
|
-
recordRelocationInterruption(meshName, unitKind, executionMode, startedAt
|
|
3026
|
+
recordRelocationInterruption(meshName, unitKind, executionMode, startedAt) {
|
|
2856
3027
|
const seconds = Math.max(0, performance.now() - startedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND;
|
|
2857
3028
|
const attributes = {
|
|
2858
3029
|
unit_kind: unitKind,
|
|
2859
3030
|
...(executionMode === undefined ? {} : { execution_mode: executionMode })
|
|
2860
3031
|
};
|
|
2861
3032
|
this.options.metrics?.duration(runtime_metrics_1.METRIC_NAMES.RelocationInterruption, seconds, attributes);
|
|
2862
|
-
this.options.metrics?.count(runtime_metrics_1.METRIC_NAMES.RelocationCompleted, 1, {
|
|
2863
|
-
mesh_name: meshName,
|
|
2864
|
-
object_kind: unitKind,
|
|
2865
|
-
policy: unitKind === 'user_spot' && executionMode === 'per_actor' ? 'recreate' : 'configured',
|
|
2866
|
-
outcome
|
|
2867
|
-
});
|
|
2868
3033
|
if (seconds <= 1)
|
|
2869
3034
|
return;
|
|
2870
3035
|
console.warn(`[zlink.runtime.relocation.changed] ${unitKind} admission interruption ` +
|
|
@@ -2907,24 +3072,25 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2907
3072
|
}
|
|
2908
3073
|
return result;
|
|
2909
3074
|
}
|
|
2910
|
-
async commitSpotMessageFollow(meshName, seal,
|
|
3075
|
+
async commitSpotMessageFollow(meshName, seal, committed, activation) {
|
|
2911
3076
|
const node = this.requireMeshNode(meshName);
|
|
2912
3077
|
const commit = node.commitSpotMessageFollowIngress;
|
|
2913
3078
|
if (commit === undefined) {
|
|
2914
3079
|
throw new Error('Spot Message Follow backend support is incomplete.');
|
|
2915
3080
|
}
|
|
2916
|
-
const
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
3081
|
+
const accepted = await commit.call(node, seal, committed === undefined
|
|
3082
|
+
? null
|
|
3083
|
+
: {
|
|
3084
|
+
spot: {
|
|
3085
|
+
spotId: String(activation.spotId),
|
|
3086
|
+
generation: committed.objectGeneration
|
|
3087
|
+
},
|
|
3088
|
+
targetNodeRid: String(committed.allocation.descriptor.rid),
|
|
3089
|
+
targetNodeGeneration: committed.allocation.descriptorLifecycleGeneration,
|
|
3090
|
+
authorityOwnerGeneration: committed.authorityOwnerGeneration,
|
|
3091
|
+
ownerLeaseGeneration: committed.ownerLeaseGeneration,
|
|
3092
|
+
storeVersion: committed.storeVersion.value
|
|
3093
|
+
}, this.relocationLimits().messageFollowDurationMs);
|
|
2928
3094
|
if (!accepted) {
|
|
2929
3095
|
throw new Error(`Spot '${String(activation.spotId)}' Message Follow route became stale.`);
|
|
2930
3096
|
}
|
|
@@ -3007,6 +3173,20 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
3007
3173
|
}
|
|
3008
3174
|
}
|
|
3009
3175
|
exports.ZLinkHostServiceRelocationRuntime = ZLinkHostServiceRelocationRuntime;
|
|
3176
|
+
function committedSpotRoute(meshName, spotId, spotKind, targetNodeRid, authority) {
|
|
3177
|
+
return {
|
|
3178
|
+
routerChannelId: meshName,
|
|
3179
|
+
targetNodeRid,
|
|
3180
|
+
spotId,
|
|
3181
|
+
spotKind,
|
|
3182
|
+
targetSpotGeneration: authority?.objectGeneration,
|
|
3183
|
+
targetNodeGeneration: authority?.allocation.descriptorLifecycleGeneration,
|
|
3184
|
+
authorityOwnerGeneration: authority?.authorityOwnerGeneration,
|
|
3185
|
+
targetOwnerId: authority?.ownerId,
|
|
3186
|
+
ownerLeaseGeneration: authority?.ownerLeaseGeneration,
|
|
3187
|
+
authorityStoreVersion: authority?.storeVersion.value
|
|
3188
|
+
};
|
|
3189
|
+
}
|
|
3010
3190
|
function canonicalActorJoinRecovery(participants) {
|
|
3011
3191
|
const matches = participants.flatMap((participant) => participant.queuedMessages.flatMap((message) => {
|
|
3012
3192
|
const recovery = (0, actor_join_recovery_codec_1.decodeCanonicalActorJoinRecoverySavedWork)(message.payload);
|
|
@@ -3062,6 +3242,9 @@ class LocalTargetPort {
|
|
|
3062
3242
|
async createHidden(participant, signal) {
|
|
3063
3243
|
const identity = (0, authority_key_codec_1.decodeAuthorityKey)({ value: participant.key }).globalId;
|
|
3064
3244
|
if (participant.objectKind === 'actor') {
|
|
3245
|
+
if (participant.timers.length !== 0) {
|
|
3246
|
+
throw new Error('Actor relocation cannot restore a Spot timer.');
|
|
3247
|
+
}
|
|
3065
3248
|
const membership = this.envelope.memberships.find((value) => value.actorKey === participant.key);
|
|
3066
3249
|
if (membership === undefined)
|
|
3067
3250
|
throw new Error(`Actor '${identity}' relocation membership is missing.`);
|
|
@@ -3074,20 +3257,28 @@ class LocalTargetPort {
|
|
|
3074
3257
|
participant,
|
|
3075
3258
|
actor,
|
|
3076
3259
|
initialized: false,
|
|
3077
|
-
restoredTimers: [],
|
|
3078
|
-
replayResults: [],
|
|
3079
3260
|
replayPackets: []
|
|
3080
3261
|
};
|
|
3081
3262
|
}
|
|
3082
3263
|
const registration = this.registration(participant.objectKind, participant.stableType);
|
|
3083
|
-
const activation = await this.requireSpotManager().prepareRelocationSpot(this.meshName, participant.objectKind, participant.stableType, registration.implementation, identity, participant.objectGeneration, participant.authorityOwnerGeneration,
|
|
3264
|
+
const activation = await this.requireSpotManager().prepareRelocationSpot(this.meshName, participant.objectKind, participant.stableType, registration.implementation, identity, participant.objectGeneration, participant.authorityOwnerGeneration, participant.timers.map((value) => ({
|
|
3265
|
+
name: value.timerId,
|
|
3266
|
+
handlerType: value.handlerType,
|
|
3267
|
+
periodMs: value.intervalMs,
|
|
3268
|
+
overrunPolicy: value.overrunPolicy,
|
|
3269
|
+
maxCatchUpTicks: value.maxCatchUpTicks,
|
|
3270
|
+
stopOnUnhandledException: value.stopOnUnhandledException,
|
|
3271
|
+
startedAtUnixMs: value.startedAtUnixMs,
|
|
3272
|
+
deliveryIndex: value.deliveryIndex,
|
|
3273
|
+
lastScheduledIndex: value.lastScheduledIndex,
|
|
3274
|
+
nextDueAtUnixMs: value.dueAtUnixMs,
|
|
3275
|
+
pendingTicks: value.pendingTicks
|
|
3276
|
+
})), signal);
|
|
3084
3277
|
return {
|
|
3085
3278
|
authorityKey: participant.key,
|
|
3086
3279
|
participant,
|
|
3087
3280
|
activation,
|
|
3088
3281
|
initialized: false,
|
|
3089
|
-
restoredTimers: [],
|
|
3090
|
-
replayResults: [],
|
|
3091
3282
|
replayPackets: []
|
|
3092
3283
|
};
|
|
3093
3284
|
}
|
|
@@ -3216,47 +3407,20 @@ class LocalTargetPort {
|
|
|
3216
3407
|
hidden.replayPackets.push(packet);
|
|
3217
3408
|
}
|
|
3218
3409
|
async openAdmission(hidden) {
|
|
3219
|
-
if (hidden.actor
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
return;
|
|
3224
|
-
const state = this.requireActorManager().getState(hidden.actor.context.actorId);
|
|
3225
|
-
if (state === undefined)
|
|
3226
|
-
throw new Error('Relocated Actor state is not staged.');
|
|
3227
|
-
const results = await (0, actor_handoff_1.replayActorHandoffBacklog)(hidden.replayPackets, (parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef) => {
|
|
3228
|
-
if (state.spotId === undefined) {
|
|
3229
|
-
const runtime = this.options.spotNodeRuntime();
|
|
3230
|
-
if (runtime === undefined) {
|
|
3231
|
-
throw new Error('Relocated Entry Spot Actor dispatch is unavailable.');
|
|
3232
|
-
}
|
|
3233
|
-
return runtime.dispatchEntryActorPacket('remote', hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
|
|
3234
|
-
}
|
|
3235
|
-
return this.requireSpotManager().dispatchRoutedActorPacket('remote', state.spotId, hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
|
|
3236
|
-
});
|
|
3237
|
-
hidden.replayResults.push(...results);
|
|
3410
|
+
if (hidden.actor !== undefined) {
|
|
3411
|
+
this.requireActorManager().publishRelocationActor(hidden.actor.context.actorId);
|
|
3412
|
+
this.requireSpotManager().openActorJoinRelocation(this.meshName, actorJoinAdmissionIdentity(this.envelope), hidden.actor);
|
|
3413
|
+
}
|
|
3238
3414
|
}
|
|
3239
|
-
async
|
|
3240
|
-
if (hidden.
|
|
3241
|
-
throw new Error('Actor relocation cannot restore a Spot timer.');
|
|
3242
|
-
hidden.restoredTimers.push(timer);
|
|
3243
|
-
if (hidden.restoredTimers.length !== hidden.participant.timers.length)
|
|
3415
|
+
async normalize(hidden) {
|
|
3416
|
+
if (hidden.actor === undefined || hidden.replayPackets.length === 0)
|
|
3244
3417
|
return;
|
|
3245
|
-
hidden.
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
maxCatchUpTicks: value.maxCatchUpTicks,
|
|
3251
|
-
stopOnUnhandledException: value.stopOnUnhandledException,
|
|
3252
|
-
startedAtUnixMs: value.startedAtUnixMs,
|
|
3253
|
-
deliveryIndex: value.deliveryIndex,
|
|
3254
|
-
lastScheduledIndex: value.lastScheduledIndex,
|
|
3255
|
-
nextDueAtUnixMs: value.dueAtUnixMs,
|
|
3256
|
-
pendingTicks: value.pendingTicks
|
|
3257
|
-
})));
|
|
3418
|
+
if (hidden.replayStart === undefined || hidden.onReplayResult === undefined) {
|
|
3419
|
+
throw new Error('Relocated Actor backlog has no target activation owner.');
|
|
3420
|
+
}
|
|
3421
|
+
this.options.actorTransfer.admitRelocationPrefix(hidden.actor, hidden.participant.objectGeneration, hidden.replayPackets, hidden.replayStart, hidden.onReplayResult);
|
|
3422
|
+
hidden.replayPackets.length = 0;
|
|
3258
3423
|
}
|
|
3259
|
-
async normalize(_hidden) { }
|
|
3260
3424
|
async abort(hidden) {
|
|
3261
3425
|
if (hidden.actor !== undefined) {
|
|
3262
3426
|
await this.requireActorManager().abortRelocationActor(hidden.actor.context.actorId);
|
|
@@ -3407,6 +3571,19 @@ function decodeQueuedHandoffPacket(message) {
|
|
|
3407
3571
|
...(messageFollowOrigin === undefined ? {} : { messageFollowOrigin })
|
|
3408
3572
|
}
|
|
3409
3573
|
])[0];
|
|
3574
|
+
const requestSource = decoded.source === undefined ? undefined : handoffSourceFence(decoded.source);
|
|
3575
|
+
if (messageFollowOrigin !== undefined &&
|
|
3576
|
+
requestSource !== undefined &&
|
|
3577
|
+
(0, routing_id_1.routingIdsEqual)(messageFollowOrigin.sourceNodeRid, requestSource.nodeRid)) {
|
|
3578
|
+
return {
|
|
3579
|
+
...decoded,
|
|
3580
|
+
index: index,
|
|
3581
|
+
messageFollowOrigin: {
|
|
3582
|
+
...messageFollowOrigin,
|
|
3583
|
+
requestSource
|
|
3584
|
+
}
|
|
3585
|
+
};
|
|
3586
|
+
}
|
|
3410
3587
|
return { ...decoded, index: index };
|
|
3411
3588
|
}
|
|
3412
3589
|
/**
|
|
@@ -3549,28 +3726,18 @@ function createServiceRelocationId(isInUse, entropy = node_crypto_1.randomBytes)
|
|
|
3549
3726
|
return candidate;
|
|
3550
3727
|
}
|
|
3551
3728
|
}
|
|
3552
|
-
function operationWireId(value) {
|
|
3553
|
-
const match = /^(\d+):(\d+)$/.exec(value);
|
|
3554
|
-
if (match === null)
|
|
3555
|
-
throw new Error(`Actor handoff operation '${value}' is not canonical.`);
|
|
3556
|
-
const result = { high: BigInt(match[1]), low: BigInt(match[2]) };
|
|
3557
|
-
if (result.high === 0n && result.low === 0n) {
|
|
3558
|
-
throw new Error('Actor handoff operation identity must not be zero.');
|
|
3559
|
-
}
|
|
3560
|
-
return result;
|
|
3561
|
-
}
|
|
3562
3729
|
function sameWireId(left, right) {
|
|
3563
3730
|
return left.high === right.high && left.low === right.low;
|
|
3564
3731
|
}
|
|
3565
3732
|
function sameCoordinator(left, right) {
|
|
3566
3733
|
return (left.ownerId === right.ownerId &&
|
|
3567
3734
|
BigInt(left.leaseGeneration) === right.leaseGeneration &&
|
|
3568
|
-
left.nodeRid
|
|
3735
|
+
(0, routing_id_1.routingIdsEqual)(left.nodeRid, right.nodeRid) &&
|
|
3569
3736
|
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
3570
3737
|
left.expectedAuthorityStoreVersion === right.expectedAuthorityStoreVersion);
|
|
3571
3738
|
}
|
|
3572
3739
|
function sameRelocationTarget(left, right) {
|
|
3573
|
-
return (left.nodeRid
|
|
3740
|
+
return ((0, routing_id_1.routingIdsEqual)(left.nodeRid, right.nodeRid) &&
|
|
3574
3741
|
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
3575
3742
|
left.ownerId === right.ownerId &&
|
|
3576
3743
|
BigInt(left.ownerLeaseGeneration) === right.ownerLeaseGeneration);
|
|
@@ -3604,7 +3771,7 @@ function isSourceOwner(current, expected) {
|
|
|
3604
3771
|
current.allocation.state === expected.allocation.state &&
|
|
3605
3772
|
current.allocation.objectKind === expected.allocation.objectKind &&
|
|
3606
3773
|
current.allocation.stableType === expected.allocation.stableType &&
|
|
3607
|
-
|
|
3774
|
+
(0, routing_id_1.routingIdsEqual)(current.allocation.descriptor.rid, expected.allocation.descriptor.rid) &&
|
|
3608
3775
|
current.allocation.descriptorLifecycleGeneration ===
|
|
3609
3776
|
expected.allocation.descriptorLifecycleGeneration);
|
|
3610
3777
|
}
|
|
@@ -3613,7 +3780,7 @@ function isExactCommittedTarget(current, expected, target, payload) {
|
|
|
3613
3780
|
current.authorityOwnerGeneration > expected.authorityOwnerGeneration &&
|
|
3614
3781
|
current.ownerId === target.ownerId &&
|
|
3615
3782
|
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
3616
|
-
|
|
3783
|
+
(0, routing_id_1.routingIdsEqual)(current.allocation.descriptor.rid, target.nodeRid) &&
|
|
3617
3784
|
current.allocation.descriptorLifecycleGeneration === target.nodeGeneration &&
|
|
3618
3785
|
Buffer.from(current.payload).equals(Buffer.from(payload)));
|
|
3619
3786
|
}
|
|
@@ -3636,40 +3803,52 @@ async function waitForRelocationRetry(delayMs, signal) {
|
|
|
3636
3803
|
function sameRequestSource(left, right) {
|
|
3637
3804
|
return (left.ownerId === right.ownerId &&
|
|
3638
3805
|
left.leaseGeneration === right.leaseGeneration &&
|
|
3639
|
-
left.nodeRid
|
|
3806
|
+
(0, routing_id_1.routingIdsEqual)(left.nodeRid, right.nodeRid) &&
|
|
3640
3807
|
left.nodeGeneration === right.nodeGeneration);
|
|
3641
3808
|
}
|
|
3642
3809
|
function handoffSourceFence(value) {
|
|
3643
3810
|
return {
|
|
3644
3811
|
ownerId: value.ownerId,
|
|
3645
3812
|
leaseGeneration: BigInt(value.ownerLeaseGeneration),
|
|
3646
|
-
nodeRid: value.nodeRid,
|
|
3813
|
+
nodeRid: (0, routing_id_1.decodeRoutingId)(value.nodeRid, value.nodeRidHex),
|
|
3647
3814
|
nodeGeneration: BigInt(value.nodeGeneration)
|
|
3648
3815
|
};
|
|
3649
3816
|
}
|
|
3650
|
-
function maintenanceReplyRelay(stage, coordinator, participantId, completion) {
|
|
3817
|
+
function maintenanceReplyRelay(stage, coordinator, participantId, completion, packet, serializers, flowEnabled) {
|
|
3651
3818
|
const value = completion.result.ok ? completion.result.response : undefined;
|
|
3819
|
+
const direct = packet.messageFollowOrigin?.requestSource !== undefined;
|
|
3820
|
+
const application = direct && completion.result.ok
|
|
3821
|
+
? (0, service_wire_m6a_codec_1.decodeApplicationPayloadView)((0, service_wire_m6a_codec_1.encodeMultipartApplicationPayload)((0, actor_reply_1.encodeActorReply)(Buffer.from(packet.header, 'base64'), protocol_1.ZLinkStreamMessageKind.Response, value, serializers, flowEnabled), service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_PACKET_NAME, service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_CONTENT_TYPE))
|
|
3822
|
+
: undefined;
|
|
3652
3823
|
return {
|
|
3653
3824
|
relocation: relocationWireId(stage.staging.envelope.aggregateId),
|
|
3654
3825
|
targetAttemptGeneration: stage.offer.prepare.targetAttemptGeneration,
|
|
3655
3826
|
coordinator,
|
|
3656
|
-
operation:
|
|
3827
|
+
operation: parseHandoffOperationId(completion.operationId),
|
|
3657
3828
|
replyRouteId: BigInt(completion.source.replyRouteId),
|
|
3658
3829
|
participantId,
|
|
3659
3830
|
sequence: BigInt(completion.index + 1),
|
|
3660
3831
|
terminalResult: completion.result.ok ? runtime_values_1.RequestResult.Ok : runtime_values_1.RequestResult.InternalError,
|
|
3661
3832
|
failureCode: completion.result.ok
|
|
3662
|
-
?
|
|
3663
|
-
:
|
|
3664
|
-
...(
|
|
3665
|
-
? {
|
|
3666
|
-
: {
|
|
3833
|
+
? service_wire_constants_generated_2.ServiceWireFrameworkErrorCode.none
|
|
3834
|
+
: service_wire_constants_generated_2.ServiceWireFrameworkErrorCode.requestFailed,
|
|
3835
|
+
...(application !== undefined
|
|
3836
|
+
? {
|
|
3667
3837
|
payload: {
|
|
3668
|
-
packetName:
|
|
3669
|
-
contentType:
|
|
3670
|
-
bytes:
|
|
3838
|
+
packetName: application.packetName,
|
|
3839
|
+
contentType: application.contentType,
|
|
3840
|
+
bytes: application.payload
|
|
3671
3841
|
}
|
|
3672
|
-
}
|
|
3842
|
+
}
|
|
3843
|
+
: value === undefined
|
|
3844
|
+
? {}
|
|
3845
|
+
: {
|
|
3846
|
+
payload: {
|
|
3847
|
+
packetName: SERVICE_RELOCATION_REPLY_PACKET_NAME,
|
|
3848
|
+
contentType: stream_wire_1.ZlinkStreamContentType.Json,
|
|
3849
|
+
bytes: Buffer.from(JSON.stringify(value), 'utf8')
|
|
3850
|
+
}
|
|
3851
|
+
})
|
|
3673
3852
|
};
|
|
3674
3853
|
}
|
|
3675
3854
|
function handoffResultFromRelay(relay) {
|
|
@@ -3701,6 +3880,13 @@ function wireIdText(value) {
|
|
|
3701
3880
|
class ServiceRelocationDataLostError extends Error {
|
|
3702
3881
|
}
|
|
3703
3882
|
exports.ServiceRelocationDataLostError = ServiceRelocationDataLostError;
|
|
3883
|
+
/** The local source lifecycle ended a pending relocation operation. */
|
|
3884
|
+
class ServiceRelocationRuntimeStoppedError extends Error {
|
|
3885
|
+
constructor() {
|
|
3886
|
+
super('Relocation runtime stopped.');
|
|
3887
|
+
this.name = 'AbortError';
|
|
3888
|
+
}
|
|
3889
|
+
}
|
|
3704
3890
|
/** Classifies relocation failure with the error model §2.1 mapping. */
|
|
3705
3891
|
function relocationFailedFailureCode(error) {
|
|
3706
3892
|
if (error instanceof ServiceRelocationDataLostError) {
|
|
@@ -3806,7 +3992,7 @@ function validatePrepareEnvelope(request, envelope) {
|
|
|
3806
3992
|
}
|
|
3807
3993
|
function validateTargetOneWayControl(stage, request, sourceNodeRid) {
|
|
3808
3994
|
const prepare = stage.offer.prepare;
|
|
3809
|
-
if (
|
|
3995
|
+
if (!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, stage.offer.authenticatedSourceNodeRid) ||
|
|
3810
3996
|
!sameWireId(request.relocation, prepare.relocation) ||
|
|
3811
3997
|
request.targetAttemptGeneration !== prepare.targetAttemptGeneration ||
|
|
3812
3998
|
!sameCoordinator(request.coordinator, prepare.coordinator) ||
|
|
@@ -3863,12 +4049,7 @@ function canonicalQueuedFrozenRecord(participant, message, coordinator, target)
|
|
|
3863
4049
|
const identity = (0, authority_key_codec_1.decodeAuthorityKey)({ value: participant.key }).globalId;
|
|
3864
4050
|
const source = packet.source === undefined
|
|
3865
4051
|
? coordinatorSource(coordinator)
|
|
3866
|
-
:
|
|
3867
|
-
ownerId: packet.source.ownerId,
|
|
3868
|
-
leaseGeneration: BigInt(packet.source.ownerLeaseGeneration),
|
|
3869
|
-
nodeRid: packet.source.nodeRid,
|
|
3870
|
-
nodeGeneration: BigInt(packet.source.nodeGeneration)
|
|
3871
|
-
};
|
|
4052
|
+
: handoffSourceFence(packet.source);
|
|
3872
4053
|
const operationId = parseHandoffOperationId(packet.messageFollowContext.operationId);
|
|
3873
4054
|
if (packet.returnResponse && packet.source === undefined) {
|
|
3874
4055
|
throw new Error('Captured Actor request has no exact request-source fence.');
|
|
@@ -3971,43 +4152,40 @@ function sessionRelocationPendingKey(request) {
|
|
|
3971
4152
|
return sessionRelocationIdentityKey(request);
|
|
3972
4153
|
}
|
|
3973
4154
|
function sessionRelocationRouteSubmitKey(request, targetNodeRid) {
|
|
3974
|
-
return `${sessionRelocationIdentityKey(request)}:${
|
|
4155
|
+
return `${sessionRelocationIdentityKey(request)}:${(0, routing_id_1.encodeRoutingIdStorageHex)(targetNodeRid)}`;
|
|
3975
4156
|
}
|
|
3976
4157
|
function sessionRelocationResponseKey(response) {
|
|
3977
4158
|
return sessionRelocationIdentityKey(response);
|
|
3978
4159
|
}
|
|
3979
|
-
function sessionRelocationResponseFingerprint(response) {
|
|
3980
|
-
return (0, service_stateful_wire_codec_1.encodeSessionRelocationSealed)(response).toString('base64');
|
|
3981
|
-
}
|
|
3982
4160
|
function sessionRelocationIdentityKey(value) {
|
|
3983
4161
|
return `session-relocation:${(0, service_stateful_wire_codec_1.serviceSessionRelocationIdentityKey)(value)}:sealed`;
|
|
3984
4162
|
}
|
|
3985
4163
|
function validateSessionRelocationResponse(request, response) {
|
|
3986
4164
|
const sameCoordinator = request.coordinator.ownerId === response.coordinator.ownerId &&
|
|
3987
4165
|
request.coordinator.leaseGeneration === response.coordinator.leaseGeneration &&
|
|
3988
|
-
request.coordinator.nodeRid
|
|
4166
|
+
(0, routing_id_1.routingIdsEqual)(request.coordinator.nodeRid, response.coordinator.nodeRid) &&
|
|
3989
4167
|
request.coordinator.nodeGeneration === response.coordinator.nodeGeneration &&
|
|
3990
4168
|
request.coordinator.expectedAuthorityStoreVersion ===
|
|
3991
4169
|
response.coordinator.expectedAuthorityStoreVersion;
|
|
3992
|
-
const sameSession = request.session.sessionOwnerNodeRid
|
|
4170
|
+
const sameSession = (0, routing_id_1.routingIdsEqual)(request.session.sessionOwnerNodeRid, response.session.sessionOwnerNodeRid) &&
|
|
3993
4171
|
request.session.sessionOwnerNodeGeneration === response.session.sessionOwnerNodeGeneration &&
|
|
3994
4172
|
request.session.sessionOwnerId === response.session.sessionOwnerId &&
|
|
3995
4173
|
request.session.sessionOwnerLeaseGeneration === response.session.sessionOwnerLeaseGeneration &&
|
|
3996
|
-
request.session.sessionRid
|
|
4174
|
+
(0, routing_id_1.routingIdsEqual)(request.session.sessionRid, response.session.sessionRid) &&
|
|
3997
4175
|
request.session.bindingGeneration === response.session.bindingGeneration;
|
|
3998
4176
|
if (request.relocation.high !== response.relocation.high ||
|
|
3999
4177
|
request.relocation.low !== response.relocation.low ||
|
|
4000
4178
|
!sameCoordinator ||
|
|
4001
4179
|
!sameSession) {
|
|
4002
|
-
throw new
|
|
4180
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation ACK changed its exact fence.');
|
|
4003
4181
|
}
|
|
4004
4182
|
if (request.actor.actor.actorId !== response.actor.actor.actorId ||
|
|
4005
4183
|
request.actor.actor.generation !== response.actor.actor.generation ||
|
|
4006
|
-
request.actor.actor.nodeRid
|
|
4184
|
+
!(0, routing_id_1.routingIdsEqual)(request.actor.actor.nodeRid, response.actor.actor.nodeRid) ||
|
|
4007
4185
|
request.actor.targetNodeGeneration !== response.actor.targetNodeGeneration ||
|
|
4008
4186
|
request.actor.authorityOwnerGeneration !== response.actor.authorityOwnerGeneration ||
|
|
4009
4187
|
request.actor.ownerLeaseGeneration !== response.actor.ownerLeaseGeneration) {
|
|
4010
|
-
throw new
|
|
4188
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Command 43 ACK does not echo command 42.');
|
|
4011
4189
|
}
|
|
4012
4190
|
}
|
|
4013
4191
|
function replyRelayIdentityKey(value) {
|
|
@@ -4019,13 +4197,16 @@ function replyRelayIdentityKey(value) {
|
|
|
4019
4197
|
value.relocation.low.toString(),
|
|
4020
4198
|
value.coordinator.ownerId,
|
|
4021
4199
|
value.coordinator.leaseGeneration.toString(),
|
|
4022
|
-
value.coordinator.nodeRid,
|
|
4200
|
+
(0, routing_id_1.encodeRoutingIdStorageHex)(value.coordinator.nodeRid),
|
|
4023
4201
|
value.coordinator.nodeGeneration.toString(),
|
|
4024
4202
|
value.coordinator.expectedAuthorityStoreVersion
|
|
4025
4203
|
]);
|
|
4026
4204
|
}
|
|
4027
4205
|
function replyRelayPendingKey(ackTargetNodeRid, value) {
|
|
4028
|
-
return JSON.stringify([
|
|
4206
|
+
return JSON.stringify([
|
|
4207
|
+
(0, routing_id_1.encodeRoutingIdStorageHex)(ackTargetNodeRid),
|
|
4208
|
+
replyRelayIdentityKey(value)
|
|
4209
|
+
]);
|
|
4029
4210
|
}
|
|
4030
4211
|
function encodeHandoffQueuedMessages(backlog) {
|
|
4031
4212
|
return backlog.map((packet) => ({
|
|
@@ -4151,6 +4332,7 @@ function encodeActorSession(target) {
|
|
|
4151
4332
|
version: ACTOR_SESSION_JOURNAL_VERSION,
|
|
4152
4333
|
routerChannelId: target.routerChannelId,
|
|
4153
4334
|
targetNodeRid: String(target.targetNodeRid),
|
|
4335
|
+
targetNodeRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.targetNodeRid),
|
|
4154
4336
|
spotId: String(target.spotId),
|
|
4155
4337
|
sessionNodeRid: target.sessionNodeRid === undefined ? undefined : String(target.sessionNodeRid),
|
|
4156
4338
|
sessionNodeRidHex: target.sessionNodeRid === undefined
|
|
@@ -4172,15 +4354,16 @@ function encodeActorSession(target) {
|
|
|
4172
4354
|
relocationLow: target.serviceWireRelocation.relocation.low.toString(),
|
|
4173
4355
|
coordinatorOwnerId: target.serviceWireRelocation.coordinator.ownerId,
|
|
4174
4356
|
coordinatorLeaseGeneration: target.serviceWireRelocation.coordinator.leaseGeneration.toString(),
|
|
4175
|
-
coordinatorNodeRid: target.serviceWireRelocation.coordinator.nodeRid,
|
|
4357
|
+
coordinatorNodeRid: String(target.serviceWireRelocation.coordinator.nodeRid),
|
|
4358
|
+
coordinatorNodeRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.serviceWireRelocation.coordinator.nodeRid),
|
|
4176
4359
|
coordinatorNodeGeneration: target.serviceWireRelocation.coordinator.nodeGeneration.toString(),
|
|
4177
4360
|
expectedAuthorityStoreVersion: target.serviceWireRelocation.coordinator.expectedAuthorityStoreVersion,
|
|
4178
|
-
sessionOwnerNodeRid: target.serviceWireRelocation.session.sessionOwnerNodeRid,
|
|
4361
|
+
sessionOwnerNodeRid: String(target.serviceWireRelocation.session.sessionOwnerNodeRid),
|
|
4179
4362
|
sessionOwnerNodeRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.serviceWireRelocation.session.sessionOwnerNodeRid),
|
|
4180
4363
|
sessionOwnerNodeGeneration: target.serviceWireRelocation.session.sessionOwnerNodeGeneration.toString(),
|
|
4181
4364
|
sessionOwnerId: target.serviceWireRelocation.session.sessionOwnerId,
|
|
4182
4365
|
sessionOwnerLeaseGeneration: target.serviceWireRelocation.session.sessionOwnerLeaseGeneration.toString(),
|
|
4183
|
-
sessionRid: target.serviceWireRelocation.session.sessionRid,
|
|
4366
|
+
sessionRid: String(target.serviceWireRelocation.session.sessionRid),
|
|
4184
4367
|
sessionRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.serviceWireRelocation.session.sessionRid),
|
|
4185
4368
|
bindingGeneration: target.serviceWireRelocation.session.bindingGeneration.toString()
|
|
4186
4369
|
}
|
|
@@ -4200,7 +4383,7 @@ function decodeActorSession(payload) {
|
|
|
4200
4383
|
const serviceWire = decodeActorSessionServiceWireFence(value.serviceWireRelocation);
|
|
4201
4384
|
return {
|
|
4202
4385
|
routerChannelId: value.routerChannelId,
|
|
4203
|
-
targetNodeRid: value.targetNodeRid,
|
|
4386
|
+
targetNodeRid: (0, routing_id_1.decodeRoutingId)(value.targetNodeRid, value.targetNodeRidHex),
|
|
4204
4387
|
spotId: value.spotId,
|
|
4205
4388
|
...(typeof value.sessionNodeRid === 'string'
|
|
4206
4389
|
? { sessionNodeRid: (0, routing_id_1.decodeRoutingId)(value.sessionNodeRid, value.sessionNodeRidHex) }
|
|
@@ -4269,7 +4452,7 @@ function decodeActorSessionServiceWireFence(input) {
|
|
|
4269
4452
|
coordinator: {
|
|
4270
4453
|
ownerId: text('coordinatorOwnerId'),
|
|
4271
4454
|
leaseGeneration: positive('coordinatorLeaseGeneration'),
|
|
4272
|
-
nodeRid: text('coordinatorNodeRid'),
|
|
4455
|
+
nodeRid: (0, routing_id_1.decodeRoutingId)(text('coordinatorNodeRid'), value.coordinatorNodeRidHex),
|
|
4273
4456
|
nodeGeneration: positive('coordinatorNodeGeneration'),
|
|
4274
4457
|
expectedAuthorityStoreVersion: text('expectedAuthorityStoreVersion')
|
|
4275
4458
|
},
|