@zlink-systems/framework 0.30.0 → 0.30.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +150 -68
- package/dist/.tsbuildinfo +1 -1
- package/dist/contracts/Actors/ZLinkActorClient.d.ts +2 -1
- package/dist/contracts/Channels/Calls.d.ts +2 -1
- package/dist/contracts/Common/ZLinkEncodedPayload.d.ts +6 -0
- package/dist/contracts/Common/ZLinkEncodedPayload.js +18 -3
- package/dist/contracts/Common/ZLinkMessageMetadata.js +11 -5
- package/dist/contracts/Common/encoded-payload-storage.d.ts +1 -1
- package/dist/contracts/Common/encoded-payload-storage.js +5 -2
- package/dist/contracts/Configuration/Builders.d.ts +1 -1
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.d.ts +2 -1
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.js +5 -1
- package/dist/contracts/Configuration/RegistrationBuilders.js +2 -2
- package/dist/contracts/Configuration/RegistrationTypes.d.ts +1 -0
- package/dist/contracts/Dispatch/ZLinkDispatchOptions.d.ts +2 -2
- package/dist/contracts/Handlers/Attributes.d.ts +5 -13
- package/dist/contracts/Handlers/Attributes.js +7 -11
- package/dist/contracts/Handlers/JsonContract.d.ts +2 -46
- package/dist/contracts/Handlers/JsonContract.js +4 -153
- package/dist/contracts/Locations/Rows.d.ts +2 -0
- package/dist/contracts/Spots/Contracts.d.ts +6 -1
- package/dist/contracts/Spots/ZLinkSpot.d.ts +3 -3
- package/dist/nest-integration.d.ts +1 -1
- package/dist/nest-integration.js +2 -1
- package/dist/runtime/abort.d.ts +5 -0
- package/dist/runtime/abort.js +10 -0
- package/dist/runtime/actors/actor-authority-publication.js +3 -2
- package/dist/runtime/actors/actor-client.d.ts +1 -2
- package/dist/runtime/actors/actor-client.js +52 -57
- package/dist/runtime/actors/actor-context.d.ts +2 -0
- package/dist/runtime/actors/actor-context.js +46 -9
- package/dist/runtime/actors/actor-handoff.d.ts +8 -4
- package/dist/runtime/actors/actor-handoff.js +219 -92
- package/dist/runtime/actors/actor-join-relocation.d.ts +3 -0
- package/dist/runtime/actors/actor-local-native-join.js +13 -12
- package/dist/runtime/actors/actor-mailbox.js +27 -1
- package/dist/runtime/actors/actor-remote-joiner.js +3 -1
- package/dist/runtime/actors/actor-remote-wire.js +15 -0
- package/dist/runtime/actors/actor-reply.d.ts +4 -0
- package/dist/runtime/actors/actor-reply.js +29 -0
- package/dist/runtime/actors/actor-routed-join-transport.d.ts +6 -2
- package/dist/runtime/actors/actor-routed-json-request.d.ts +2 -0
- package/dist/runtime/actors/actor-runtime-contracts.d.ts +2 -0
- package/dist/runtime/actors/actor-runtime-state.js +2 -20
- package/dist/runtime/actors/actor-source-transfer.d.ts +1 -17
- package/dist/runtime/actors/index.d.ts +8 -3
- package/dist/runtime/actors/index.js +55 -27
- package/dist/runtime/actors/session-disconnected-wire.d.ts +8 -0
- package/dist/runtime/actors/session-disconnected-wire.js +48 -0
- package/dist/runtime/actors/spot-actor-dispatch.js +2 -2
- package/dist/runtime/admission.d.ts +9 -1
- package/dist/runtime/admission.js +56 -23
- package/dist/runtime/application-jobs/application-ingress-record-owner.d.ts +6 -8
- package/dist/runtime/application-jobs/application-ingress-record-owner.js +39 -22
- package/dist/runtime/application-jobs/application-job-queue-scope.d.ts +15 -1
- package/dist/runtime/application-jobs/application-job-queue-scope.js +61 -20
- package/dist/runtime/application-jobs/contracts.d.ts +12 -1
- package/dist/runtime/backend/contracts/index.d.ts +37 -12
- package/dist/runtime/backend/mesh-actor-session-node-adapter.js +2 -2
- package/dist/runtime/backend/mesh-completion-table.d.ts +9 -1
- package/dist/runtime/backend/mesh-completion-table.js +154 -49
- package/dist/runtime/backend/mesh-dispatch-pump.d.ts +28 -1
- package/dist/runtime/backend/mesh-dispatch-pump.js +208 -101
- package/dist/runtime/backend/node/node-backend-adapter-support.d.ts +2 -2
- package/dist/runtime/backend/node/node-backend-adapter-support.js +16 -9
- package/dist/runtime/backend/node/node-dealer-channel-client-transport.d.ts +1 -1
- package/dist/runtime/backend/node/node-dealer-channel-client-transport.js +5 -2
- package/dist/runtime/backend/node/node-raw-binding-port.js +124 -72
- package/dist/runtime/backend/node/node-raw-mesh-backend.d.ts +46 -13
- package/dist/runtime/backend/node/node-raw-mesh-backend.js +181 -164
- package/dist/runtime/backend/node/node-socket-backend-adapter.d.ts +1 -1
- package/dist/runtime/backend/node/node-socket-backend-adapter.js +11 -11
- package/dist/runtime/backend/raw-binding-port.d.ts +2 -2
- package/dist/runtime/backend/runtime-message.d.ts +1 -0
- package/dist/runtime/backend/runtime-message.js +5 -0
- package/dist/runtime/backend/runtime-values.d.ts +1 -0
- package/dist/runtime/backend/runtime-values.js +2 -1
- package/dist/runtime/channels/channel-clients.d.ts +2 -0
- package/dist/runtime/channels/channel-clients.js +75 -23
- package/dist/runtime/channels/channel-dispatch-pipeline.d.ts +25 -5
- package/dist/runtime/channels/channel-dispatch-pipeline.js +111 -56
- package/dist/runtime/channels/channel-dispatch-services.d.ts +1 -0
- package/dist/runtime/channels/channel-dispatch-services.js +45 -0
- package/dist/runtime/channels/channel-dispatchers.d.ts +4 -3
- package/dist/runtime/channels/channel-dispatchers.js +62 -54
- package/dist/runtime/channels/channel-envelope-inspection.d.ts +1 -2
- package/dist/runtime/channels/channel-envelope-inspection.js +1 -16
- package/dist/runtime/channels/channel-envelope.d.ts +2 -0
- package/dist/runtime/channels/channel-envelope.js +146 -28
- package/dist/runtime/channels/channel-outbound-operations.d.ts +2 -2
- package/dist/runtime/channels/channel-outbound-operations.js +8 -8
- package/dist/runtime/channels/channel-runtime-lifecycle.d.ts +4 -3
- package/dist/runtime/channels/channel-runtime-lifecycle.js +19 -24
- package/dist/runtime/channels/channel-runtime-manager.d.ts +9 -8
- package/dist/runtime/channels/channel-runtime-manager.js +16 -16
- package/dist/runtime/channels/channel-transports.d.ts +18 -16
- package/dist/runtime/channels/channel-transports.js +77 -58
- package/dist/runtime/channels/client-server-location-runtime.js +5 -4
- package/dist/runtime/channels/dispatch-error-reporter.d.ts +2 -1
- package/dist/runtime/channels/dispatch-error-reporter.js +2 -1
- package/dist/runtime/channels/fanout-location-runtime.d.ts +1 -1
- package/dist/runtime/channels/fanout-location-runtime.js +3 -3
- package/dist/runtime/channels/local-spot-route-transport.d.ts +1 -1
- package/dist/runtime/channels/local-spot-route-transport.js +2 -1
- package/dist/runtime/channels/router-socket-spot-route-transport.d.ts +2 -2
- package/dist/runtime/channels/router-socket-spot-route-transport.js +4 -4
- package/dist/runtime/channels/source-spot-router.d.ts +2 -2
- package/dist/runtime/channels/source-spot-router.js +4 -4
- package/dist/runtime/channels/spot-node-route-transport.d.ts +2 -2
- package/dist/runtime/channels/spot-node-route-transport.js +4 -4
- package/dist/runtime/channels/spot-route-bridge-transport.d.ts +2 -2
- package/dist/runtime/channels/spot-route-bridge-transport.js +4 -4
- package/dist/runtime/channels/spot-route-dispatch-strategy.d.ts +5 -4
- package/dist/runtime/channels/spot-route-dispatch-strategy.js +18 -18
- package/dist/runtime/diagnostics/flow-context.js +19 -7
- package/dist/runtime/diagnostics/index.js +4 -9
- package/dist/runtime/diagnostics/message-flow.d.ts +8 -0
- package/dist/runtime/diagnostics/message-flow.js +35 -5
- package/dist/runtime/diagnostics/runtime-metrics.d.ts +2 -2
- package/dist/runtime/diagnostics/runtime-metrics.js +10 -8
- package/dist/runtime/execution/index.d.ts +4 -2
- package/dist/runtime/execution/index.js +33 -10
- package/dist/runtime/execution/serial-execution-queue.d.ts +5 -0
- package/dist/runtime/execution/serial-execution-queue.js +22 -2
- package/dist/runtime/execution/state-lane.d.ts +2 -0
- package/dist/runtime/execution/state-lane.js +5 -0
- package/dist/runtime/foundation/actor-join-recovery-codec.js +2 -2
- package/dist/runtime/foundation/durable-service-request.d.ts +14 -0
- package/dist/runtime/foundation/durable-service-request.js +98 -0
- package/dist/runtime/foundation/operation-identity.d.ts +1 -0
- package/dist/runtime/foundation/operation-identity.js +5 -1
- package/dist/runtime/foundation/operation-registry.d.ts +25 -1
- package/dist/runtime/foundation/operation-registry.js +86 -18
- package/dist/runtime/foundation/raw-service-mesh-runtime.d.ts +61 -8
- package/dist/runtime/foundation/raw-service-mesh-runtime.js +486 -266
- package/dist/runtime/foundation/service-liveness-registry.d.ts +12 -1
- package/dist/runtime/foundation/service-liveness-registry.js +38 -0
- package/dist/runtime/foundation/service-mailbox.d.ts +7 -0
- package/dist/runtime/foundation/service-maintenance-runtime.js +1 -6
- package/dist/runtime/foundation/service-relocation-object-owner.d.ts +3 -3
- package/dist/runtime/foundation/service-relocation-object-owner.js +48 -34
- package/dist/runtime/foundation/service-relocation-runtime.d.ts +2 -0
- package/dist/runtime/foundation/service-relocation-runtime.js +20 -4
- package/dist/runtime/foundation/service-runtime-contracts.d.ts +10 -1
- package/dist/runtime/foundation/service-session-binding-ingress-port.d.ts +2 -1
- package/dist/runtime/foundation/service-stateful-registry.d.ts +5 -4
- package/dist/runtime/foundation/service-stateful-registry.js +24 -26
- package/dist/runtime/foundation/service-stateful-runtime.d.ts +75 -9
- package/dist/runtime/foundation/service-stateful-runtime.js +486 -218
- package/dist/runtime/foundation/service-stateful-wire-codec.d.ts +14 -6
- package/dist/runtime/foundation/service-stateful-wire-codec.js +48 -27
- package/dist/runtime/foundation/service-topology-registry.d.ts +12 -2
- package/dist/runtime/foundation/service-topology-registry.js +61 -27
- package/dist/runtime/foundation/service-wire-binary-primitives.d.ts +6 -1
- package/dist/runtime/foundation/service-wire-binary-primitives.js +27 -11
- package/dist/runtime/foundation/service-wire-constants.generated.d.ts +1 -0
- package/dist/runtime/foundation/service-wire-constants.generated.js +1 -0
- package/dist/runtime/foundation/service-wire-m6a-codec.d.ts +13 -3
- package/dist/runtime/foundation/service-wire-m6a-codec.js +189 -78
- package/dist/runtime/handlers/handler-instance-scope.d.ts +8 -0
- package/dist/runtime/handlers/handler-instance-scope.js +54 -11
- package/dist/runtime/host/actor-packet-relay.d.ts +6 -4
- package/dist/runtime/host/actor-packet-relay.js +59 -72
- package/dist/runtime/host/actor-placement-coordinator.js +25 -8
- package/dist/runtime/host/actor-runtime-options-factory.js +2 -2
- package/dist/runtime/host/actor-transfer-runtime.d.ts +7 -31
- package/dist/runtime/host/actor-transfer-runtime.js +85 -331
- package/dist/runtime/host/application-job-queue.d.ts +40 -7
- package/dist/runtime/host/application-job-queue.js +137 -42
- package/dist/runtime/host/entry-actor-runtime.d.ts +1 -0
- package/dist/runtime/host/entry-actor-runtime.js +3 -0
- package/dist/runtime/host/index.d.ts +12 -2
- package/dist/runtime/host/index.js +300 -202
- package/dist/runtime/host/location-runtime-owner.js +2 -6
- package/dist/runtime/host/mesh-router-resolver.d.ts +4 -0
- package/dist/runtime/host/mesh-router-resolver.js +14 -6
- package/dist/runtime/host/remote-bound-session-relay.d.ts +2 -4
- package/dist/runtime/host/remote-bound-session-relay.js +73 -208
- package/dist/runtime/host/route-mesh-runtime.d.ts +2 -2
- package/dist/runtime/host/route-mesh-runtime.js +6 -4
- package/dist/runtime/host/runtime-shutdown.d.ts +1 -0
- package/dist/runtime/host/runtime-shutdown.js +3 -2
- package/dist/runtime/host/service-relocation-control.d.ts +1 -0
- package/dist/runtime/host/service-relocation-control.js +10 -4
- package/dist/runtime/host/service-relocation-host-runtime.d.ts +22 -14
- package/dist/runtime/host/service-relocation-host-runtime.js +754 -599
- package/dist/runtime/host/spot-address-transport.js +3 -2
- package/dist/runtime/host/spot-runtime-options-factory.js +16 -7
- package/dist/runtime/host/user-spot-creation-coordinator.js +33 -84
- package/dist/runtime/locations/actor-location-claims.d.ts +2 -2
- package/dist/runtime/locations/actor-location-claims.js +34 -19
- package/dist/runtime/locations/auto-connect-loop.d.ts +2 -0
- package/dist/runtime/locations/auto-connect-loop.js +22 -14
- package/dist/runtime/locations/auto-connect-reconciler.js +0 -6
- package/dist/runtime/locations/auto-connect-types.d.ts +0 -1
- package/dist/runtime/locations/descriptor-publication.d.ts +4 -0
- package/dist/runtime/locations/descriptor-publication.js +11 -0
- package/dist/runtime/locations/domain-store-contract.d.ts +1 -0
- package/dist/runtime/locations/in-memory-authority-store.d.ts +7 -1
- package/dist/runtime/locations/in-memory-authority-store.js +56 -19
- package/dist/runtime/locations/in-memory-location-store.js +1 -10
- package/dist/runtime/locations/index.d.ts +1 -0
- package/dist/runtime/locations/index.js +1 -0
- package/dist/runtime/locations/lease-tracker.js +1 -6
- package/dist/runtime/locations/lifecycle.d.ts +1 -2
- package/dist/runtime/locations/lifecycle.js +2 -5
- package/dist/runtime/locations/location-store-repository.d.ts +51 -1
- package/dist/runtime/locations/location-store-repository.js +363 -139
- package/dist/runtime/locations/relocation-blob.d.ts +18 -1
- package/dist/runtime/locations/relocation-blob.js +52 -1
- package/dist/runtime/locations/request-source-resolver.d.ts +37 -0
- package/dist/runtime/locations/request-source-resolver.js +80 -0
- package/dist/runtime/locations/resolvers.js +1 -1
- package/dist/runtime/locations/runtime.d.ts +7 -3
- package/dist/runtime/locations/runtime.js +61 -53
- package/dist/runtime/messaging/framework-json-v1.d.ts +4 -6
- package/dist/runtime/messaging/framework-json-v1.js +38 -294
- package/dist/runtime/messaging/packet-name.d.ts +1 -1
- package/dist/runtime/messaging/packet-name.js +8 -24
- package/dist/runtime/messaging/payload-codec.d.ts +7 -1
- package/dist/runtime/messaging/payload-codec.js +7 -6
- package/dist/runtime/messaging/request-result.d.ts +2 -0
- package/dist/runtime/messaging/request-result.js +57 -0
- package/dist/runtime/messaging/submission-result.d.ts +1 -0
- package/dist/runtime/messaging/submission-result.js +5 -2
- package/dist/runtime/protocol/service_wire_codec.generated.d.ts +34 -1
- package/dist/runtime/protocol/service_wire_codec.generated.js +229 -0
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.d.ts +16 -1
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.js +37 -8
- package/dist/runtime/spots/formal-remote-actor-admission-registry.d.ts +7 -2
- package/dist/runtime/spots/formal-remote-actor-admission-registry.js +36 -21
- package/dist/runtime/spots/index.d.ts +21 -11
- package/dist/runtime/spots/index.js +243 -178
- package/dist/runtime/spots/spot-activation-registry.d.ts +4 -2
- package/dist/runtime/spots/spot-activation-registry.js +21 -11
- package/dist/runtime/spots/spot-activation-state.d.ts +1 -1
- package/dist/runtime/spots/spot-activation-state.js +7 -7
- package/dist/runtime/spots/spot-activation.d.ts +7 -3
- package/dist/runtime/spots/spot-activation.js +140 -52
- package/dist/runtime/spots/spot-actor-admission-coordinator.d.ts +1 -0
- package/dist/runtime/spots/spot-actor-admission-coordinator.js +13 -22
- package/dist/runtime/spots/spot-actor-membership.d.ts +0 -1
- package/dist/runtime/spots/spot-actor-membership.js +0 -8
- package/dist/runtime/spots/spot-actor-packet-dispatch.js +33 -8
- package/dist/runtime/spots/spot-actor-packet-drain.js +5 -33
- package/dist/runtime/spots/spot-entry-activation.d.ts +2 -0
- package/dist/runtime/spots/spot-entry-activation.js +20 -11
- package/dist/runtime/spots/spot-handler-registry.d.ts +7 -4
- package/dist/runtime/spots/spot-handler-registry.js +24 -10
- package/dist/runtime/spots/spot-manager-internal-contracts.d.ts +2 -2
- package/dist/runtime/spots/spot-manager-public.js +2 -2
- package/dist/runtime/spots/spot-node-autoconnect.d.ts +2 -1
- package/dist/runtime/spots/spot-node-autoconnect.js +22 -42
- package/dist/runtime/spots/spot-node-runtime-manager.d.ts +19 -2
- package/dist/runtime/spots/spot-node-runtime-manager.js +115 -50
- package/dist/runtime/spots/spot-outbound.d.ts +3 -0
- package/dist/runtime/spots/spot-outbound.js +27 -18
- package/dist/runtime/spots/spot-remote-codec.js +5 -1
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.d.ts +1 -0
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.js +1 -1
- package/dist/runtime/spots/spot-runtime-ports.d.ts +1 -1
- package/dist/runtime/spots/spot-serial-executor.d.ts +10 -0
- package/dist/runtime/spots/spot-serial-executor.js +61 -8
- package/dist/runtime/spots/spot-serial-turn-executor.js +19 -2
- package/dist/runtime/spots/spot-subscription-dispatch.js +1 -4
- package/dist/runtime/spots/spot-timer.d.ts +57 -5
- package/dist/runtime/spots/spot-timer.js +288 -103
- package/dist/runtime/streams/actor-session-binding-registry.d.ts +42 -11
- package/dist/runtime/streams/actor-session-binding-registry.js +165 -48
- package/dist/runtime/streams/actor-session-binding-runtime-owner.d.ts +8 -4
- package/dist/runtime/streams/bound-actor-relay-sender.d.ts +2 -1
- package/dist/runtime/streams/bound-actor-relay-sender.js +15 -8
- package/dist/runtime/streams/bound-session-service.d.ts +1 -1
- package/dist/runtime/streams/bound-session-service.js +6 -11
- package/dist/runtime/streams/index.d.ts +3 -2
- package/dist/runtime/streams/index.js +6 -4
- package/dist/runtime/streams/managed-stream.js +3 -2
- package/dist/runtime/streams/native-fallback-bound-session.js +8 -73
- package/dist/runtime/streams/session-actor-coordinator.d.ts +2 -8
- package/dist/runtime/streams/session-actor-coordinator.js +15 -32
- package/dist/runtime/streams/stream-binding-runtime-ports.d.ts +3 -10
- package/dist/runtime/streams/stream-session-runtime.d.ts +1 -1
- package/dist/runtime/streams/stream-session-runtime.js +14 -31
- package/dist/runtime/workers/index.js +1 -1
- package/package.json +3 -3
- package/src/contracts/Actors/ZLinkActorClient.ts +2 -1
- package/src/contracts/Channels/Calls.ts +2 -1
- package/src/contracts/Common/ZLinkEncodedPayload.ts +24 -4
- package/src/contracts/Common/ZLinkMessageMetadata.ts +13 -8
- package/src/contracts/Common/encoded-payload-storage.ts +6 -3
- package/src/contracts/Configuration/Builders.ts +4 -1
- package/src/contracts/Configuration/RegistrationBuilderPolicy.ts +7 -2
- package/src/contracts/Configuration/RegistrationBuilders.ts +6 -2
- package/src/contracts/Configuration/RegistrationTypes.ts +1 -0
- package/src/contracts/Dispatch/ZLinkDispatchOptions.ts +2 -1
- package/src/contracts/Handlers/Attributes.ts +14 -21
- package/src/contracts/Handlers/JsonContract.ts +8 -216
- package/src/contracts/Locations/Rows.ts +2 -0
- package/src/contracts/Spots/Contracts.ts +10 -1
- package/src/contracts/Spots/ZLinkSpot.ts +3 -3
- package/src/nest-integration.ts +1 -0
- package/src/runtime/abort.ts +12 -0
- package/src/runtime/actors/actor-authority-publication.ts +3 -2
- package/src/runtime/actors/actor-client.ts +100 -85
- package/src/runtime/actors/actor-context.ts +49 -9
- package/src/runtime/actors/actor-handoff.ts +291 -110
- package/src/runtime/actors/actor-join-relocation.ts +3 -0
- package/src/runtime/actors/actor-local-native-join.ts +21 -26
- package/src/runtime/actors/actor-mailbox.ts +40 -2
- package/src/runtime/actors/actor-remote-joiner.ts +6 -1
- package/src/runtime/actors/actor-remote-wire.ts +17 -0
- package/src/runtime/actors/actor-reply.ts +42 -0
- package/src/runtime/actors/actor-routed-join-transport.ts +14 -4
- package/src/runtime/actors/actor-routed-json-request.ts +10 -2
- package/src/runtime/actors/actor-runtime-contracts.ts +2 -0
- package/src/runtime/actors/actor-runtime-state.ts +2 -20
- package/src/runtime/actors/actor-source-transfer.ts +1 -34
- package/src/runtime/actors/index.ts +60 -32
- package/src/runtime/actors/session-disconnected-wire.ts +66 -0
- package/src/runtime/actors/spot-actor-dispatch.ts +4 -2
- package/src/runtime/admission.ts +69 -21
- package/src/runtime/application-jobs/application-ingress-record-owner.ts +41 -25
- package/src/runtime/application-jobs/application-job-queue-scope.ts +83 -22
- package/src/runtime/application-jobs/contracts.ts +16 -1
- package/src/runtime/backend/contracts/index.ts +92 -18
- package/src/runtime/backend/mesh-actor-session-node-adapter.ts +5 -3
- package/src/runtime/backend/mesh-completion-table.ts +160 -57
- package/src/runtime/backend/mesh-dispatch-pump.ts +237 -108
- package/src/runtime/backend/node/node-backend-adapter-support.ts +17 -8
- package/src/runtime/backend/node/node-dealer-channel-client-transport.ts +12 -2
- package/src/runtime/backend/node/node-raw-binding-port.ts +148 -85
- package/src/runtime/backend/node/node-raw-mesh-backend.ts +315 -201
- package/src/runtime/backend/node/node-socket-backend-adapter.ts +26 -11
- package/src/runtime/backend/raw-binding-port.ts +7 -2
- package/src/runtime/backend/runtime-message.ts +6 -0
- package/src/runtime/backend/runtime-values.ts +1 -0
- package/src/runtime/channels/channel-clients.ts +132 -58
- package/src/runtime/channels/channel-dispatch-pipeline.ts +198 -114
- package/src/runtime/channels/channel-dispatch-services.ts +69 -1
- package/src/runtime/channels/channel-dispatchers.ts +136 -90
- package/src/runtime/channels/channel-envelope-inspection.ts +2 -16
- package/src/runtime/channels/channel-envelope.ts +225 -31
- package/src/runtime/channels/channel-outbound-operations.ts +22 -8
- package/src/runtime/channels/channel-runtime-lifecycle.ts +13 -13
- package/src/runtime/channels/channel-runtime-manager.ts +41 -17
- package/src/runtime/channels/channel-transports.ts +171 -112
- package/src/runtime/channels/client-server-location-runtime.ts +11 -10
- package/src/runtime/channels/dispatch-error-reporter.ts +4 -2
- package/src/runtime/channels/fanout-location-runtime.ts +17 -9
- package/src/runtime/channels/local-spot-route-transport.ts +3 -1
- package/src/runtime/channels/router-socket-spot-route-transport.ts +13 -4
- package/src/runtime/channels/source-spot-router.ts +8 -4
- package/src/runtime/channels/spot-node-route-transport.ts +8 -4
- package/src/runtime/channels/spot-route-bridge-transport.ts +6 -4
- package/src/runtime/channels/spot-route-dispatch-strategy.ts +47 -18
- package/src/runtime/diagnostics/flow-context.ts +20 -7
- package/src/runtime/diagnostics/index.ts +1 -7
- package/src/runtime/diagnostics/message-flow.ts +47 -9
- package/src/runtime/diagnostics/runtime-metrics.ts +12 -10
- package/src/runtime/execution/index.ts +40 -14
- package/src/runtime/execution/serial-execution-queue.ts +33 -4
- package/src/runtime/execution/state-lane.ts +7 -0
- package/src/runtime/foundation/actor-join-recovery-codec.ts +2 -2
- package/src/runtime/foundation/durable-service-request.ts +137 -0
- package/src/runtime/foundation/operation-identity.ts +5 -1
- package/src/runtime/foundation/operation-registry.ts +98 -20
- package/src/runtime/foundation/raw-service-mesh-runtime.ts +704 -336
- package/src/runtime/foundation/service-liveness-registry.ts +39 -0
- package/src/runtime/foundation/service-mailbox.ts +8 -0
- package/src/runtime/foundation/service-maintenance-runtime.ts +1 -8
- package/src/runtime/foundation/service-relocation-object-owner.ts +55 -36
- package/src/runtime/foundation/service-relocation-runtime.ts +25 -4
- package/src/runtime/foundation/service-runtime-contracts.ts +19 -2
- package/src/runtime/foundation/service-session-binding-ingress-port.ts +2 -1
- package/src/runtime/foundation/service-stateful-registry.ts +38 -35
- package/src/runtime/foundation/service-stateful-runtime.ts +732 -275
- package/src/runtime/foundation/service-stateful-wire-codec.ts +49 -16
- package/src/runtime/foundation/service-topology-registry.ts +91 -36
- package/src/runtime/foundation/service-wire-binary-primitives.ts +35 -12
- package/src/runtime/foundation/service-wire-constants.generated.ts +1 -0
- package/src/runtime/foundation/service-wire-m6a-codec.ts +231 -76
- package/src/runtime/handlers/handler-instance-scope.ts +67 -16
- package/src/runtime/host/actor-packet-relay.ts +113 -94
- package/src/runtime/host/actor-placement-coordinator.ts +31 -8
- package/src/runtime/host/actor-runtime-options-factory.ts +13 -2
- package/src/runtime/host/actor-transfer-runtime.ts +104 -430
- package/src/runtime/host/application-job-queue.ts +160 -41
- package/src/runtime/host/entry-actor-runtime.ts +4 -0
- package/src/runtime/host/index.ts +473 -271
- package/src/runtime/host/location-runtime-owner.ts +2 -6
- package/src/runtime/host/mesh-router-resolver.ts +13 -5
- package/src/runtime/host/remote-bound-session-relay.ts +88 -271
- package/src/runtime/host/route-mesh-runtime.ts +12 -6
- package/src/runtime/host/runtime-shutdown.ts +15 -4
- package/src/runtime/host/service-relocation-control.ts +11 -5
- package/src/runtime/host/service-relocation-host-runtime.ts +1092 -759
- package/src/runtime/host/spot-address-transport.ts +4 -2
- package/src/runtime/host/spot-runtime-options-factory.ts +20 -7
- package/src/runtime/host/user-spot-creation-coordinator.ts +41 -100
- package/src/runtime/locations/actor-location-claims.ts +52 -30
- package/src/runtime/locations/auto-connect-loop.ts +22 -14
- package/src/runtime/locations/auto-connect-reconciler.ts +0 -6
- package/src/runtime/locations/auto-connect-types.ts +0 -1
- package/src/runtime/locations/descriptor-publication.ts +7 -0
- package/src/runtime/locations/domain-store-contract.ts +5 -0
- package/src/runtime/locations/in-memory-authority-store.ts +85 -23
- package/src/runtime/locations/in-memory-location-store.ts +1 -12
- package/src/runtime/locations/index.ts +1 -0
- package/src/runtime/locations/lease-tracker.ts +2 -9
- package/src/runtime/locations/lifecycle.ts +8 -26
- package/src/runtime/locations/location-store-repository.ts +628 -152
- package/src/runtime/locations/relocation-blob.ts +75 -1
- package/src/runtime/locations/request-source-resolver.ts +138 -0
- package/src/runtime/locations/resolvers.ts +2 -2
- package/src/runtime/locations/runtime.ts +98 -64
- package/src/runtime/messaging/framework-json-v1.ts +5 -325
- package/src/runtime/messaging/packet-name.ts +5 -25
- package/src/runtime/messaging/payload-codec.ts +8 -11
- package/src/runtime/messaging/request-result.ts +62 -0
- package/src/runtime/messaging/submission-result.ts +5 -1
- package/src/runtime/protocol/service_wire_codec.generated.ts +218 -5
- package/src/runtime/protocol/service_wire_pilot_codec.generated.ts +16 -13
- package/src/runtime/spots/formal-remote-actor-admission-registry.ts +46 -25
- package/src/runtime/spots/index.ts +389 -283
- package/src/runtime/spots/spot-activation-registry.ts +25 -13
- package/src/runtime/spots/spot-activation-state.ts +12 -19
- package/src/runtime/spots/spot-activation.ts +198 -96
- package/src/runtime/spots/spot-actor-admission-coordinator.ts +25 -55
- package/src/runtime/spots/spot-actor-membership.ts +0 -15
- package/src/runtime/spots/spot-actor-packet-dispatch.ts +56 -11
- package/src/runtime/spots/spot-actor-packet-drain.ts +9 -52
- package/src/runtime/spots/spot-entry-activation.ts +33 -21
- package/src/runtime/spots/spot-handler-registry.ts +56 -12
- package/src/runtime/spots/spot-manager-internal-contracts.ts +2 -2
- package/src/runtime/spots/spot-manager-public.ts +2 -2
- package/src/runtime/spots/spot-node-autoconnect.ts +28 -47
- package/src/runtime/spots/spot-node-runtime-manager.ts +197 -67
- package/src/runtime/spots/spot-outbound.ts +39 -19
- package/src/runtime/spots/spot-remote-codec.ts +9 -2
- package/src/runtime/spots/spot-routed-spot-packet-dispatch.ts +2 -1
- package/src/runtime/spots/spot-runtime-ports.ts +1 -1
- package/src/runtime/spots/spot-serial-executor.ts +97 -25
- package/src/runtime/spots/spot-serial-turn-executor.ts +22 -3
- package/src/runtime/spots/spot-subscription-dispatch.ts +1 -4
- package/src/runtime/spots/spot-timer.ts +418 -158
- package/src/runtime/streams/actor-session-binding-registry.ts +236 -64
- package/src/runtime/streams/actor-session-binding-runtime-owner.ts +16 -8
- package/src/runtime/streams/bound-actor-relay-sender.ts +23 -9
- package/src/runtime/streams/bound-session-service.ts +7 -10
- package/src/runtime/streams/index.ts +12 -6
- package/src/runtime/streams/managed-stream.ts +4 -2
- package/src/runtime/streams/native-fallback-bound-session.ts +14 -103
- package/src/runtime/streams/session-actor-coordinator.ts +39 -88
- package/src/runtime/streams/stream-binding-runtime-ports.ts +3 -9
- package/src/runtime/streams/stream-session-runtime.ts +13 -34
- package/src/runtime/workers/index.ts +1 -1
|
@@ -1,3 +1,15 @@
|
|
|
1
|
+
import { encodeActorReply } from '../actors/actor-reply';
|
|
2
|
+
import { ZLinkStreamMessageKind } from '../streams/protocol';
|
|
3
|
+
import {
|
|
4
|
+
decodeApplicationPayloadView,
|
|
5
|
+
encodeMultipartApplicationPayload
|
|
6
|
+
} from '../foundation/service-wire-m6a-codec';
|
|
7
|
+
import {
|
|
8
|
+
SERVICE_FRAMEWORK_MULTIPART_PACKET_NAME,
|
|
9
|
+
SERVICE_FRAMEWORK_MULTIPART_CONTENT_TYPE
|
|
10
|
+
} from '../foundation/service-wire-constants.generated';
|
|
11
|
+
import { OperationTimeoutError } from '../foundation/operation-registry';
|
|
12
|
+
import { durableRequestCanReplay } from '../foundation/durable-service-request';
|
|
1
13
|
import { UINT64_MAX, ZlinkStreamContentType } from '@zlink-systems/stream-wire';
|
|
2
14
|
import { randomBytes } from 'node:crypto';
|
|
3
15
|
import {
|
|
@@ -18,6 +30,12 @@ import {
|
|
|
18
30
|
ZLinkSpotRelocationReadyOutcome,
|
|
19
31
|
ZLinkUserSpotExecutionMode
|
|
20
32
|
} from '../../contracts';
|
|
33
|
+
import {
|
|
34
|
+
ZLinkDispatchErrorSurface,
|
|
35
|
+
ZLinkDispatchMessageKind,
|
|
36
|
+
ZLinkRuntimeDispatchErrorAction as ZLinkDispatchErrorAction,
|
|
37
|
+
ZLinkRuntimeMessageFlowOutcome as ZLinkMessageFlowOutcome
|
|
38
|
+
} from '../../contracts/Dispatch/ZLinkDispatchOptions';
|
|
21
39
|
|
|
22
40
|
import type { ZLinkProviderResolver } from '../../contracts/Common/ZLinkProviderResolver';
|
|
23
41
|
import { zlinkRuntimeDefaultLocationOptions } from '../../contracts/Locations/Options';
|
|
@@ -29,7 +47,6 @@ import {
|
|
|
29
47
|
rewriteActorAuthorityRoute
|
|
30
48
|
} from '../actors/actor-authority-publication';
|
|
31
49
|
import {
|
|
32
|
-
replayActorHandoffBacklog,
|
|
33
50
|
type ZLinkActorHandoffPacket,
|
|
34
51
|
type ZLinkActorHandoffResult
|
|
35
52
|
} from '../actors/actor-handoff';
|
|
@@ -44,16 +61,23 @@ import {
|
|
|
44
61
|
type ZLinkRemoteBoundSessionTarget
|
|
45
62
|
} from '../actors/actor-runtime-state';
|
|
46
63
|
import type { ZLinkBackendMeshNode, ZLinkMeshCompletionTable } from '../backend';
|
|
47
|
-
import { RequestResult, SubmitResult } from '../backend/runtime-values';
|
|
64
|
+
import { RequestResult, SubmitResult, ZLinkBackendResultError } from '../backend/runtime-values';
|
|
48
65
|
import type { ZLinkFrameworkRegistration } from '../configuration';
|
|
49
66
|
import {
|
|
50
67
|
type ZLinkRuntimeEventPublisher,
|
|
68
|
+
type ZLinkRelocationMetricOperation,
|
|
51
69
|
type ZLinkRuntimeMetrics,
|
|
52
70
|
isRelocationDebugEnabled,
|
|
53
71
|
relocationDebug
|
|
54
72
|
} from '../diagnostics';
|
|
55
73
|
|
|
56
74
|
import { METRIC_NAMES, MILLISECONDS_PER_SECOND } from '../diagnostics/runtime-metrics';
|
|
75
|
+
import {
|
|
76
|
+
traceSessionBindingControlDrop,
|
|
77
|
+
flowIfEnabled,
|
|
78
|
+
type ZLinkMessageFlowTracer
|
|
79
|
+
} from '../diagnostics/message-flow';
|
|
80
|
+
import { dispatchErrorDetails } from '../diagnostics/dispatch-error-details';
|
|
57
81
|
import { isStructuralGuardEnabled, trackDiagnosticCompletion } from '../execution/state-lane';
|
|
58
82
|
import {
|
|
59
83
|
RECREATE_RELOCATION_CONTENT_TYPE,
|
|
@@ -159,8 +183,9 @@ import type {
|
|
|
159
183
|
} from '../locations/internal-location-contracts';
|
|
160
184
|
import type { ZLinkTrackedInstanceAuthority } from '../locations/spot-location-claims';
|
|
161
185
|
import { decodeRoutingId, encodeRoutingIdStorageHex, routingIdsEqual } from '../routing-id';
|
|
162
|
-
import type { DefaultZLinkSpotManager
|
|
186
|
+
import type { DefaultZLinkSpotManager } from '../spots';
|
|
163
187
|
import type { ZLinkSpotActivation } from '../spots/spot-activation-state';
|
|
188
|
+
import type { ZLinkSpotRouteTarget } from '../spots/spot-routing-internal';
|
|
164
189
|
import { createProviderInstance } from '../spots/spot-provider';
|
|
165
190
|
import { decodeHandoffBacklog } from '../spots/spot-remote-codec';
|
|
166
191
|
import { committedActorOwnerFence, type ZLinkActorTransferRuntime } from './actor-transfer-runtime';
|
|
@@ -184,13 +209,13 @@ import {
|
|
|
184
209
|
} from './relocation-state-adapter';
|
|
185
210
|
import {
|
|
186
211
|
decodeServiceRelocationControlRequest,
|
|
212
|
+
serviceRelocationControlCommand,
|
|
187
213
|
decodeServiceRelocationControlResponse,
|
|
188
214
|
encodeServiceRelocationControlRequest,
|
|
189
215
|
encodeServiceRelocationControlResponse,
|
|
190
216
|
type ZLinkServiceRelocationControlRequest,
|
|
191
217
|
type ZLinkServiceRelocationControlResponse
|
|
192
218
|
} from './service-relocation-control';
|
|
193
|
-
const RELOCATION_ATTEMPT_TIMEOUT_MS = 250;
|
|
194
219
|
|
|
195
220
|
const RELOCATION_AUTHORITY_RETRY_DELAY_MS = 25;
|
|
196
221
|
|
|
@@ -215,13 +240,18 @@ const RELOCATION_PARTICIPANT_STATE_LIMIT_BYTES = 64 * 1024 * 1024;
|
|
|
215
240
|
const RELOCATION_TARGET_TOMBSTONE_LIMIT = 1024;
|
|
216
241
|
const RELOCATION_OPERATION_RETENTION_MS = 5 * 60_000;
|
|
217
242
|
const SERVICE_CONTROL_TERMINAL_CAPACITY = 4096;
|
|
243
|
+
const INITIAL_MEMBERSHIP_EPOCH = 1n;
|
|
218
244
|
|
|
219
245
|
interface ZLinkHostRelocationOptions {
|
|
246
|
+
readonly flowCreationEnabled?: () => boolean;
|
|
247
|
+
readonly messageFlow?: () => ZLinkMessageFlowTracer | undefined;
|
|
220
248
|
readonly registration: ZLinkFrameworkRegistration;
|
|
221
249
|
readonly providerResolver?: ZLinkProviderResolver;
|
|
222
250
|
readonly locationStore: () => ZLinkLocationStore | undefined;
|
|
223
251
|
readonly currentOwner: () => ZLinkLocationOwnerToken | undefined;
|
|
224
252
|
readonly targetAdmissionSealed?: () => boolean;
|
|
253
|
+
readonly requestPeerReconcile?: (meshName: string, peerRid: string) => Promise<void>;
|
|
254
|
+
readonly sourceQuiescenceChanged?: () => void;
|
|
225
255
|
readonly liveDescriptors: (
|
|
226
256
|
meshName: string,
|
|
227
257
|
signal?: AbortSignal
|
|
@@ -230,16 +260,16 @@ interface ZLinkHostRelocationOptions {
|
|
|
230
260
|
readonly meshNode: (meshName: string) => ZLinkBackendMeshNode | undefined;
|
|
231
261
|
readonly completions: (meshName: string) => ZLinkMeshCompletionTable | undefined;
|
|
232
262
|
readonly spotManager: () => DefaultZLinkSpotManager | undefined;
|
|
233
|
-
readonly spotNodeRuntime: () => ZLinkSpotNodeRuntimeManager | undefined;
|
|
234
263
|
readonly actorManager: () => DefaultZLinkActorManager | undefined;
|
|
235
264
|
readonly actorTransfer: ZLinkActorTransferRuntime;
|
|
236
265
|
readonly boundSessionRelocation?: {
|
|
237
266
|
receiveSeal(
|
|
238
267
|
value: ServiceSessionRelocationSeal,
|
|
239
|
-
signal?: AbortSignal
|
|
240
|
-
|
|
268
|
+
signal?: AbortSignal,
|
|
269
|
+
onInstalled?: () => void
|
|
270
|
+
): Promise<ServiceSessionRelocationSealed | undefined>;
|
|
241
271
|
receiveRoute(value: ServiceSessionRelocationRoute): Promise<void>;
|
|
242
|
-
clear?(): void
|
|
272
|
+
clear?(): void | Promise<void>;
|
|
243
273
|
};
|
|
244
274
|
readonly trackInstanceSpot?: (input: ZLinkTrackedInstanceAuthority) => void;
|
|
245
275
|
readonly invalidateActorRoute?: (actorId: string) => void;
|
|
@@ -261,8 +291,11 @@ interface LocalHidden extends ServiceRelocationHiddenObject {
|
|
|
261
291
|
readonly actor?: ZLinkActor;
|
|
262
292
|
readonly activation?: ZLinkSpotActivation;
|
|
263
293
|
initialized: boolean;
|
|
264
|
-
|
|
265
|
-
|
|
294
|
+
replayStart?: Promise<void>;
|
|
295
|
+
onReplayResult?: (
|
|
296
|
+
packet: ZLinkActorHandoffPacket,
|
|
297
|
+
result: ZLinkActorHandoffResult
|
|
298
|
+
) => Promise<void>;
|
|
266
299
|
readonly replayPackets: ZLinkActorHandoffPacket[];
|
|
267
300
|
targetAuthorityOwnerGeneration?: bigint;
|
|
268
301
|
}
|
|
@@ -280,6 +313,7 @@ interface LocalStage {
|
|
|
280
313
|
finalize?: Promise<void>;
|
|
281
314
|
/** The Restore's activation admission (MeshNode §5.1); held until target commit or abort. */
|
|
282
315
|
readonly releaseActivation: () => void;
|
|
316
|
+
readonly releaseJoinAdmission: () => void;
|
|
283
317
|
}
|
|
284
318
|
|
|
285
319
|
interface TargetPayloadAssembly {
|
|
@@ -289,13 +323,19 @@ interface TargetPayloadAssembly {
|
|
|
289
323
|
readonly assembly: ZLinkRelocationPayloadAssembly;
|
|
290
324
|
}
|
|
291
325
|
|
|
326
|
+
/** One authority read decides commit and the snapshots eligible for Follow. */
|
|
327
|
+
type SourceAuthorityCommit = {
|
|
328
|
+
readonly kind: 'target';
|
|
329
|
+
readonly followAuthorities: ReadonlyMap<string, ZLinkAuthoritySnapshot>;
|
|
330
|
+
};
|
|
331
|
+
|
|
292
332
|
interface SourceCutoverWindow {
|
|
293
333
|
readonly meshName: string;
|
|
294
334
|
readonly targetNodeRid: RoutingId;
|
|
295
335
|
/** Boundary batch frames plus the cutover frame, retained until authority settlement. */
|
|
296
336
|
frames?: readonly Buffer[];
|
|
297
|
-
/**
|
|
298
|
-
|
|
337
|
+
/** The ready route used to submit the retained batch. */
|
|
338
|
+
submittedRouteGeneration?: bigint;
|
|
299
339
|
readonly cutoverSubmittedAtMs: number;
|
|
300
340
|
followTimer?: ReturnType<typeof setTimeout>;
|
|
301
341
|
windowClosed: boolean;
|
|
@@ -339,12 +379,14 @@ interface SourceActorJoinProfile {
|
|
|
339
379
|
readonly canonicalRecovery?: NonNullable<
|
|
340
380
|
Parameters<ZLinkActorJoinRelocation['relocateActorJoin']>[0]['canonicalRecovery']
|
|
341
381
|
>;
|
|
382
|
+
readonly relayReady: () => void;
|
|
342
383
|
readonly ready: Promise<void>;
|
|
343
384
|
readonly resolveReady: (failure?: ServiceRelocationPostCommitError) => void;
|
|
385
|
+
retirement?: Promise<void>;
|
|
344
386
|
}
|
|
345
387
|
|
|
346
388
|
interface PendingRelocationReplyRelay {
|
|
347
|
-
readonly ackTargetNodeRid:
|
|
389
|
+
readonly ackTargetNodeRid: RoutingId;
|
|
348
390
|
readonly identityKey: string;
|
|
349
391
|
readonly request: ServiceMaintenanceReplyRelay;
|
|
350
392
|
readonly expectedRequestSource: ServiceWireRequestSourceFence;
|
|
@@ -366,9 +408,8 @@ interface PendingSessionRelocation {
|
|
|
366
408
|
|
|
367
409
|
interface TerminalSessionRelocationControl {
|
|
368
410
|
readonly targetNodeRid: string;
|
|
369
|
-
readonly request: ServiceSessionRelocationSeal;
|
|
370
411
|
readonly requestFingerprint: string;
|
|
371
|
-
readonly
|
|
412
|
+
readonly response: ServiceSessionRelocationSealed;
|
|
372
413
|
}
|
|
373
414
|
|
|
374
415
|
/** Production host bridge from Retire inventory to remote RouteMesh owners. */
|
|
@@ -377,7 +418,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
377
418
|
private readonly terminalTargets = new BoundedReplayMap<string, string>(
|
|
378
419
|
RELOCATION_TARGET_TOMBSTONE_LIMIT
|
|
379
420
|
);
|
|
380
|
-
private readonly relocationAuthorityKeys = new Map<string, string>();
|
|
381
421
|
private readonly targetPrepareOperations = new Map<string, TargetPrepareOperation>();
|
|
382
422
|
private readonly pendingReplyRelays = new Map<string, PendingRelocationReplyRelay>();
|
|
383
423
|
private readonly pendingSessionRelocations = new Map<string, PendingSessionRelocation>();
|
|
@@ -428,6 +468,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
428
468
|
* checksum integrity failure — it must not be reported as DataLost(35).
|
|
429
469
|
*/
|
|
430
470
|
private disposed = false;
|
|
471
|
+
private readonly targetPreparationStop = new AbortController();
|
|
431
472
|
|
|
432
473
|
constructor(private readonly options: ZLinkHostRelocationOptions) {}
|
|
433
474
|
|
|
@@ -463,6 +504,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
463
504
|
else reject(failure.cause);
|
|
464
505
|
};
|
|
465
506
|
});
|
|
507
|
+
let acceptRelay!: () => void;
|
|
508
|
+
const relayReady = new Promise<void>((resolve) => {
|
|
509
|
+
acceptRelay = resolve;
|
|
510
|
+
});
|
|
466
511
|
this.sourceActorJoinProfiles.set(input.relocationId, {
|
|
467
512
|
actor: input.actor,
|
|
468
513
|
state: input.state,
|
|
@@ -477,42 +522,53 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
477
522
|
: RECREATE_RELOCATION_CONTENT_TYPE,
|
|
478
523
|
completionOperationId: input.completionOperationId,
|
|
479
524
|
canonicalRecovery: input.canonicalRecovery,
|
|
525
|
+
relayReady: acceptRelay,
|
|
480
526
|
ready,
|
|
481
527
|
resolveReady
|
|
482
528
|
});
|
|
483
529
|
const membershipEpoch =
|
|
484
530
|
input.state.spotMembershipEpoch > 0n ? input.state.spotMembershipEpoch : 1n;
|
|
485
531
|
const spotGeneration = input.target.targetSpotGeneration ?? target.lifecycleGeneration;
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
532
|
+
const cleanup = (async () => {
|
|
533
|
+
try {
|
|
534
|
+
await this.relocateStandaloneActor(
|
|
535
|
+
input.meshName,
|
|
536
|
+
input.state,
|
|
537
|
+
target,
|
|
538
|
+
target.applicationVersion,
|
|
539
|
+
input.signal,
|
|
540
|
+
{
|
|
541
|
+
...input.target,
|
|
542
|
+
spotKind:
|
|
543
|
+
input.target.spotKind === ZLinkSpotKind.Entry
|
|
544
|
+
? ZLinkSpotKind.Entry
|
|
545
|
+
: ZLinkSpotKind.User,
|
|
546
|
+
spotObjectGeneration: spotGeneration
|
|
547
|
+
},
|
|
548
|
+
input.relocationId,
|
|
549
|
+
true,
|
|
550
|
+
input.advertisedReceiveChunkLimitBytes,
|
|
551
|
+
input.deadlineAtMs
|
|
552
|
+
);
|
|
553
|
+
resolveReady();
|
|
554
|
+
} catch (error) {
|
|
555
|
+
if (error instanceof ServiceRelocationPostCommitError) {
|
|
556
|
+
resolveReady(error);
|
|
557
|
+
this.completeActorJoinSourceCleanup(input.state.actorId);
|
|
558
|
+
} else {
|
|
559
|
+
this.sourceActorJoinProfiles.delete(input.relocationId);
|
|
560
|
+
resolveReady();
|
|
561
|
+
throw error;
|
|
562
|
+
}
|
|
513
563
|
}
|
|
514
|
-
}
|
|
564
|
+
})();
|
|
565
|
+
const settledCleanup = Promise.allSettled([cleanup]);
|
|
566
|
+
await Promise.race([relayReady, cleanup]);
|
|
515
567
|
return {
|
|
568
|
+
sourceCleanup: async () => {
|
|
569
|
+
const [outcome] = await settledCleanup;
|
|
570
|
+
if (outcome.status === 'rejected') throw outcome.reason;
|
|
571
|
+
},
|
|
516
572
|
actorRef: {
|
|
517
573
|
actorId: sourceRef.actorId,
|
|
518
574
|
generation: sourceRef.generation,
|
|
@@ -542,9 +598,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
542
598
|
for (const window of this.sourceCutoverWindows.values()) {
|
|
543
599
|
if (window.followTimer !== undefined) clearTimeout(window.followTimer);
|
|
544
600
|
}
|
|
601
|
+
const hadSourceWindows = this.sourceCutoverWindows.size !== 0;
|
|
545
602
|
this.sourceCutoverWindows.clear();
|
|
546
|
-
|
|
547
|
-
stopped
|
|
603
|
+
if (hadSourceWindows) this.options.sourceQuiescenceChanged?.();
|
|
604
|
+
const stopped = new ServiceRelocationRuntimeStoppedError();
|
|
548
605
|
this.targetPrepareOperations.clear();
|
|
549
606
|
for (const pending of this.pendingReplyRelays.values()) {
|
|
550
607
|
if (pending.timer !== undefined) clearTimeout(pending.timer);
|
|
@@ -559,16 +616,27 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
559
616
|
this.pendingSessionRelocations.clear();
|
|
560
617
|
this.terminalSessionRelocations.clear();
|
|
561
618
|
this.failedSessionRouteSubmits.clear();
|
|
562
|
-
this.
|
|
619
|
+
await this.drainActorJoinSources();
|
|
563
620
|
this.terminalActorJoinSourceLeaves.clear();
|
|
564
|
-
this.options.boundSessionRelocation?.clear?.();
|
|
565
|
-
this.relocationAuthorityKeys.clear();
|
|
621
|
+
await this.options.boundSessionRelocation?.clear?.();
|
|
566
622
|
if (errors.length === 1) throw errors[0];
|
|
567
623
|
if (errors.length > 1) throw new AggregateError(errors, 'Relocation runtime stop failed.');
|
|
568
624
|
}
|
|
569
625
|
|
|
570
626
|
/** Host shutdown waits for the existing attempt terminal before releasing its lease. */
|
|
571
627
|
async drainTargetAttempts(): Promise<void> {
|
|
628
|
+
this.targetPreparationStop.abort(
|
|
629
|
+
new ZLinkFrameworkException(
|
|
630
|
+
ZLinkFrameworkErrorKind.ShuttingDown,
|
|
631
|
+
'Relocation target is shutting down.'
|
|
632
|
+
)
|
|
633
|
+
);
|
|
634
|
+
for (const pending of this.targetAssemblies.values()) {
|
|
635
|
+
pending.assembly.fail('Relocation target is shutting down.');
|
|
636
|
+
}
|
|
637
|
+
await Promise.allSettled(
|
|
638
|
+
[...this.targetPrepareOperations.values()].map((value) => value.promise)
|
|
639
|
+
);
|
|
572
640
|
const results = await Promise.allSettled(
|
|
573
641
|
[...this.targetStages.entries()].map(([stagingId, stage]) => {
|
|
574
642
|
if (stage.fallback !== undefined) clearTimeout(stage.fallback);
|
|
@@ -596,27 +664,58 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
596
664
|
|
|
597
665
|
/** Spec 28 §4.4: the target owner alone orders shutdown seal and verified cutover. */
|
|
598
666
|
private targetStageSealed(stage: LocalStage): boolean {
|
|
599
|
-
return (
|
|
600
|
-
|
|
601
|
-
|
|
667
|
+
return !stage.cutoverReceived && this.targetAdmissionSealed();
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
/** The Shutdown admission seal: closed once the target owner starts shutdown. */
|
|
671
|
+
private targetAdmissionSealed(): boolean {
|
|
672
|
+
return this.disposed || this.options.targetAdmissionSealed?.() === true;
|
|
602
673
|
}
|
|
603
674
|
|
|
604
675
|
completeActorJoinSourceCleanup(actorId: string): void {
|
|
605
676
|
for (const [relocationId, profile] of this.sourceActorJoinProfiles) {
|
|
606
677
|
if (profile.state.actorId !== actorId) continue;
|
|
607
|
-
this.
|
|
608
|
-
this.terminalActorJoinSourceLeaves.remember(relocationId, actorId);
|
|
609
|
-
void this.finalizeActorJoinSource(profile);
|
|
678
|
+
void this.retireActorJoinSource(relocationId, profile);
|
|
610
679
|
}
|
|
611
680
|
}
|
|
612
681
|
|
|
613
|
-
|
|
682
|
+
/** Drains the exact source scopes before their Spot lifecycle owners stop. */
|
|
683
|
+
async drainActorJoinSources(meshName?: string): Promise<void> {
|
|
684
|
+
await Promise.all(
|
|
685
|
+
[...this.sourceActorJoinProfiles].map(([relocationId, profile]) =>
|
|
686
|
+
meshName === undefined || profile.state.meshName === meshName
|
|
687
|
+
? this.retireActorJoinSource(relocationId, profile)
|
|
688
|
+
: undefined
|
|
689
|
+
)
|
|
690
|
+
);
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
private retireActorJoinSource(
|
|
694
|
+
relocationId: string,
|
|
695
|
+
profile: SourceActorJoinProfile
|
|
696
|
+
): Promise<void> {
|
|
697
|
+
return (profile.retirement ??= this.finalizeActorJoinSource(profile, relocationId).finally(
|
|
698
|
+
() => {
|
|
699
|
+
if (this.sourceActorJoinProfiles.get(relocationId) === profile) {
|
|
700
|
+
this.sourceActorJoinProfiles.delete(relocationId);
|
|
701
|
+
this.terminalActorJoinSourceLeaves.remember(relocationId, profile.state.actorId);
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
));
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
private async finalizeActorJoinSource(
|
|
708
|
+
profile: SourceActorJoinProfile,
|
|
709
|
+
relocationId: string
|
|
710
|
+
): Promise<void> {
|
|
614
711
|
const failures: unknown[] = [];
|
|
615
712
|
try {
|
|
616
713
|
await profile.ready;
|
|
617
714
|
} catch (error) {
|
|
618
715
|
failures.push(error);
|
|
619
716
|
}
|
|
717
|
+
// A pre-commit failure removed this attempt and restored the source.
|
|
718
|
+
if (this.sourceActorJoinProfiles.get(relocationId) !== profile) return;
|
|
620
719
|
try {
|
|
621
720
|
await this.options.actorTransfer.completeRelocationSourceLeave(
|
|
622
721
|
profile.actor,
|
|
@@ -628,7 +727,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
628
727
|
try {
|
|
629
728
|
await this.options
|
|
630
729
|
.actorManager()
|
|
631
|
-
?.
|
|
730
|
+
?.completeCoreRelocationSource(profile.state.actorId, profile.state);
|
|
632
731
|
} catch (error) {
|
|
633
732
|
failures.push(error);
|
|
634
733
|
}
|
|
@@ -692,6 +791,11 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
692
791
|
id: `actor:${state.actorId}`,
|
|
693
792
|
relocate: async (relocationSignal) => {
|
|
694
793
|
await relocateShell(relocationSignal);
|
|
794
|
+
const spotAuthority = await requireAuthority(
|
|
795
|
+
this.requireLocationStore(),
|
|
796
|
+
encodeAuthorityKey('user_spot', String(activation.spotId)),
|
|
797
|
+
relocationSignal
|
|
798
|
+
);
|
|
695
799
|
await this.relocateStandaloneActor(
|
|
696
800
|
meshName,
|
|
697
801
|
state,
|
|
@@ -699,15 +803,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
699
803
|
targetApplicationVersion,
|
|
700
804
|
relocationSignal,
|
|
701
805
|
{
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
).objectGeneration
|
|
806
|
+
...committedSpotRoute(
|
|
807
|
+
meshName,
|
|
808
|
+
activation.spotId,
|
|
809
|
+
ZLinkSpotKind.User,
|
|
810
|
+
spotAuthority.allocation.descriptor.rid,
|
|
811
|
+
spotAuthority
|
|
812
|
+
),
|
|
813
|
+
spotObjectGeneration: spotAuthority.objectGeneration
|
|
711
814
|
}
|
|
712
815
|
);
|
|
713
816
|
}
|
|
@@ -806,14 +909,31 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
806
909
|
throw failure;
|
|
807
910
|
}
|
|
808
911
|
|
|
809
|
-
|
|
912
|
+
tryHandleControl(
|
|
810
913
|
meshName: string,
|
|
811
914
|
record: ReceiveRecord,
|
|
812
915
|
signal?: AbortSignal
|
|
813
|
-
): Promise<boolean> {
|
|
916
|
+
): boolean | Promise<boolean> {
|
|
814
917
|
if (record.parts.length === 0) return false;
|
|
815
918
|
const payload = record.parts[0]!.data();
|
|
816
|
-
|
|
919
|
+
const command = serviceRelocationControlCommand(payload);
|
|
920
|
+
if (command === undefined) {
|
|
921
|
+
if (record.kind !== ReceiveKind.NodeSend || record.parts.length !== 1) return false;
|
|
922
|
+
const sourceLeave = decodeRemoteActorSourceLeaveTerminal(payload);
|
|
923
|
+
if (sourceLeave === undefined) return false;
|
|
924
|
+
return this.acceptActorJoinSourceLeave(sourceLeave);
|
|
925
|
+
}
|
|
926
|
+
return this.handleControl(meshName, record, payload, command, signal);
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
private async handleControl(
|
|
930
|
+
meshName: string,
|
|
931
|
+
record: ReceiveRecord,
|
|
932
|
+
payload: Buffer,
|
|
933
|
+
command: number,
|
|
934
|
+
signal?: AbortSignal
|
|
935
|
+
): Promise<boolean> {
|
|
936
|
+
if (command === M6bServiceWireCommand.replyRelay) {
|
|
817
937
|
await this.handleReplyRelay(
|
|
818
938
|
meshName,
|
|
819
939
|
decodeMaintenanceReplyRelay(record.parts.map((part) => part.data())),
|
|
@@ -822,45 +942,75 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
822
942
|
);
|
|
823
943
|
return true;
|
|
824
944
|
}
|
|
825
|
-
|
|
826
|
-
record.parts.length === 1
|
|
827
|
-
? undefined
|
|
828
|
-
: decodePrepareSideband(record.parts.slice(1).map((part) => part.data()));
|
|
829
|
-
if (record.parts.length > 1 && sideband === undefined) return false;
|
|
945
|
+
if (record.parts.length !== 1) return false;
|
|
830
946
|
if (record.kind === ReceiveKind.NodeSend) {
|
|
831
947
|
const sourceLeave = decodeRemoteActorSourceLeaveTerminal(payload);
|
|
832
948
|
if (sourceLeave !== undefined) {
|
|
833
949
|
return this.acceptActorJoinSourceLeave(sourceLeave);
|
|
834
950
|
}
|
|
835
951
|
}
|
|
836
|
-
if (
|
|
952
|
+
if (command === M6bServiceWireCommand.sessionRelocationSealed) {
|
|
837
953
|
this.acceptSessionRelocationResponse(
|
|
838
954
|
decodeSessionRelocationSealed(payload),
|
|
839
955
|
record.sourceNodeRid
|
|
840
956
|
);
|
|
841
957
|
return true;
|
|
842
958
|
}
|
|
843
|
-
if (
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
959
|
+
if (command === M6bServiceWireCommand.sessionRelocationSeal) {
|
|
960
|
+
let request: ServiceSessionRelocationSeal;
|
|
961
|
+
try {
|
|
962
|
+
request = decodeSessionRelocationSeal(payload);
|
|
963
|
+
} catch (error) {
|
|
964
|
+
const outcome = ZLinkMessageFlowOutcome.Error;
|
|
965
|
+
flowIfEnabled(this.options.messageFlow?.(), outcome)?.trace({
|
|
966
|
+
outcome,
|
|
967
|
+
surface: ZLinkDispatchErrorSurface.StreamSession,
|
|
968
|
+
messageKind: ZLinkDispatchMessageKind.Control,
|
|
969
|
+
packetName: 'sessionRelocationSeal',
|
|
970
|
+
sourceRid: record.sourceNodeRid ?? undefined,
|
|
971
|
+
errorReason: 'invalid_frame',
|
|
972
|
+
errorAction: ZLinkDispatchErrorAction.Drop,
|
|
973
|
+
...dispatchErrorDetails(error)
|
|
974
|
+
});
|
|
975
|
+
throw error;
|
|
976
|
+
}
|
|
977
|
+
let response: ServiceSessionRelocationSealed | undefined;
|
|
978
|
+
try {
|
|
979
|
+
response = await this.handleSessionRelocationSeal(
|
|
980
|
+
meshName,
|
|
981
|
+
request,
|
|
982
|
+
record.sourceNodeRid,
|
|
983
|
+
signal
|
|
984
|
+
);
|
|
985
|
+
} catch (error) {
|
|
986
|
+
traceSessionBindingControlDrop(
|
|
987
|
+
this.options.messageFlow?.(),
|
|
988
|
+
record.sourceNodeRid ?? '',
|
|
989
|
+
request.actor.actor.actorId,
|
|
990
|
+
'sessionRelocationSeal',
|
|
991
|
+
error instanceof ZLinkFrameworkException &&
|
|
992
|
+
error.kind === ZLinkFrameworkErrorKind.Unavailable
|
|
993
|
+
? 'location_unavailable'
|
|
994
|
+
: 'stale_target',
|
|
995
|
+
error
|
|
996
|
+
);
|
|
997
|
+
throw error;
|
|
998
|
+
}
|
|
999
|
+
if (response !== undefined) {
|
|
1000
|
+
await this.sendSessionRelocationResponse(
|
|
1001
|
+
meshName,
|
|
1002
|
+
record.sourceNodeRid,
|
|
1003
|
+
encodeSessionRelocationSealed(response)
|
|
1004
|
+
);
|
|
1005
|
+
}
|
|
856
1006
|
return true;
|
|
857
1007
|
}
|
|
858
|
-
if (
|
|
1008
|
+
if (command === M6bServiceWireCommand.sessionRelocationRoute) {
|
|
859
1009
|
const request = decodeSessionRelocationRoute(payload);
|
|
860
1010
|
await this.handleSessionRelocationRoute(meshName, request, record.sourceNodeRid, signal);
|
|
861
1011
|
return true;
|
|
862
1012
|
}
|
|
863
|
-
if (
|
|
1013
|
+
if (command === M6bServiceWireCommand.replyRelayAck) {
|
|
864
1014
|
await this.acceptReplyRelayAck(
|
|
865
1015
|
meshName,
|
|
866
1016
|
decodeMaintenanceReplyRelayAck(payload),
|
|
@@ -871,10 +1021,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
871
1021
|
}
|
|
872
1022
|
let request = decodeServiceRelocationControlRequest(payload);
|
|
873
1023
|
if (request === undefined) return false;
|
|
874
|
-
if (sideband !== undefined) {
|
|
875
|
-
if (request.kind !== 'prepare') return false;
|
|
876
|
-
request = { ...request, nodeInternalBoundSessions: sideband };
|
|
877
|
-
}
|
|
878
1024
|
// Ready(30) and Failed(53) are replies to Prepare(40), never independent
|
|
879
1025
|
// inbound controls. The source consumes them on Core's request reply leg.
|
|
880
1026
|
if (request.kind === 'ready' || request.kind === 'failed') return false;
|
|
@@ -901,7 +1047,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
901
1047
|
this.targetAssemblies.get(operationKey)?.fingerprint ??
|
|
902
1048
|
this.targetStages.get(operationKey)?.offer.prepareFingerprint ??
|
|
903
1049
|
this.terminalTargets.get(operationKey);
|
|
904
|
-
if (request.sourceNodeRid
|
|
1050
|
+
if (!routingIdsEqual(request.sourceNodeRid, record.sourceNodeRid)) return true;
|
|
905
1051
|
if (acceptedFingerprint !== undefined && acceptedFingerprint !== fingerprint) {
|
|
906
1052
|
if (!samePrepareIdentity(acceptedFingerprint, request)) return true;
|
|
907
1053
|
this.replyPrepare(
|
|
@@ -931,7 +1077,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
931
1077
|
// finish the restore and the Ready reply asynchronously.
|
|
932
1078
|
let promise: Promise<ServiceMaintenanceRelocationReady>;
|
|
933
1079
|
try {
|
|
934
|
-
this.registerTargetAssembly(operationKey, request,
|
|
1080
|
+
this.registerTargetAssembly(operationKey, request, record.sourceNodeRid);
|
|
935
1081
|
promise = this.handlePrepareControl(
|
|
936
1082
|
meshName,
|
|
937
1083
|
operationKey,
|
|
@@ -1066,7 +1212,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1066
1212
|
if (
|
|
1067
1213
|
pending === undefined ||
|
|
1068
1214
|
sourceNodeRid === null ||
|
|
1069
|
-
|
|
1215
|
+
!routingIdsEqual(sourceNodeRid, pending.authenticatedSourceNodeRid) ||
|
|
1070
1216
|
request.senderRole !== 'source' ||
|
|
1071
1217
|
!sameCoordinator(request.coordinator, pending.prepare.coordinator) ||
|
|
1072
1218
|
!sameRelocationObject(request.object, pending.prepare.object)
|
|
@@ -1145,7 +1291,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1145
1291
|
// to itself; RouteMesh has no self connection (NotConnected), so the
|
|
1146
1292
|
// control dispatches through the same inbound handler locally.
|
|
1147
1293
|
const localRid = this.requireMeshNode(meshName).status().routingId;
|
|
1148
|
-
if (
|
|
1294
|
+
if (routingIdsEqual(localRid, targetNodeRid)) {
|
|
1149
1295
|
await this.handleSessionRelocationRoute(meshName, request, localRid, signal);
|
|
1150
1296
|
return;
|
|
1151
1297
|
}
|
|
@@ -1174,7 +1320,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1174
1320
|
const key = sessionRelocationPendingKey(request);
|
|
1175
1321
|
const bytes = encodeSessionRelocationSeal(request);
|
|
1176
1322
|
const requestFingerprint = bytes.toString('base64');
|
|
1177
|
-
const expectedTarget =
|
|
1323
|
+
const expectedTarget = encodeRoutingIdStorageHex(targetNodeRid);
|
|
1178
1324
|
const terminal = this.terminalSessionRelocations.get(key);
|
|
1179
1325
|
if (terminal !== undefined) {
|
|
1180
1326
|
if (
|
|
@@ -1188,6 +1334,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1188
1334
|
);
|
|
1189
1335
|
}
|
|
1190
1336
|
this.terminalSessionRelocations.touch(key);
|
|
1337
|
+
return Promise.resolve(terminal.response);
|
|
1191
1338
|
}
|
|
1192
1339
|
const existing = this.pendingSessionRelocations.get(key);
|
|
1193
1340
|
if (existing !== undefined) {
|
|
@@ -1251,7 +1398,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1251
1398
|
// connection (NotConnected); dispatch through the inbound handler
|
|
1252
1399
|
// locally instead of the wire.
|
|
1253
1400
|
const localRid = this.requireMeshNode(meshName).status().routingId;
|
|
1254
|
-
if (
|
|
1401
|
+
if (encodeRoutingIdStorageHex(localRid) === expectedTarget) {
|
|
1255
1402
|
try {
|
|
1256
1403
|
const response = await this.handleSessionRelocationSeal(
|
|
1257
1404
|
meshName,
|
|
@@ -1259,7 +1406,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1259
1406
|
localRid,
|
|
1260
1407
|
signal
|
|
1261
1408
|
);
|
|
1262
|
-
pending.resolve(response);
|
|
1409
|
+
if (response !== undefined) pending.resolve(response);
|
|
1263
1410
|
} catch (error) {
|
|
1264
1411
|
pending.reject(error);
|
|
1265
1412
|
}
|
|
@@ -1310,9 +1457,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1310
1457
|
finish(() => {
|
|
1311
1458
|
this.terminalSessionRelocations.remember(key, {
|
|
1312
1459
|
targetNodeRid: expectedTarget,
|
|
1313
|
-
request,
|
|
1314
1460
|
requestFingerprint,
|
|
1315
|
-
|
|
1461
|
+
response
|
|
1316
1462
|
});
|
|
1317
1463
|
resolvePromise(response);
|
|
1318
1464
|
}),
|
|
@@ -1358,19 +1504,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1358
1504
|
if (pending === undefined) {
|
|
1359
1505
|
const terminal = this.terminalSessionRelocations.get(key);
|
|
1360
1506
|
if (terminal === undefined) return;
|
|
1361
|
-
|
|
1362
|
-
throw new ServiceWireProtocolError(`Session relocation ACK '${key}' source node changed.`);
|
|
1363
|
-
}
|
|
1364
|
-
validateSessionRelocationResponse(terminal.request, response);
|
|
1365
|
-
if (sessionRelocationResponseFingerprint(response) !== terminal.responseFingerprint) {
|
|
1366
|
-
throw new ServiceWireProtocolError(
|
|
1367
|
-
`Session relocation ACK '${key}' repeated with different bytes.`
|
|
1368
|
-
);
|
|
1369
|
-
}
|
|
1507
|
+
// Session–Actor binding §8.3: the first command 43 is final for this participant.
|
|
1370
1508
|
this.terminalSessionRelocations.touch(key);
|
|
1371
1509
|
return;
|
|
1372
1510
|
}
|
|
1373
|
-
if (
|
|
1511
|
+
if (
|
|
1512
|
+
sourceNodeRid === null ||
|
|
1513
|
+
encodeRoutingIdStorageHex(sourceNodeRid) !== pending.targetNodeRid
|
|
1514
|
+
) {
|
|
1374
1515
|
pending.reject(
|
|
1375
1516
|
new ServiceWireProtocolError(`Session relocation ACK '${key}' source node changed.`)
|
|
1376
1517
|
);
|
|
@@ -1394,12 +1535,12 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1394
1535
|
if (sourceNodeRid === null) return undefined;
|
|
1395
1536
|
const node = this.requireMeshNode(meshName);
|
|
1396
1537
|
const status = node.status();
|
|
1397
|
-
if (
|
|
1538
|
+
if (routingIdsEqual(status.routingId, sourceNodeRid)) {
|
|
1398
1539
|
return status.lifecycleGeneration;
|
|
1399
1540
|
}
|
|
1400
1541
|
return node
|
|
1401
1542
|
.peers()
|
|
1402
|
-
.find((peer) => peer.routingId !== null &&
|
|
1543
|
+
.find((peer) => peer.routingId !== null && routingIdsEqual(peer.routingId, sourceNodeRid))
|
|
1403
1544
|
?.lifecycleGeneration;
|
|
1404
1545
|
}
|
|
1405
1546
|
|
|
@@ -1408,11 +1549,11 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1408
1549
|
request: ServiceSessionRelocationSeal,
|
|
1409
1550
|
sourceNodeRid: RoutingId | null,
|
|
1410
1551
|
signal?: AbortSignal
|
|
1411
|
-
): Promise<ServiceSessionRelocationSealed> {
|
|
1552
|
+
): Promise<ServiceSessionRelocationSealed | undefined> {
|
|
1412
1553
|
if (
|
|
1413
1554
|
sourceNodeRid === null ||
|
|
1414
|
-
|
|
1415
|
-
|
|
1555
|
+
!routingIdsEqual(sourceNodeRid, request.coordinator.nodeRid) ||
|
|
1556
|
+
!routingIdsEqual(sourceNodeRid, request.actor.actor.nodeRid)
|
|
1416
1557
|
) {
|
|
1417
1558
|
throw new ServiceWireProtocolError(
|
|
1418
1559
|
'Session relocation seal source does not match its coordinator and Actor fence.'
|
|
@@ -1432,7 +1573,19 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1432
1573
|
if (handler === undefined) {
|
|
1433
1574
|
throw new Error('Session relocation seal ingress is not configured.');
|
|
1434
1575
|
}
|
|
1435
|
-
return await handler(request, signal)
|
|
1576
|
+
return await handler(request, signal, () => {
|
|
1577
|
+
const node = this.requireMeshNode(meshName);
|
|
1578
|
+
if (
|
|
1579
|
+
routingIdsEqual(node.status().routingId, request.targetNodeRid) ||
|
|
1580
|
+
node.isPeerRouteReady?.(request.targetNodeRid, request.targetNodeGeneration) === true
|
|
1581
|
+
)
|
|
1582
|
+
return;
|
|
1583
|
+
void this.options
|
|
1584
|
+
.requestPeerReconcile?.(meshName, request.targetNodeRid)
|
|
1585
|
+
.catch((error) =>
|
|
1586
|
+
console.error('[zlink.runtime.relocation.session_route_reconcile_failed]', error)
|
|
1587
|
+
);
|
|
1588
|
+
});
|
|
1436
1589
|
}
|
|
1437
1590
|
|
|
1438
1591
|
private async handleSessionRelocationRoute(
|
|
@@ -1444,7 +1597,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1444
1597
|
this.validateSessionOwnerFence(meshName, request.session);
|
|
1445
1598
|
const expectedSource =
|
|
1446
1599
|
request.route.action === 'commit' ? request.route.targetNodeRid : request.coordinator.nodeRid;
|
|
1447
|
-
if (sourceNodeRid === null ||
|
|
1600
|
+
if (sourceNodeRid === null || !routingIdsEqual(sourceNodeRid, expectedSource)) {
|
|
1448
1601
|
throw new ServiceWireProtocolError(
|
|
1449
1602
|
'Session relocation route source does not match its action fence.'
|
|
1450
1603
|
);
|
|
@@ -1472,10 +1625,17 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1472
1625
|
): void {
|
|
1473
1626
|
const status = this.requireMeshNode(meshName).status();
|
|
1474
1627
|
const owner = this.options.currentOwner();
|
|
1628
|
+
// Session-Actor binding §4: without this host's own descriptor the command is Unavailable.
|
|
1629
|
+
if (owner === undefined) {
|
|
1630
|
+
throw createInternalFrameworkException(
|
|
1631
|
+
ZLinkFrameworkInternalErrorKind.RouteNotConnected,
|
|
1632
|
+
'Session owner has no live MeshNode descriptor.'
|
|
1633
|
+
);
|
|
1634
|
+
}
|
|
1475
1635
|
if (
|
|
1476
|
-
|
|
1636
|
+
!routingIdsEqual(status.routingId, session.sessionOwnerNodeRid) ||
|
|
1477
1637
|
status.lifecycleGeneration !== session.sessionOwnerNodeGeneration ||
|
|
1478
|
-
owner
|
|
1638
|
+
owner.ownerId !== session.sessionOwnerId ||
|
|
1479
1639
|
owner.leaseGeneration !== session.sessionOwnerLeaseGeneration
|
|
1480
1640
|
) {
|
|
1481
1641
|
throw new ServiceWireProtocolError(
|
|
@@ -1552,7 +1712,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1552
1712
|
);
|
|
1553
1713
|
target ??= await this.selectTarget(
|
|
1554
1714
|
meshName,
|
|
1555
|
-
|
|
1715
|
+
this.requireMeshNode(meshName).status().routingId,
|
|
1556
1716
|
targetApplicationVersion,
|
|
1557
1717
|
relocationTargetRequirements([
|
|
1558
1718
|
{
|
|
@@ -1584,17 +1744,9 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1584
1744
|
objectGeneration: spotAuthority.objectGeneration,
|
|
1585
1745
|
authorityOwnerGeneration: spotAuthority.authorityOwnerGeneration,
|
|
1586
1746
|
seal: async (captureSignal) => {
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
() => {
|
|
1591
|
-
interruptionStartedAt = performance.now();
|
|
1592
|
-
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
1593
|
-
// Both seals run in the same event-loop turn after the Close decision.
|
|
1594
|
-
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
1595
|
-
}
|
|
1596
|
-
);
|
|
1597
|
-
const preparedSessions = Promise.all(
|
|
1747
|
+
// Relocation 04 §4.1: every bound member's Session is sealed before
|
|
1748
|
+
// the Spot capture stops member dispatch.
|
|
1749
|
+
const preparations = await Promise.allSettled(
|
|
1598
1750
|
actorStates.map(async (state) => ({
|
|
1599
1751
|
state,
|
|
1600
1752
|
actor: state.actor!,
|
|
@@ -1603,27 +1755,40 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1603
1755
|
state,
|
|
1604
1756
|
captureSignal,
|
|
1605
1757
|
false,
|
|
1606
|
-
relocationWireId(aggregateId)
|
|
1758
|
+
relocationWireId(aggregateId),
|
|
1759
|
+
target
|
|
1607
1760
|
)
|
|
1608
1761
|
}))
|
|
1609
1762
|
);
|
|
1610
|
-
const
|
|
1611
|
-
|
|
1612
|
-
|
|
1763
|
+
const failures: unknown[] = [];
|
|
1764
|
+
for (const preparation of preparations) {
|
|
1765
|
+
if (preparation.status === 'fulfilled') sessions.push(preparation.value);
|
|
1766
|
+
else failures.push(preparation.reason);
|
|
1767
|
+
}
|
|
1768
|
+
if (failures.length === 1) throw failures[0];
|
|
1769
|
+
if (failures.length > 1) {
|
|
1770
|
+
throw new AggregateError(failures, 'Spot relocation source preparation failed.');
|
|
1771
|
+
}
|
|
1772
|
+
const { spotCaptureOperation } = await this.afterSpotCloseDecision(
|
|
1773
|
+
meshName,
|
|
1774
|
+
activation.spotId,
|
|
1775
|
+
() => {
|
|
1776
|
+
interruptionStartedAt = performance.now();
|
|
1777
|
+
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
1778
|
+
// Both Spot seals run in the same event-loop turn after the Close decision.
|
|
1779
|
+
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
1780
|
+
}
|
|
1613
1781
|
);
|
|
1614
1782
|
spotCapture = await spotCaptureOperation;
|
|
1615
|
-
const preparation = await preparedSessionsOutcome;
|
|
1616
|
-
if ('error' in preparation) throw preparation.error;
|
|
1617
|
-
sessions.push(...preparation.value);
|
|
1618
1783
|
return {
|
|
1619
1784
|
boundSessionState: Buffer.alloc(0),
|
|
1620
1785
|
queuedMessages: [],
|
|
1621
|
-
timers: spotCapture
|
|
1786
|
+
timers: spotCapture!.timers.map(toServiceTimer)
|
|
1622
1787
|
};
|
|
1623
1788
|
},
|
|
1624
1789
|
captureApplicationState: (captureSignal) =>
|
|
1625
1790
|
this.captureApplication(spotRegistration.relocation, activation.spot, captureSignal),
|
|
1626
|
-
commitSeal: async () => {
|
|
1791
|
+
commitSeal: async (followAuthorities) => {
|
|
1627
1792
|
const failures: unknown[] = [];
|
|
1628
1793
|
try {
|
|
1629
1794
|
if (!spotSealCommitted) {
|
|
@@ -1636,9 +1801,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1636
1801
|
await this.commitSpotMessageFollow(
|
|
1637
1802
|
meshName,
|
|
1638
1803
|
spotMessageFollowSeal,
|
|
1639
|
-
spotKey,
|
|
1640
|
-
activation
|
|
1641
|
-
target
|
|
1804
|
+
followAuthorities.get(spotKey.value),
|
|
1805
|
+
activation
|
|
1642
1806
|
);
|
|
1643
1807
|
}
|
|
1644
1808
|
} catch (error) {
|
|
@@ -1646,10 +1810,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1646
1810
|
}
|
|
1647
1811
|
for (const session of sessions) {
|
|
1648
1812
|
try {
|
|
1649
|
-
const committedAuthority =
|
|
1650
|
-
|
|
1651
|
-
encodeAuthorityKey('actor', session.state.actorId)
|
|
1813
|
+
const committedAuthority = followAuthorities.get(
|
|
1814
|
+
encodeAuthorityKey('actor', session.state.actorId).value
|
|
1652
1815
|
);
|
|
1816
|
+
const committedSpot = followAuthorities.get(spotKey.value);
|
|
1653
1817
|
const targetActorRef = {
|
|
1654
1818
|
actorId: session.state.actorId,
|
|
1655
1819
|
objectGeneration: actorAuthorities.get(session.state.actorId)!.objectGeneration,
|
|
@@ -1657,15 +1821,21 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1657
1821
|
nodeRid: target.rid
|
|
1658
1822
|
};
|
|
1659
1823
|
await session.prepared.commit(
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1824
|
+
committedSpotRoute(
|
|
1825
|
+
meshName,
|
|
1826
|
+
activation.spotId,
|
|
1827
|
+
kind === 'user_spot' ? ZLinkSpotKind.User : ZLinkSpotKind.Instance,
|
|
1828
|
+
target.rid,
|
|
1829
|
+
committedSpot
|
|
1830
|
+
),
|
|
1667
1831
|
targetActorRef,
|
|
1668
|
-
|
|
1832
|
+
committedAuthority === undefined || committedSpot === undefined
|
|
1833
|
+
? null
|
|
1834
|
+
: committedActorOwnerFence(
|
|
1835
|
+
session.state.actorId,
|
|
1836
|
+
targetActorRef,
|
|
1837
|
+
committedAuthority
|
|
1838
|
+
)
|
|
1669
1839
|
);
|
|
1670
1840
|
} catch (error) {
|
|
1671
1841
|
failures.push(error);
|
|
@@ -1694,6 +1864,16 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1694
1864
|
abortSeal: async () => {
|
|
1695
1865
|
const failures: unknown[] = [];
|
|
1696
1866
|
if (spotCapture !== undefined) {
|
|
1867
|
+
// Spot model §5.1: Continued completes while the source seal still holds jobs.
|
|
1868
|
+
if (readinessBoundaryConsumed) {
|
|
1869
|
+
try {
|
|
1870
|
+
await activation.completeConsumedRelocationBoundary(
|
|
1871
|
+
ZLinkSpotRelocationReadyOutcome.Continued
|
|
1872
|
+
);
|
|
1873
|
+
} catch (error) {
|
|
1874
|
+
failures.push(error);
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1697
1877
|
try {
|
|
1698
1878
|
activation.abortRelocation(spotCapture);
|
|
1699
1879
|
} catch (error) {
|
|
@@ -1742,7 +1922,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1742
1922
|
spotObjectGeneration: spotAuthority.objectGeneration,
|
|
1743
1923
|
membershipEpoch: state.spotMembershipEpoch > 0n ? state.spotMembershipEpoch : 1n
|
|
1744
1924
|
}));
|
|
1745
|
-
|
|
1925
|
+
const relocationMetric =
|
|
1926
|
+
spotRegistration.relocation.kind === 'disabled'
|
|
1927
|
+
? undefined
|
|
1928
|
+
: this.options.metrics?.startRelocation(meshName, kind, spotRegistration.relocation.kind);
|
|
1746
1929
|
try {
|
|
1747
1930
|
const waitForRelocationBoundary = (
|
|
1748
1931
|
activation as unknown as {
|
|
@@ -1791,10 +1974,12 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1791
1974
|
)
|
|
1792
1975
|
]),
|
|
1793
1976
|
signal,
|
|
1794
|
-
sessions
|
|
1977
|
+
sessions,
|
|
1978
|
+
undefined,
|
|
1979
|
+
undefined,
|
|
1980
|
+
relocationMetric
|
|
1795
1981
|
);
|
|
1796
1982
|
} catch (error) {
|
|
1797
|
-
outcome = 'failed';
|
|
1798
1983
|
if (
|
|
1799
1984
|
readinessBoundaryConsumed &&
|
|
1800
1985
|
!(error instanceof ServiceRelocationPostCommitError) &&
|
|
@@ -1806,14 +1991,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1806
1991
|
}
|
|
1807
1992
|
throw error;
|
|
1808
1993
|
} finally {
|
|
1994
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1809
1995
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1810
1996
|
if (interruptionStartedAt !== undefined) {
|
|
1811
1997
|
this.recordRelocationInterruption(
|
|
1812
1998
|
meshName,
|
|
1813
1999
|
kind,
|
|
1814
2000
|
kind === 'user_spot' ? 'spot_wide' : undefined,
|
|
1815
|
-
interruptionStartedAt
|
|
1816
|
-
outcome
|
|
2001
|
+
interruptionStartedAt
|
|
1817
2002
|
);
|
|
1818
2003
|
}
|
|
1819
2004
|
}
|
|
@@ -1825,15 +2010,15 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1825
2010
|
target: ZLinkMeshNodeDescriptor | undefined,
|
|
1826
2011
|
targetApplicationVersion: bigint | undefined,
|
|
1827
2012
|
signal?: AbortSignal,
|
|
1828
|
-
targetMembership?: {
|
|
1829
|
-
readonly spotId: RoutingId;
|
|
2013
|
+
targetMembership?: ZLinkSpotRouteTarget & {
|
|
1830
2014
|
readonly spotKind: ZLinkSpotKind.Entry | ZLinkSpotKind.User;
|
|
1831
2015
|
readonly spotObjectGeneration: bigint;
|
|
1832
2016
|
},
|
|
1833
2017
|
aggregateIdOverride?: string,
|
|
1834
2018
|
retainSourceForActorJoin = false,
|
|
1835
2019
|
/** Target's advertised relocation state chunk cap (Actor Join only; spec 15 §4.2). */
|
|
1836
|
-
advertisedReceiveChunkLimitBytes?: number
|
|
2020
|
+
advertisedReceiveChunkLimitBytes?: number,
|
|
2021
|
+
deadlineAtMs?: number
|
|
1837
2022
|
): Promise<void> {
|
|
1838
2023
|
if (isRelocationDebugEnabled()) {
|
|
1839
2024
|
relocationDebug('standalone_actor.begin', { actorId: state.actorId });
|
|
@@ -1843,18 +2028,19 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1843
2028
|
encodeAuthorityKey('actor', state.actorId),
|
|
1844
2029
|
signal
|
|
1845
2030
|
);
|
|
2031
|
+
const actorPolicy = this.actorRegistration(
|
|
2032
|
+
state.meshName ?? meshName,
|
|
2033
|
+
authority.allocation.stableType
|
|
2034
|
+
).relocation.kind;
|
|
1846
2035
|
target ??= await this.selectTarget(
|
|
1847
2036
|
meshName,
|
|
1848
|
-
|
|
2037
|
+
this.requireMeshNode(meshName).status().routingId,
|
|
1849
2038
|
targetApplicationVersion,
|
|
1850
2039
|
[
|
|
1851
2040
|
{
|
|
1852
2041
|
objectKind: 'actor',
|
|
1853
2042
|
stableType: authority.allocation.stableType,
|
|
1854
|
-
policy:
|
|
1855
|
-
state.meshName ?? meshName,
|
|
1856
|
-
authority.allocation.stableType
|
|
1857
|
-
).relocation.kind,
|
|
2043
|
+
policy: actorPolicy,
|
|
1858
2044
|
count: 1
|
|
1859
2045
|
}
|
|
1860
2046
|
],
|
|
@@ -1871,10 +2057,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1871
2057
|
true,
|
|
1872
2058
|
target,
|
|
1873
2059
|
meshName,
|
|
1874
|
-
|
|
1875
|
-
spotId: membershipSpotId,
|
|
1876
|
-
spotKind: targetMembership?.spotKind ?? ZLinkSpotKind.Entry
|
|
1877
|
-
},
|
|
2060
|
+
targetMembership,
|
|
1878
2061
|
relocationWireId(aggregateId),
|
|
1879
2062
|
retainSourceForActorJoin
|
|
1880
2063
|
);
|
|
@@ -1892,7 +2075,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1892
2075
|
return unit.seal(captureSignal);
|
|
1893
2076
|
}
|
|
1894
2077
|
};
|
|
1895
|
-
|
|
2078
|
+
const relocationMetric =
|
|
2079
|
+
actorPolicy === 'disabled'
|
|
2080
|
+
? undefined
|
|
2081
|
+
: this.options.metrics?.startRelocation(meshName, 'actor', actorPolicy);
|
|
1896
2082
|
try {
|
|
1897
2083
|
if (isRelocationDebugEnabled()) {
|
|
1898
2084
|
relocationDebug('standalone_actor.wait_budget', {
|
|
@@ -1922,21 +2108,15 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1922
2108
|
new Map([[measuredUnit.authorityKey, authority]]),
|
|
1923
2109
|
signal,
|
|
1924
2110
|
sessions,
|
|
1925
|
-
advertisedReceiveChunkLimitBytes
|
|
2111
|
+
advertisedReceiveChunkLimitBytes,
|
|
2112
|
+
deadlineAtMs,
|
|
2113
|
+
relocationMetric
|
|
1926
2114
|
);
|
|
1927
|
-
} catch (error) {
|
|
1928
|
-
outcome = 'failed';
|
|
1929
|
-
throw error;
|
|
1930
2115
|
} finally {
|
|
2116
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1931
2117
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1932
2118
|
if (interruptionStartedAt !== undefined) {
|
|
1933
|
-
this.recordRelocationInterruption(
|
|
1934
|
-
meshName,
|
|
1935
|
-
'actor',
|
|
1936
|
-
undefined,
|
|
1937
|
-
interruptionStartedAt,
|
|
1938
|
-
outcome
|
|
1939
|
-
);
|
|
2119
|
+
this.recordRelocationInterruption(meshName, 'actor', undefined, interruptionStartedAt);
|
|
1940
2120
|
}
|
|
1941
2121
|
}
|
|
1942
2122
|
}
|
|
@@ -1975,7 +2155,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
1975
2155
|
);
|
|
1976
2156
|
target ??= await this.selectTarget(
|
|
1977
2157
|
meshName,
|
|
1978
|
-
|
|
2158
|
+
this.requireMeshNode(meshName).status().routingId,
|
|
1979
2159
|
targetApplicationVersion,
|
|
1980
2160
|
[
|
|
1981
2161
|
{
|
|
@@ -2017,7 +2197,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2017
2197
|
// PerActor User Spots are stateless shells. Application state and the
|
|
2018
2198
|
// Spot relocation adapter are intentionally not part of this unit.
|
|
2019
2199
|
captureApplicationState: async () => Buffer.alloc(0),
|
|
2020
|
-
commitSeal: async () => {
|
|
2200
|
+
commitSeal: async (followAuthorities) => {
|
|
2021
2201
|
if (!sealCommitted) {
|
|
2022
2202
|
if (spotCapture === undefined || !(await activation.commitRelocation(spotCapture))) {
|
|
2023
2203
|
throw new Error(
|
|
@@ -2030,9 +2210,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2030
2210
|
await this.commitSpotMessageFollow(
|
|
2031
2211
|
meshName,
|
|
2032
2212
|
spotMessageFollowSeal,
|
|
2033
|
-
spotKey,
|
|
2034
|
-
activation
|
|
2035
|
-
target
|
|
2213
|
+
followAuthorities.get(spotKey.value),
|
|
2214
|
+
activation
|
|
2036
2215
|
);
|
|
2037
2216
|
}
|
|
2038
2217
|
await this.requireSpotManager().completeRelocationSource(activation);
|
|
@@ -2049,7 +2228,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2049
2228
|
}
|
|
2050
2229
|
};
|
|
2051
2230
|
const aggregateId = this.reserveRelocationId();
|
|
2052
|
-
|
|
2231
|
+
const relocationMetric =
|
|
2232
|
+
spotRegistration.relocation.kind === 'disabled'
|
|
2233
|
+
? undefined
|
|
2234
|
+
: this.options.metrics?.startRelocation(
|
|
2235
|
+
meshName,
|
|
2236
|
+
'user_spot',
|
|
2237
|
+
spotRegistration.relocation.kind
|
|
2238
|
+
);
|
|
2053
2239
|
try {
|
|
2054
2240
|
await this.waitForRelocationBudgetHeadroom(String(target.rid), signal);
|
|
2055
2241
|
const captured = await new ServiceRelocationObjectCaptureOwner().captureUserSpotAggregate(
|
|
@@ -2067,20 +2253,20 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2067
2253
|
captured,
|
|
2068
2254
|
new Map([[spotKey.value, spotAuthority]]),
|
|
2069
2255
|
signal,
|
|
2070
|
-
[]
|
|
2256
|
+
[],
|
|
2257
|
+
undefined,
|
|
2258
|
+
undefined,
|
|
2259
|
+
relocationMetric
|
|
2071
2260
|
);
|
|
2072
|
-
} catch (error) {
|
|
2073
|
-
outcome = 'failed';
|
|
2074
|
-
throw error;
|
|
2075
2261
|
} finally {
|
|
2262
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
2076
2263
|
this.sourceRelocationIds.delete(aggregateId);
|
|
2077
2264
|
if (interruptionStartedAt !== undefined) {
|
|
2078
2265
|
this.recordRelocationInterruption(
|
|
2079
2266
|
meshName,
|
|
2080
2267
|
'user_spot',
|
|
2081
2268
|
'per_actor',
|
|
2082
|
-
interruptionStartedAt
|
|
2083
|
-
outcome
|
|
2269
|
+
interruptionStartedAt
|
|
2084
2270
|
);
|
|
2085
2271
|
}
|
|
2086
2272
|
}
|
|
@@ -2093,10 +2279,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2093
2279
|
standalone = false,
|
|
2094
2280
|
target?: ZLinkMeshNodeDescriptor,
|
|
2095
2281
|
meshName?: string,
|
|
2096
|
-
targetMembership?:
|
|
2097
|
-
readonly spotId: RoutingId;
|
|
2098
|
-
readonly spotKind: ZLinkSpotKind.Entry | ZLinkSpotKind.User;
|
|
2099
|
-
},
|
|
2282
|
+
targetMembership?: ZLinkSpotRouteTarget,
|
|
2100
2283
|
sessionRelocation?: ServiceWireOperationId,
|
|
2101
2284
|
retainSourceForActorJoin = false
|
|
2102
2285
|
): ServiceRelocationCaptureUnit {
|
|
@@ -2121,24 +2304,48 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2121
2304
|
state,
|
|
2122
2305
|
signal,
|
|
2123
2306
|
true,
|
|
2124
|
-
sessionRelocation
|
|
2307
|
+
sessionRelocation,
|
|
2308
|
+
target
|
|
2125
2309
|
)
|
|
2126
2310
|
};
|
|
2127
2311
|
sessions.push(ownSession);
|
|
2128
2312
|
}
|
|
2129
2313
|
const prepared = standalone ? ownSession : sessions.find((value) => value.state === state);
|
|
2130
2314
|
return {
|
|
2131
|
-
boundSessionState:
|
|
2315
|
+
boundSessionState: Buffer.alloc(0),
|
|
2316
|
+
...(prepared?.prepared.target?.serviceWireRelocation === undefined
|
|
2317
|
+
? {}
|
|
2318
|
+
: {
|
|
2319
|
+
sessionBinding: {
|
|
2320
|
+
participantId: 1n,
|
|
2321
|
+
sessionOwnerNodeRid: Buffer.from(
|
|
2322
|
+
encodeRoutingIdStorageHex(prepared.prepared.target.sessionNodeRid!),
|
|
2323
|
+
'hex'
|
|
2324
|
+
),
|
|
2325
|
+
sessionOwnerNodeGeneration:
|
|
2326
|
+
prepared.prepared.target.serviceWireRelocation.session
|
|
2327
|
+
.sessionOwnerNodeGeneration,
|
|
2328
|
+
sessionOwnerId:
|
|
2329
|
+
prepared.prepared.target.serviceWireRelocation.session.sessionOwnerId,
|
|
2330
|
+
sessionOwnerLeaseGeneration:
|
|
2331
|
+
prepared.prepared.target.serviceWireRelocation.session
|
|
2332
|
+
.sessionOwnerLeaseGeneration,
|
|
2333
|
+
sessionRid: Buffer.from(
|
|
2334
|
+
encodeRoutingIdStorageHex(prepared.prepared.target.sessionRid!),
|
|
2335
|
+
'hex'
|
|
2336
|
+
),
|
|
2337
|
+
bindingGeneration:
|
|
2338
|
+
prepared.prepared.target.serviceWireRelocation.session.bindingGeneration
|
|
2339
|
+
}
|
|
2340
|
+
}),
|
|
2132
2341
|
queuedMessages: encodeHandoffQueuedMessages(prepared?.prepared.handoffBacklog ?? []),
|
|
2133
2342
|
timers: []
|
|
2134
2343
|
};
|
|
2135
2344
|
},
|
|
2136
|
-
// The
|
|
2137
|
-
// state. The sealed bound-session target remains in sealed work and is
|
|
2138
|
-
// transported only by Prepare's NODE-INTERNAL sideband frames.
|
|
2345
|
+
// The sealed Session identity is part of the canonical participant vector.
|
|
2139
2346
|
captureApplicationState: (signal) =>
|
|
2140
2347
|
this.captureApplication(registration.relocation, state.actor!, signal),
|
|
2141
|
-
commitSeal: async () => {
|
|
2348
|
+
commitSeal: async (followAuthorities) => {
|
|
2142
2349
|
if (
|
|
2143
2350
|
!standalone ||
|
|
2144
2351
|
ownSession === undefined ||
|
|
@@ -2148,11 +2355,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2148
2355
|
return;
|
|
2149
2356
|
const failures: unknown[] = [];
|
|
2150
2357
|
try {
|
|
2151
|
-
const
|
|
2152
|
-
|
|
2153
|
-
const committedAuthority = await requireAuthority(
|
|
2154
|
-
this.requireLocationStore(),
|
|
2155
|
-
encodeAuthorityKey('actor', state.actorId)
|
|
2358
|
+
const committedAuthority = followAuthorities.get(
|
|
2359
|
+
encodeAuthorityKey('actor', state.actorId).value
|
|
2156
2360
|
);
|
|
2157
2361
|
const targetActorRef = {
|
|
2158
2362
|
actorId: state.actorId,
|
|
@@ -2161,25 +2365,28 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2161
2365
|
nodeRid: target.rid
|
|
2162
2366
|
};
|
|
2163
2367
|
await ownSession.prepared.commit(
|
|
2164
|
-
{
|
|
2368
|
+
targetMembership ?? {
|
|
2165
2369
|
routerChannelId: meshName,
|
|
2166
2370
|
targetNodeRid: target.rid,
|
|
2167
|
-
spotId:
|
|
2168
|
-
spotKind:
|
|
2169
|
-
|
|
2170
|
-
} as never,
|
|
2371
|
+
spotId: (target.entrySpotId ?? String(target.rid)) as RoutingId,
|
|
2372
|
+
spotKind: ZLinkSpotKind.Entry
|
|
2373
|
+
},
|
|
2171
2374
|
targetActorRef,
|
|
2172
|
-
|
|
2375
|
+
committedAuthority === undefined
|
|
2376
|
+
? null
|
|
2377
|
+
: committedActorOwnerFence(state.actorId, targetActorRef, committedAuthority)
|
|
2173
2378
|
);
|
|
2174
2379
|
} catch (error) {
|
|
2175
2380
|
failures.push(error);
|
|
2176
2381
|
}
|
|
2177
|
-
|
|
2178
|
-
|
|
2382
|
+
try {
|
|
2383
|
+
if (retainSourceForActorJoin) {
|
|
2384
|
+
this.requireActorManager().detachRelocationSource(state);
|
|
2385
|
+
} else {
|
|
2179
2386
|
await this.requireActorManager().completeRelocationSource(state.actorId);
|
|
2180
|
-
} catch (error) {
|
|
2181
|
-
failures.push(error);
|
|
2182
2387
|
}
|
|
2388
|
+
} catch (error) {
|
|
2389
|
+
failures.push(error);
|
|
2183
2390
|
}
|
|
2184
2391
|
if (failures.length > 0)
|
|
2185
2392
|
throw failures.length === 1
|
|
@@ -2229,7 +2436,9 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2229
2436
|
signal?: AbortSignal,
|
|
2230
2437
|
sessions: readonly SourceActorSession[] = [],
|
|
2231
2438
|
/** Target's advertised relocation state chunk cap (Actor Join only; spec 15 §4.2). */
|
|
2232
|
-
advertisedReceiveChunkLimitBytes?: number
|
|
2439
|
+
advertisedReceiveChunkLimitBytes?: number,
|
|
2440
|
+
controlDeadlineAtMs = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS,
|
|
2441
|
+
relocationMetric?: ZLinkRelocationMetricOperation
|
|
2233
2442
|
): Promise<void> {
|
|
2234
2443
|
if (isRelocationDebugEnabled()) {
|
|
2235
2444
|
relocationDebug('coordinator.begin', {
|
|
@@ -2238,19 +2447,15 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2238
2447
|
});
|
|
2239
2448
|
}
|
|
2240
2449
|
const localStatus = this.requireMeshNode(meshName).status();
|
|
2241
|
-
const controlDeadlineAtMs =
|
|
2242
|
-
signal === undefined
|
|
2243
|
-
? performance.now() + RELOCATION_OPERATION_TIMEOUT_MS
|
|
2244
|
-
: Number.MAX_SAFE_INTEGER;
|
|
2245
2450
|
const coordinator = {
|
|
2246
2451
|
ownerId: primary.ownerId,
|
|
2247
2452
|
leaseGeneration: primary.ownerLeaseGeneration,
|
|
2248
|
-
nodeRid:
|
|
2453
|
+
nodeRid: localStatus.routingId,
|
|
2249
2454
|
nodeGeneration: localStatus.lifecycleGeneration,
|
|
2250
2455
|
expectedAuthorityStoreVersion: primary.storeVersion.value
|
|
2251
2456
|
} satisfies ServiceWireRelocationCoordinatorFence;
|
|
2252
2457
|
const targetFence = {
|
|
2253
|
-
nodeRid:
|
|
2458
|
+
nodeRid: target.rid,
|
|
2254
2459
|
nodeGeneration: target.lifecycleGeneration,
|
|
2255
2460
|
ownerId: target.ownerId,
|
|
2256
2461
|
ownerLeaseGeneration: target.leaseGeneration
|
|
@@ -2263,13 +2468,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2263
2468
|
// attempt even when it reuses the same routing id.
|
|
2264
2469
|
const targetAttemptGeneration = target.lifecycleGeneration;
|
|
2265
2470
|
const relayAuthorityId = wireIdText(relocationWireId(captured.envelope.aggregateId));
|
|
2266
|
-
const relayAuthorityKey = primaryKey(captured.envelope);
|
|
2267
|
-
if (this.relocationAuthorityKeys.has(relayAuthorityId)) {
|
|
2268
|
-
throw new Error(
|
|
2269
|
-
`Relocation reply relay authority '${relayAuthorityId}' is already registered.`
|
|
2270
|
-
);
|
|
2271
|
-
}
|
|
2272
|
-
this.relocationAuthorityKeys.set(relayAuthorityId, relayAuthorityKey);
|
|
2273
2471
|
// The captured payload stays in source memory: this copy is the only
|
|
2274
2472
|
// handoff original until the cutover submit terminal and the
|
|
2275
2473
|
// retransmission window end (spec 28 §4.2).
|
|
@@ -2289,6 +2487,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2289
2487
|
primary
|
|
2290
2488
|
);
|
|
2291
2489
|
const encoded = encodeServiceRelocationEnvelope(transportEnvelope, target.applicationVersion);
|
|
2490
|
+
relocationMetric?.recordBytes(encoded.byteLength);
|
|
2292
2491
|
const limits = this.relocationLimits();
|
|
2293
2492
|
// Actor Join threads the target's admission-time advertised cap in;
|
|
2294
2493
|
// other relocation paths have no such negotiation and fall back to the
|
|
@@ -2305,7 +2504,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2305
2504
|
const restoreDeadlineReached = () =>
|
|
2306
2505
|
signal?.aborted === true || performance.now() >= controlDeadlineAtMs;
|
|
2307
2506
|
let readyReceived = false;
|
|
2308
|
-
let settlement:
|
|
2507
|
+
let settlement:
|
|
2508
|
+
Awaited<ReturnType<ZLinkHostServiceRelocationRuntime['settleSourceAuthority']>> | undefined;
|
|
2309
2509
|
try {
|
|
2310
2510
|
const prepare = {
|
|
2311
2511
|
kind: 'prepare',
|
|
@@ -2320,35 +2520,40 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2320
2520
|
payloadTotalLength: BigInt(plan.totalLength),
|
|
2321
2521
|
payloadChunkCount: plan.chunkCount,
|
|
2322
2522
|
payloadChecksumCrc32c: plan.checksumCrc32c,
|
|
2323
|
-
applicationVersion: target.applicationVersion
|
|
2523
|
+
applicationVersion: target.applicationVersion,
|
|
2524
|
+
deadlineUnixMs: BigInt(
|
|
2525
|
+
Date.now() + Math.max(0, Math.ceil(controlDeadlineAtMs - performance.now()))
|
|
2526
|
+
)
|
|
2324
2527
|
} satisfies ServiceMaintenanceRelocationPrepare;
|
|
2325
2528
|
// Start the Restore request first (it also registers the temporary
|
|
2326
2529
|
// queue and the assembly on the target), then stream the payload chunks
|
|
2327
2530
|
// on the same ordered connection. The Ready reply arrives only after
|
|
2328
2531
|
// the target assembled, verified, and restored the payload.
|
|
2329
|
-
const
|
|
2532
|
+
const ready = await this.sendControl(
|
|
2330
2533
|
meshName,
|
|
2331
2534
|
target.rid,
|
|
2332
2535
|
prepare,
|
|
2333
2536
|
signal,
|
|
2334
2537
|
controlDeadlineAtMs,
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2538
|
+
async (attemptSignal) => {
|
|
2539
|
+
if (isRelocationDebugEnabled())
|
|
2540
|
+
relocationDebug('coordinator.state_transfer_begin', {
|
|
2541
|
+
aggregateId: captured.envelope.aggregateId
|
|
2542
|
+
});
|
|
2543
|
+
await this.sendRelocationStateChunks(
|
|
2544
|
+
meshName,
|
|
2545
|
+
target,
|
|
2546
|
+
prepare,
|
|
2547
|
+
encoded,
|
|
2548
|
+
plan,
|
|
2549
|
+
attemptSignal
|
|
2550
|
+
);
|
|
2551
|
+
if (isRelocationDebugEnabled())
|
|
2552
|
+
relocationDebug('coordinator.await_ready', {
|
|
2553
|
+
aggregateId: captured.envelope.aggregateId
|
|
2554
|
+
});
|
|
2342
2555
|
}
|
|
2343
|
-
|
|
2344
|
-
} catch (chunkError) {
|
|
2345
|
-
readyOperation.catch(() => undefined);
|
|
2346
|
-
throw chunkError;
|
|
2347
|
-
}
|
|
2348
|
-
if (isRelocationDebugEnabled()) {
|
|
2349
|
-
relocationDebug('coordinator.await_ready', { aggregateId: captured.envelope.aggregateId });
|
|
2350
|
-
}
|
|
2351
|
-
const ready = await readyOperation;
|
|
2556
|
+
);
|
|
2352
2557
|
if (isRelocationDebugEnabled()) {
|
|
2353
2558
|
relocationDebug('coordinator.ready_received', {
|
|
2354
2559
|
aggregateId: captured.envelope.aggregateId
|
|
@@ -2359,6 +2564,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2359
2564
|
// forward the source can no longer reopen dispatch without risking two
|
|
2360
2565
|
// owners.
|
|
2361
2566
|
readyReceived = true;
|
|
2567
|
+
actorJoinProfile?.relayReady();
|
|
2362
2568
|
|
|
2363
2569
|
const boundaryBatch: Array<{
|
|
2364
2570
|
readonly frame: Buffer;
|
|
@@ -2413,14 +2619,16 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2413
2619
|
// The cutover submit result is not the authority result. The memory
|
|
2414
2620
|
// copy of the boundary batch and cutover stays until the authority
|
|
2415
2621
|
// settlement below (spec 28 §4.4).
|
|
2416
|
-
|
|
2622
|
+
const submittedRouteGeneration = this.requireMeshNode(meshName).observePeerRoute?.(
|
|
2623
|
+
target.rid
|
|
2624
|
+
).generation;
|
|
2417
2625
|
try {
|
|
2418
2626
|
for (const record of boundaryBatch) {
|
|
2419
2627
|
await this.submitControlFrame(meshName, target.rid, record.frame, 'relay');
|
|
2420
2628
|
}
|
|
2421
2629
|
await this.submitControlFrame(meshName, target.rid, cutoverFrame, 'cutover');
|
|
2422
2630
|
} catch (error) {
|
|
2423
|
-
|
|
2631
|
+
if (!durableRequestCanReplay(error)) throw error;
|
|
2424
2632
|
console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
|
|
2425
2633
|
}
|
|
2426
2634
|
const window = this.beginSourceCutoverWindow(
|
|
@@ -2428,7 +2636,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2428
2636
|
target.rid,
|
|
2429
2637
|
relayAuthorityId,
|
|
2430
2638
|
[...boundaryBatch.map((value) => value.frame), cutoverFrame],
|
|
2431
|
-
|
|
2639
|
+
submittedRouteGeneration,
|
|
2432
2640
|
limits
|
|
2433
2641
|
);
|
|
2434
2642
|
try {
|
|
@@ -2437,7 +2645,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2437
2645
|
primary,
|
|
2438
2646
|
targetFence,
|
|
2439
2647
|
restoreDeadlineReached,
|
|
2440
|
-
window
|
|
2648
|
+
window,
|
|
2649
|
+
relocationMetric
|
|
2441
2650
|
);
|
|
2442
2651
|
} finally {
|
|
2443
2652
|
this.closeSourceCutoverWindow(relayAuthorityId, window);
|
|
@@ -2452,26 +2661,37 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2452
2661
|
// Preserve won: the captured queue, timers and held records return to
|
|
2453
2662
|
// the source queue in their accepted order, and dispatch reopens.
|
|
2454
2663
|
await captured.abortSource();
|
|
2664
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
2455
2665
|
throw new Error(
|
|
2456
2666
|
`Relocation '${relayAuthorityId}' ended by the source Preserve fence; the source kept the authority.`
|
|
2457
2667
|
);
|
|
2458
2668
|
}
|
|
2459
2669
|
if (settlement === 'lost') {
|
|
2670
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
2460
2671
|
// Expired owner: dispatch and Store changes stop for good and every
|
|
2461
2672
|
// pending request without a terminal gets one Unavailable.
|
|
2462
2673
|
const unavailable = createInternalFrameworkException(
|
|
2463
2674
|
ZLinkFrameworkInternalErrorKind.RouteNotConnected,
|
|
2464
|
-
`Relocation '${relayAuthorityId}' source owner lease
|
|
2675
|
+
`Relocation '${relayAuthorityId}' source authority ended (source owner lease expired, authority Missing, or owner changed).`,
|
|
2465
2676
|
true
|
|
2466
2677
|
);
|
|
2467
2678
|
await captured.discardSource(unavailable);
|
|
2468
2679
|
throw new ServiceRelocationAuthorityError(unavailable.message, { cause: unavailable });
|
|
2469
2680
|
}
|
|
2681
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
2470
2682
|
for (const session of sessions) session.prepared.setReplayResults([]);
|
|
2471
|
-
await captured.commitSource();
|
|
2683
|
+
await captured.commitSource(settlement.followAuthorities);
|
|
2472
2684
|
await this.options.reconcileStatefulAuthorityRoutes?.(signal);
|
|
2473
2685
|
} catch (error) {
|
|
2474
|
-
if (
|
|
2686
|
+
if (
|
|
2687
|
+
this.disposed &&
|
|
2688
|
+
(error instanceof ServiceRelocationRuntimeStoppedError ||
|
|
2689
|
+
(error instanceof ZLinkBackendResultError &&
|
|
2690
|
+
error.result ===
|
|
2691
|
+
(error.operation === 'request' ? RequestResult.Terminated : SubmitResult.Terminated)))
|
|
2692
|
+
)
|
|
2693
|
+
this.completeSourceRelocationMetric(relocationMetric, undefined, true);
|
|
2694
|
+
if (settlement === undefined || typeof settlement === 'string') throw error;
|
|
2475
2695
|
// Target commit is confirmed; later source cleanup failures cannot move
|
|
2476
2696
|
// the authority back (spec 28 §10).
|
|
2477
2697
|
const authority = await requireAuthority(
|
|
@@ -2485,17 +2705,33 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2485
2705
|
error
|
|
2486
2706
|
);
|
|
2487
2707
|
} finally {
|
|
2708
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
2488
2709
|
if (!readyReceived) {
|
|
2489
2710
|
// Only an explicit failure before the Ready reply becomes accepted
|
|
2490
2711
|
// restores the source from the memory payload (spec 28 §9).
|
|
2491
2712
|
await captured.abortSource().catch(() => undefined);
|
|
2492
2713
|
}
|
|
2493
|
-
if (this.relocationAuthorityKeys.get(relayAuthorityId) === relayAuthorityKey) {
|
|
2494
|
-
this.relocationAuthorityKeys.delete(relayAuthorityId);
|
|
2495
|
-
}
|
|
2496
2714
|
}
|
|
2497
2715
|
}
|
|
2498
2716
|
|
|
2717
|
+
private completeSourceRelocationMetric(
|
|
2718
|
+
metric: ZLinkRelocationMetricOperation | undefined,
|
|
2719
|
+
settlement?: SourceAuthorityCommit | 'source' | 'lost',
|
|
2720
|
+
sourceStopped = false
|
|
2721
|
+
): void {
|
|
2722
|
+
metric?.complete(
|
|
2723
|
+
settlement === 'source'
|
|
2724
|
+
? 'aborted'
|
|
2725
|
+
: settlement === 'lost'
|
|
2726
|
+
? 'failed'
|
|
2727
|
+
: settlement !== undefined
|
|
2728
|
+
? 'completed'
|
|
2729
|
+
: sourceStopped
|
|
2730
|
+
? 'shutdown'
|
|
2731
|
+
: 'failed'
|
|
2732
|
+
);
|
|
2733
|
+
}
|
|
2734
|
+
|
|
2499
2735
|
/** Streams the payload chunks of one stage under the effective in-flight budgets. */
|
|
2500
2736
|
private async sendRelocationStateChunks(
|
|
2501
2737
|
meshName: string,
|
|
@@ -2518,6 +2754,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2518
2754
|
signal
|
|
2519
2755
|
);
|
|
2520
2756
|
try {
|
|
2757
|
+
signal?.throwIfAborted();
|
|
2521
2758
|
await this.sendControlOneWay(meshName, target.rid, {
|
|
2522
2759
|
kind: 'state',
|
|
2523
2760
|
relocation: identity.relocation,
|
|
@@ -2597,7 +2834,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2597
2834
|
targetNodeRid: RoutingId,
|
|
2598
2835
|
relocationKey: string,
|
|
2599
2836
|
frames: readonly Buffer[],
|
|
2600
|
-
|
|
2837
|
+
submittedRouteGeneration: bigint | undefined,
|
|
2601
2838
|
limits: ReturnType<ZLinkHostServiceRelocationRuntime['relocationLimits']>
|
|
2602
2839
|
): SourceCutoverWindow {
|
|
2603
2840
|
const submittedAt = performance.now();
|
|
@@ -2605,12 +2842,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2605
2842
|
meshName,
|
|
2606
2843
|
targetNodeRid,
|
|
2607
2844
|
frames,
|
|
2608
|
-
|
|
2845
|
+
submittedRouteGeneration,
|
|
2609
2846
|
cutoverSubmittedAtMs: submittedAt,
|
|
2610
2847
|
windowClosed: false,
|
|
2611
2848
|
followExpired: false
|
|
2612
2849
|
};
|
|
2850
|
+
const wasQuiescent = this.isSafeToShutdown();
|
|
2613
2851
|
this.sourceCutoverWindows.set(relocationKey, window);
|
|
2852
|
+
if (wasQuiescent) this.options.sourceQuiescenceChanged?.();
|
|
2614
2853
|
window.followTimer = setTimeout(() => {
|
|
2615
2854
|
window.followExpired = true;
|
|
2616
2855
|
// S1→S4 route convergence: the span the source must keep its Message
|
|
@@ -2640,27 +2879,17 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2640
2879
|
this.sourceCutoverWindows.get(relocationKey) === window
|
|
2641
2880
|
) {
|
|
2642
2881
|
this.sourceCutoverWindows.delete(relocationKey);
|
|
2882
|
+
if (this.isSafeToShutdown()) this.options.sourceQuiescenceChanged?.();
|
|
2643
2883
|
}
|
|
2644
2884
|
}
|
|
2645
2885
|
|
|
2646
|
-
/**
|
|
2647
|
-
* Sends the whole boundary batch and cutover again after a failed submit
|
|
2648
|
-
* or an observed loss of the target connection (spec 28 §4.4). The target
|
|
2649
|
-
* replaces its partial span with the whole batch.
|
|
2650
|
-
*/
|
|
2886
|
+
/** Relocation flow §4.4: replay the whole batch on a different ready route. */
|
|
2651
2887
|
private async resendSourceCutover(window: SourceCutoverWindow): Promise<void> {
|
|
2652
2888
|
const frames = window.frames;
|
|
2653
2889
|
if (frames === undefined) return;
|
|
2654
|
-
const
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
value.routingId !== null && String(value.routingId) === String(window.targetNodeRid)
|
|
2658
|
-
);
|
|
2659
|
-
if (peer !== undefined && peer.state !== MeshPeerRuntimeState.Serving) {
|
|
2660
|
-
window.resendPending = true;
|
|
2661
|
-
return;
|
|
2662
|
-
}
|
|
2663
|
-
if (!window.resendPending) return;
|
|
2890
|
+
const route = this.requireMeshNode(window.meshName).observePeerRoute?.(window.targetNodeRid);
|
|
2891
|
+
if (route?.ready !== true || route.generation === window.submittedRouteGeneration) return;
|
|
2892
|
+
window.submittedRouteGeneration = route.generation;
|
|
2664
2893
|
try {
|
|
2665
2894
|
for (const frame of frames) {
|
|
2666
2895
|
await this.submitControlFrame(
|
|
@@ -2670,9 +2899,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2670
2899
|
'cutover retransmission'
|
|
2671
2900
|
);
|
|
2672
2901
|
}
|
|
2673
|
-
|
|
2674
|
-
|
|
2675
|
-
// The next settlement round retries the whole batch.
|
|
2902
|
+
} catch (error) {
|
|
2903
|
+
if (!durableRequestCanReplay(error)) throw error;
|
|
2676
2904
|
}
|
|
2677
2905
|
}
|
|
2678
2906
|
|
|
@@ -2684,20 +2912,25 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2684
2912
|
* excludes a late target commit. `target`: the
|
|
2685
2913
|
* target committed. `source`: Preserve fenced the target out while the
|
|
2686
2914
|
* source owner lease was valid. `lost`: the source owner lease ended first
|
|
2687
|
-
* or
|
|
2688
|
-
*
|
|
2915
|
+
* or authority is Missing or fails the same-object, advanced-generation fence.
|
|
2916
|
+
* A committed later owner has no Follow snapshot (Location runtime §10). An uncertain
|
|
2917
|
+
* Store or lease result keeps the work retained and repeats the same decision.
|
|
2689
2918
|
*/
|
|
2690
2919
|
private async settleSourceAuthority(
|
|
2691
2920
|
envelope: ServiceRelocationEnvelope,
|
|
2692
2921
|
primary: ZLinkAuthoritySnapshot,
|
|
2693
2922
|
target: ServiceWireRelocationTarget,
|
|
2694
2923
|
restoreDeadlineReached: () => boolean,
|
|
2695
|
-
window: SourceCutoverWindow
|
|
2696
|
-
|
|
2924
|
+
window: SourceCutoverWindow,
|
|
2925
|
+
relocationMetric?: ZLinkRelocationMetricOperation
|
|
2926
|
+
): Promise<SourceAuthorityCommit | 'source' | 'lost'> {
|
|
2697
2927
|
const owner = { ownerId: primary.ownerId, leaseGeneration: primary.ownerLeaseGeneration };
|
|
2698
2928
|
const store = this.requireLocationStore();
|
|
2699
2929
|
for (;;) {
|
|
2700
|
-
if (this.disposed)
|
|
2930
|
+
if (this.disposed) {
|
|
2931
|
+
this.completeSourceRelocationMetric(relocationMetric, undefined, true);
|
|
2932
|
+
throw new ServiceRelocationRuntimeStoppedError();
|
|
2933
|
+
}
|
|
2701
2934
|
await this.resendSourceCutover(window);
|
|
2702
2935
|
const observed = await this.readSourceSettlement(envelope, primary, target).catch(
|
|
2703
2936
|
(error: unknown) => {
|
|
@@ -2705,7 +2938,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2705
2938
|
return undefined;
|
|
2706
2939
|
}
|
|
2707
2940
|
);
|
|
2708
|
-
if (observed?.kind === 'target') return
|
|
2941
|
+
if (observed?.kind === 'target') return observed;
|
|
2709
2942
|
if (observed?.kind === 'other') return 'lost';
|
|
2710
2943
|
if (restoreDeadlineReached() && observed?.kind === 'source') {
|
|
2711
2944
|
// An unreadable lease is an uncertain result, not an expiry.
|
|
@@ -2748,12 +2981,13 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2748
2981
|
primary: ZLinkAuthoritySnapshot,
|
|
2749
2982
|
target: ServiceWireRelocationTarget
|
|
2750
2983
|
): Promise<
|
|
2751
|
-
|
|
|
2984
|
+
| SourceAuthorityCommit
|
|
2752
2985
|
| { readonly kind: 'source'; readonly current: ZLinkAuthoritySnapshot }
|
|
2753
2986
|
| { readonly kind: 'other' }
|
|
2754
2987
|
> {
|
|
2755
2988
|
const primaryAuthorityKey = primaryKey(envelope);
|
|
2756
2989
|
let committed = 0;
|
|
2990
|
+
const followAuthorities = new Map<string, ZLinkAuthoritySnapshot>();
|
|
2757
2991
|
let primaryCurrent: ZLinkAuthoritySnapshot | undefined;
|
|
2758
2992
|
for (const participant of envelope.participants) {
|
|
2759
2993
|
const current = await this.requireLocationStore().readAuthority({
|
|
@@ -2762,17 +2996,23 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2762
2996
|
if (current.kind !== 'snapshot') return { kind: 'other' };
|
|
2763
2997
|
if (
|
|
2764
2998
|
current.objectGeneration === participant.objectGeneration &&
|
|
2765
|
-
current.ownerId === target.ownerId &&
|
|
2766
|
-
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
2767
2999
|
current.authorityOwnerGeneration > participant.authorityOwnerGeneration &&
|
|
2768
|
-
|
|
2769
|
-
|
|
3000
|
+
// Location runtime §6: an owner is the (OwnerId, OwnerLeaseGeneration) pair;
|
|
3001
|
+
// a restarted node keeps its OwnerId under a new lease generation.
|
|
3002
|
+
(current.ownerId !== primary.ownerId ||
|
|
3003
|
+
current.ownerLeaseGeneration !== primary.ownerLeaseGeneration)
|
|
2770
3004
|
) {
|
|
2771
3005
|
committed += 1;
|
|
3006
|
+
// Location runtime §10: only the observed first target can receive Follow.
|
|
3007
|
+
if (
|
|
3008
|
+
current.ownerId === target.ownerId &&
|
|
3009
|
+
current.ownerLeaseGeneration === target.ownerLeaseGeneration
|
|
3010
|
+
)
|
|
3011
|
+
followAuthorities.set(participant.key, current);
|
|
2772
3012
|
}
|
|
2773
3013
|
if (participant.key === primaryAuthorityKey) primaryCurrent = current;
|
|
2774
3014
|
}
|
|
2775
|
-
if (committed === envelope.participants.length) return { kind: 'target' };
|
|
3015
|
+
if (committed === envelope.participants.length) return { kind: 'target', followAuthorities };
|
|
2776
3016
|
if (primaryCurrent !== undefined && isSourceOwner(primaryCurrent, primary)) {
|
|
2777
3017
|
return { kind: 'source', current: primaryCurrent };
|
|
2778
3018
|
}
|
|
@@ -2823,12 +3063,25 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2823
3063
|
sourceNodeRid: RoutingId,
|
|
2824
3064
|
signal?: AbortSignal
|
|
2825
3065
|
): Promise<ServiceMaintenanceRelocationReady> {
|
|
3066
|
+
// Host relocation flow 05: a sealed target refuses a new PREPARE before any
|
|
3067
|
+
// staging. The typed ShuttingDown reaches the source as relocationFailed.
|
|
3068
|
+
if (this.targetAdmissionSealed()) {
|
|
3069
|
+
throw new ZLinkFrameworkException(
|
|
3070
|
+
ZLinkFrameworkErrorKind.ShuttingDown,
|
|
3071
|
+
'Relocation target admission is sealed for shutdown.'
|
|
3072
|
+
);
|
|
3073
|
+
}
|
|
3074
|
+
signal =
|
|
3075
|
+
signal === undefined
|
|
3076
|
+
? this.targetPreparationStop.signal
|
|
3077
|
+
: AbortSignal.any([signal, this.targetPreparationStop.signal]);
|
|
3078
|
+
signal.throwIfAborted();
|
|
2826
3079
|
const targetStatus = this.requireMeshNode(meshName).status();
|
|
2827
3080
|
const targetOwner = this.options.currentOwner();
|
|
2828
3081
|
const targetDescriptor = this.options.localDescriptor?.(meshName);
|
|
2829
3082
|
if (
|
|
2830
3083
|
targetOwner === undefined ||
|
|
2831
|
-
|
|
3084
|
+
!routingIdsEqual(targetStatus.routingId, request.target.nodeRid) ||
|
|
2832
3085
|
targetStatus.lifecycleGeneration !== request.target.nodeGeneration ||
|
|
2833
3086
|
targetOwner.ownerId !== request.target.ownerId ||
|
|
2834
3087
|
targetOwner.leaseGeneration !== request.target.ownerLeaseGeneration ||
|
|
@@ -2858,11 +3111,9 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2858
3111
|
try {
|
|
2859
3112
|
payload = await pendingAssembly.assembly.payload();
|
|
2860
3113
|
} catch (error) {
|
|
2861
|
-
//
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
// wire code in the vocabulary yet, so it falls through unwrapped to
|
|
2865
|
-
// the generic InternalFailure code (spec 15 §4).
|
|
3114
|
+
// Target shutdown preserves its lifecycle failure instead of reporting payload corruption.
|
|
3115
|
+
this.targetAssemblies.delete(stagingId);
|
|
3116
|
+
if (this.targetPreparationStop.signal.aborted) throw this.targetPreparationStop.signal.reason;
|
|
2866
3117
|
if (this.disposed) throw error;
|
|
2867
3118
|
const message =
|
|
2868
3119
|
error !== null && typeof error === 'object' && 'message' in error
|
|
@@ -2871,23 +3122,75 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2871
3122
|
throw new ServiceRelocationDataLostError(String(message));
|
|
2872
3123
|
}
|
|
2873
3124
|
const envelope = decodeServiceRelocationEnvelope(payload, request.targetAttemptGeneration);
|
|
2874
|
-
const
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
const releaseActivation =
|
|
2880
|
-
(await this.options.activationAdmission?.acquire(meshName, signal)) ?? (() => undefined);
|
|
3125
|
+
const recovery = canonicalActorJoinRecovery(envelope.participants);
|
|
3126
|
+
const releaseJoinAdmission =
|
|
3127
|
+
recovery === undefined
|
|
3128
|
+
? () => undefined
|
|
3129
|
+
: this.options.spotManager()!.beginActorJoinRelocation(recovery.request.handoffId);
|
|
2881
3130
|
let materialized:
|
|
2882
3131
|
(Pick<LocalStage, 'owner' | 'staging'> & { readonly target: LocalTargetPort }) | undefined;
|
|
2883
3132
|
let reservation: TargetRelocationReservation | undefined;
|
|
3133
|
+
let releaseActivation: () => void = () => undefined;
|
|
2884
3134
|
try {
|
|
3135
|
+
const inventoryEnvelope = attachCanonicalBoundSessions(
|
|
3136
|
+
await this.resolveTargetParticipantInventory(meshName, request, envelope, signal),
|
|
3137
|
+
request,
|
|
3138
|
+
meshName
|
|
3139
|
+
);
|
|
3140
|
+
validatePrepareEnvelope(request, inventoryEnvelope);
|
|
3141
|
+
releaseActivation =
|
|
3142
|
+
(await this.options.activationAdmission?.acquire(meshName, signal)) ?? (() => undefined);
|
|
2885
3143
|
// Deferred Join completion publishes into the source authority's
|
|
2886
3144
|
// canonical relocation slot. Do that while the row is still CAS-able;
|
|
2887
3145
|
// the aggregate marker installed by reserveTargetForPrepare deliberately
|
|
2888
3146
|
// rejects unrelated authority CAS until commit/abort.
|
|
2889
3147
|
materialized = await this.materializeTargetEnvelope(meshName, inventoryEnvelope, signal);
|
|
2890
3148
|
await materialized.owner.restoreSavedWork(materialized.staging, signal);
|
|
3149
|
+
for (const participant of inventoryEnvelope.participants) {
|
|
3150
|
+
const binding = participant.sessionBinding;
|
|
3151
|
+
if (binding === undefined) continue;
|
|
3152
|
+
const peerRid = decodeRoutingId(
|
|
3153
|
+
'',
|
|
3154
|
+
Buffer.from(binding.sessionOwnerNodeRid).toString('hex')
|
|
3155
|
+
);
|
|
3156
|
+
const node = this.requireMeshNode(meshName);
|
|
3157
|
+
if (routingIdsEqual(node.status().routingId, peerRid)) continue;
|
|
3158
|
+
if (node.isPeerRouteReady?.(peerRid, binding.sessionOwnerNodeGeneration) !== true) {
|
|
3159
|
+
// Subscribe before requesting reconciliation so a synchronous admission cannot be lost.
|
|
3160
|
+
const wait = node.waitForPeerRoute;
|
|
3161
|
+
if (wait === undefined) throw new Error('Mesh peer route observation is unavailable.');
|
|
3162
|
+
const ready = wait.call(
|
|
3163
|
+
node,
|
|
3164
|
+
peerRid,
|
|
3165
|
+
binding.sessionOwnerNodeGeneration,
|
|
3166
|
+
request.deadlineUnixMs,
|
|
3167
|
+
signal
|
|
3168
|
+
);
|
|
3169
|
+
void this.options
|
|
3170
|
+
.requestPeerReconcile?.(meshName, String(peerRid))
|
|
3171
|
+
.catch((error) =>
|
|
3172
|
+
console.error('[zlink.runtime.relocation.session_route_reconcile_failed]', error)
|
|
3173
|
+
);
|
|
3174
|
+
try {
|
|
3175
|
+
await ready;
|
|
3176
|
+
} catch (error) {
|
|
3177
|
+
if (error instanceof OperationTimeoutError) {
|
|
3178
|
+
throw new ZLinkFrameworkException(
|
|
3179
|
+
ZLinkFrameworkErrorKind.DeadlineExceeded,
|
|
3180
|
+
'Relocation source operation deadline exceeded waiting for Session owner route.'
|
|
3181
|
+
);
|
|
3182
|
+
}
|
|
3183
|
+
throw error;
|
|
3184
|
+
}
|
|
3185
|
+
}
|
|
3186
|
+
}
|
|
3187
|
+
if (request.deadlineUnixMs <= BigInt(Date.now())) {
|
|
3188
|
+
throw new ZLinkFrameworkException(
|
|
3189
|
+
ZLinkFrameworkErrorKind.DeadlineExceeded,
|
|
3190
|
+
'Relocation source operation deadline exceeded before relay-ready.'
|
|
3191
|
+
);
|
|
3192
|
+
}
|
|
3193
|
+
signal.throwIfAborted();
|
|
2891
3194
|
reservation = await this.reserveTargetForPrepare(
|
|
2892
3195
|
meshName,
|
|
2893
3196
|
request,
|
|
@@ -2897,7 +3200,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2897
3200
|
const offer: TargetRelocationOffer = {
|
|
2898
3201
|
prepare: request,
|
|
2899
3202
|
prepareFingerprint: fingerprint,
|
|
2900
|
-
authenticatedSourceNodeRid:
|
|
3203
|
+
authenticatedSourceNodeRid: sourceNodeRid,
|
|
2901
3204
|
envelope: inventoryEnvelope,
|
|
2902
3205
|
reservation
|
|
2903
3206
|
};
|
|
@@ -2908,10 +3211,13 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
2908
3211
|
lane: Promise.resolve(),
|
|
2909
3212
|
cutoverReceived: false,
|
|
2910
3213
|
boundaryRelay: [],
|
|
2911
|
-
releaseActivation
|
|
3214
|
+
releaseActivation,
|
|
3215
|
+
releaseJoinAdmission
|
|
2912
3216
|
};
|
|
2913
3217
|
this.targetStages.set(stagingId, stage);
|
|
2914
3218
|
} catch (error) {
|
|
3219
|
+
this.targetAssemblies.delete(stagingId);
|
|
3220
|
+
releaseJoinAdmission();
|
|
2915
3221
|
releaseActivation();
|
|
2916
3222
|
if (reservation !== undefined) {
|
|
2917
3223
|
await this.abortTargetReservation(reservation, signal).catch(() => undefined);
|
|
@@ -3078,30 +3384,51 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3078
3384
|
stage.releaseActivation();
|
|
3079
3385
|
resumeStartedAt = performance.now();
|
|
3080
3386
|
stage.phase = 'committed';
|
|
3081
|
-
await stage.owner.normalize(stage.staging, authority, signal);
|
|
3082
3387
|
await stage.owner.publish(stage.staging, authority, signal);
|
|
3083
|
-
// The target may have visited this Actor before and still retain that
|
|
3084
|
-
// tenure in its private Actor-client resolver. The target-only CAS is
|
|
3085
|
-
// now authoritative, so expire the old projection before OnJoinedActor
|
|
3086
|
-
// can issue an Actor request and deadlock behind source leave.
|
|
3087
|
-
for (const hidden of stage.staging.hidden.values()) {
|
|
3088
|
-
if (hidden.actor !== undefined) {
|
|
3089
|
-
this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
|
|
3090
|
-
}
|
|
3091
|
-
}
|
|
3092
|
-
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
3093
3388
|
// Clear the final relocation publication before admission becomes
|
|
3094
3389
|
// externally visible so a later Join cannot reserve against the
|
|
3095
3390
|
// previous root. A bounded CAS conflict must not roll back already
|
|
3096
3391
|
// committed target ownership or strand its staged object.
|
|
3097
3392
|
try {
|
|
3098
|
-
await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
3393
|
+
authority = await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
3099
3394
|
} catch (error) {
|
|
3100
3395
|
console.warn('[zlink.runtime.relocation.publication_clear_failed]', stagingId, error);
|
|
3101
3396
|
this.options.metrics?.count('zlink.relocation.publication_clear_failed', 1, {
|
|
3102
3397
|
mesh_name: meshName
|
|
3103
3398
|
});
|
|
3104
3399
|
}
|
|
3400
|
+
for (const [index, participant] of stage.staging.envelope.participants.entries()) {
|
|
3401
|
+
const hidden = stage.staging.hidden.get(participant.key);
|
|
3402
|
+
if (hidden?.actor === undefined || hidden.replayPackets.length === 0) continue;
|
|
3403
|
+
const coordinator = this.targetReplyRelayCoordinator(meshName, stage, authority);
|
|
3404
|
+
hidden.replayStart = stage.finalize;
|
|
3405
|
+
hidden.onReplayResult = (packet, result) =>
|
|
3406
|
+
this.relayTerminalReply(
|
|
3407
|
+
meshName,
|
|
3408
|
+
stage,
|
|
3409
|
+
coordinator,
|
|
3410
|
+
BigInt(index + 1),
|
|
3411
|
+
packet,
|
|
3412
|
+
result,
|
|
3413
|
+
signal
|
|
3414
|
+
);
|
|
3415
|
+
}
|
|
3416
|
+
await stage.owner.normalize(stage.staging, authority, signal);
|
|
3417
|
+
// The target may have visited this Actor before and still retain that
|
|
3418
|
+
// tenure in its private Actor-client resolver. The target-only CAS is
|
|
3419
|
+
// now authoritative, so expire the old projection before OnJoinedActor
|
|
3420
|
+
// can issue an Actor request and deadlock behind source leave.
|
|
3421
|
+
for (const hidden of stage.staging.hidden.values()) {
|
|
3422
|
+
if (hidden.actor !== undefined) {
|
|
3423
|
+
this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
3427
|
+
await this.publishSessionRoutes(stage.staging);
|
|
3428
|
+
stage.phase = 'open';
|
|
3429
|
+
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
3430
|
+
this.targetStages.delete(stagingId);
|
|
3431
|
+
this.targetAssemblies.delete(stagingId);
|
|
3105
3432
|
await stage.owner.openAdmission(stage.staging, signal);
|
|
3106
3433
|
// S2→S3: owner confirmation to application dispatch opening.
|
|
3107
3434
|
this.options.metrics?.duration(
|
|
@@ -3109,17 +3436,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3109
3436
|
Math.max(0, performance.now() - resumeStartedAt) / MILLISECONDS_PER_SECOND,
|
|
3110
3437
|
{ mesh_name: meshName }
|
|
3111
3438
|
);
|
|
3112
|
-
await this.publishSessionRoutes(stage.staging);
|
|
3113
|
-
await this.relayTerminalReplies(
|
|
3114
|
-
meshName,
|
|
3115
|
-
stage,
|
|
3116
|
-
this.targetReplyRelayCoordinator(meshName, stage, authority),
|
|
3117
|
-
signal
|
|
3118
|
-
);
|
|
3119
|
-
stage.phase = 'open';
|
|
3120
|
-
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
3121
|
-
this.targetStages.delete(stagingId);
|
|
3122
|
-
this.targetAssemblies.delete(stagingId);
|
|
3123
3439
|
} catch (error) {
|
|
3124
3440
|
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
3125
3441
|
this.targetStages.delete(stagingId);
|
|
@@ -3134,6 +3450,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3134
3450
|
throw new ServiceRelocationPostCommitError(authority, stage.staging, error);
|
|
3135
3451
|
}
|
|
3136
3452
|
throw error;
|
|
3453
|
+
} finally {
|
|
3454
|
+
stage.releaseJoinAdmission();
|
|
3137
3455
|
}
|
|
3138
3456
|
}
|
|
3139
3457
|
|
|
@@ -3154,7 +3472,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3154
3472
|
target.generation !== request.object.objectGeneration ||
|
|
3155
3473
|
target.targetNodeRid !== preparedTarget.nodeRid ||
|
|
3156
3474
|
target.targetNodeGeneration !== preparedTarget.nodeGeneration ||
|
|
3157
|
-
target.authorityOwnerGeneration !== request.object.expectedAuthorityOwnerGeneration
|
|
3475
|
+
target.authorityOwnerGeneration !== request.object.expectedAuthorityOwnerGeneration ||
|
|
3158
3476
|
target.ownerLeaseGeneration !== preparedTarget.ownerLeaseGeneration
|
|
3159
3477
|
) {
|
|
3160
3478
|
throw new Error('Relocation data does not target the exact temporary Actor queue.');
|
|
@@ -3263,9 +3581,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3263
3581
|
if (profile.state.actorId !== input.actorId || !input.succeeded) {
|
|
3264
3582
|
throw new Error(`Actor Join relocation '${input.transferId}' source leave identity changed.`);
|
|
3265
3583
|
}
|
|
3266
|
-
this.
|
|
3267
|
-
this.terminalActorJoinSourceLeaves.remember(input.transferId, input.actorId);
|
|
3268
|
-
void this.finalizeActorJoinSource(profile);
|
|
3584
|
+
void this.retireActorJoinSource(input.transferId, profile);
|
|
3269
3585
|
return true;
|
|
3270
3586
|
}
|
|
3271
3587
|
|
|
@@ -3293,7 +3609,9 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3293
3609
|
): Promise<void> {
|
|
3294
3610
|
const submitted = await this.sendInfrastructureControl(meshName, targetNodeRid, frame);
|
|
3295
3611
|
if (submitted !== SubmitResult.Ok) {
|
|
3296
|
-
throw new
|
|
3612
|
+
throw new ZLinkBackendResultError('submit', submitted, undefined, {
|
|
3613
|
+
cause: new Error(`Relocation ${kind} was not accepted by RouteMesh.`)
|
|
3614
|
+
});
|
|
3297
3615
|
}
|
|
3298
3616
|
}
|
|
3299
3617
|
|
|
@@ -3301,7 +3619,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3301
3619
|
stage: LocalStage,
|
|
3302
3620
|
primary: ZLinkAuthoritySnapshot,
|
|
3303
3621
|
signal?: AbortSignal
|
|
3304
|
-
): Promise<
|
|
3622
|
+
): Promise<ZLinkAuthoritySnapshot> {
|
|
3305
3623
|
await this.releaseParticipantAuthorities(
|
|
3306
3624
|
stage.staging.envelope,
|
|
3307
3625
|
stage.staging.primaryAuthorityKey.value,
|
|
@@ -3312,7 +3630,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3312
3630
|
// committed snapshot decides. A conflict means another owner changed the
|
|
3313
3631
|
// row; the caller records it and leaves the committed ownership intact.
|
|
3314
3632
|
const publication = this.codec.read(primary.payload);
|
|
3315
|
-
if (publication === undefined) return;
|
|
3633
|
+
if (publication === undefined) return primary;
|
|
3316
3634
|
const result = await this.requireLocationStore().compareExchangeAuthority(
|
|
3317
3635
|
stage.staging.primaryAuthorityKey,
|
|
3318
3636
|
primary.storeVersion,
|
|
@@ -3326,6 +3644,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3326
3644
|
if (result.kind !== 'stored') {
|
|
3327
3645
|
throw new Error('Relocation target authority normalization CAS failed.');
|
|
3328
3646
|
}
|
|
3647
|
+
return { ...result, kind: 'snapshot' };
|
|
3329
3648
|
}
|
|
3330
3649
|
|
|
3331
3650
|
private async handleReplyRelay(
|
|
@@ -3337,49 +3656,53 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3337
3656
|
if (sourceNodeRid === null) {
|
|
3338
3657
|
throw new Error('Relocation reply relay has no authenticated target node.');
|
|
3339
3658
|
}
|
|
3340
|
-
await this.requireAdmittedPeer(meshName,
|
|
3341
|
-
const
|
|
3342
|
-
const
|
|
3343
|
-
if (authorityKey === undefined) {
|
|
3344
|
-
throw new Error('Relocation reply relay has no registered durable authority.');
|
|
3345
|
-
}
|
|
3346
|
-
const authority = await requireAuthority(
|
|
3347
|
-
this.requireLocationStore(),
|
|
3348
|
-
{ value: authorityKey } as ZLinkAuthorityKey,
|
|
3349
|
-
signal
|
|
3350
|
-
);
|
|
3351
|
-
if (
|
|
3352
|
-
authority.storeVersion.value !== relay.coordinator.expectedAuthorityStoreVersion ||
|
|
3353
|
-
authority.ownerId !== relay.coordinator.ownerId ||
|
|
3354
|
-
authority.ownerLeaseGeneration !== relay.coordinator.leaseGeneration ||
|
|
3355
|
-
String(authority.allocation.descriptor.rid) !== relay.coordinator.nodeRid ||
|
|
3356
|
-
authority.allocation.descriptorLifecycleGeneration !== relay.coordinator.nodeGeneration
|
|
3357
|
-
) {
|
|
3358
|
-
throw new Error('Relocation reply relay coordinator authority fence is stale.');
|
|
3359
|
-
}
|
|
3360
|
-
const accepted = this.options.actorTransfer.relayCanonicalMaintenanceTerminal(
|
|
3361
|
-
wireIdText(relay.operation),
|
|
3362
|
-
relay.replyRouteId.toString(),
|
|
3363
|
-
handoffResultFromRelay(relay),
|
|
3364
|
-
sourceNodeRid
|
|
3365
|
-
);
|
|
3366
|
-
if (
|
|
3367
|
-
(accepted.status !== 'terminalReceived' && accepted.status !== 'alreadyTerminal') ||
|
|
3368
|
-
accepted.source === undefined
|
|
3369
|
-
) {
|
|
3370
|
-
throw new Error('Canonical relocation reply relay collided with another source route.');
|
|
3371
|
-
}
|
|
3372
|
-
const requestSource = handoffSourceFence(accepted.source);
|
|
3373
|
-
const localStatus = this.requireMeshNode(meshName).status();
|
|
3659
|
+
await this.requireAdmittedPeer(meshName, sourceNodeRid, relay.coordinator, signal);
|
|
3660
|
+
const node = this.requireMeshNode(meshName);
|
|
3661
|
+
const localStatus = node.status();
|
|
3374
3662
|
const localOwner = this.options.currentOwner();
|
|
3375
|
-
if (
|
|
3376
|
-
|
|
3377
|
-
|
|
3378
|
-
|
|
3379
|
-
|
|
3380
|
-
|
|
3381
|
-
|
|
3382
|
-
|
|
3663
|
+
if (localOwner === undefined)
|
|
3664
|
+
throw new Error('Relocation requester owner lease is unavailable.');
|
|
3665
|
+
let status = node.acceptRelocatedActorTerminal?.(relay.operation, relay.replyRouteId, {
|
|
3666
|
+
terminalResult: relay.terminalResult,
|
|
3667
|
+
failureCode: relay.failureCode,
|
|
3668
|
+
...(relay.payload === undefined
|
|
3669
|
+
? {}
|
|
3670
|
+
: {
|
|
3671
|
+
payload: {
|
|
3672
|
+
packetName: relay.payload.packetName,
|
|
3673
|
+
contentType: relay.payload.contentType,
|
|
3674
|
+
payload: Buffer.from(relay.payload.bytes)
|
|
3675
|
+
}
|
|
3676
|
+
})
|
|
3677
|
+
});
|
|
3678
|
+
let requestSource: ServiceWireRequestSourceFence = {
|
|
3679
|
+
ownerId: localOwner.ownerId,
|
|
3680
|
+
leaseGeneration: localOwner.leaseGeneration,
|
|
3681
|
+
nodeRid: localStatus.routingId,
|
|
3682
|
+
nodeGeneration: localStatus.lifecycleGeneration
|
|
3683
|
+
};
|
|
3684
|
+
if (status === undefined) {
|
|
3685
|
+
const accepted = this.options.actorTransfer.relayCanonicalMaintenanceTerminal(
|
|
3686
|
+
relay.operation.high.toString(16).padStart(16, '0') +
|
|
3687
|
+
relay.operation.low.toString(16).padStart(16, '0'),
|
|
3688
|
+
relay.replyRouteId.toString(),
|
|
3689
|
+
handoffResultFromRelay(relay),
|
|
3690
|
+
sourceNodeRid
|
|
3691
|
+
);
|
|
3692
|
+
if (accepted.status === 'notAcknowledged' || accepted.source === undefined) {
|
|
3693
|
+
throw new Error('Canonical relocation reply relay collided with another source route.');
|
|
3694
|
+
}
|
|
3695
|
+
requestSource = handoffSourceFence(accepted.source);
|
|
3696
|
+
if (
|
|
3697
|
+
!sameRequestSource(requestSource, {
|
|
3698
|
+
ownerId: localOwner.ownerId,
|
|
3699
|
+
leaseGeneration: localOwner.leaseGeneration,
|
|
3700
|
+
nodeRid: localStatus.routingId,
|
|
3701
|
+
nodeGeneration: localStatus.lifecycleGeneration
|
|
3702
|
+
})
|
|
3703
|
+
)
|
|
3704
|
+
throw new Error('Relocation reply relay request-source fence changed after capture.');
|
|
3705
|
+
status = accepted.status;
|
|
3383
3706
|
}
|
|
3384
3707
|
const ack: ServiceMaintenanceReplyRelayAck = {
|
|
3385
3708
|
relocation: relay.relocation,
|
|
@@ -3387,7 +3710,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3387
3710
|
operation: relay.operation,
|
|
3388
3711
|
replyRouteId: relay.replyRouteId,
|
|
3389
3712
|
requestSource,
|
|
3390
|
-
status
|
|
3713
|
+
status
|
|
3391
3714
|
};
|
|
3392
3715
|
if (
|
|
3393
3716
|
(await this.sendInfrastructureControl(
|
|
@@ -3407,9 +3730,8 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3407
3730
|
expectedRequestSource: ServiceWireRequestSourceFence,
|
|
3408
3731
|
signal?: AbortSignal
|
|
3409
3732
|
): Promise<RelocationTerminalDelivery> {
|
|
3410
|
-
const targetRid = String(ackTargetNodeRid);
|
|
3411
3733
|
const identityKey = replyRelayIdentityKey(request);
|
|
3412
|
-
const key = replyRelayPendingKey(
|
|
3734
|
+
const key = replyRelayPendingKey(ackTargetNodeRid, request);
|
|
3413
3735
|
if (this.pendingReplyRelays.has(key)) {
|
|
3414
3736
|
throw new Error(`Relocation reply relay ACK '${key}' is already pending.`);
|
|
3415
3737
|
}
|
|
@@ -3422,7 +3744,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3422
3744
|
action();
|
|
3423
3745
|
};
|
|
3424
3746
|
const pending: PendingRelocationReplyRelay = {
|
|
3425
|
-
ackTargetNodeRid
|
|
3747
|
+
ackTargetNodeRid,
|
|
3426
3748
|
identityKey,
|
|
3427
3749
|
request,
|
|
3428
3750
|
expectedRequestSource,
|
|
@@ -3467,8 +3789,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3467
3789
|
signal?: AbortSignal
|
|
3468
3790
|
): Promise<void> {
|
|
3469
3791
|
const identityKey = replyRelayIdentityKey(ack);
|
|
3470
|
-
const exactKey =
|
|
3471
|
-
sourceNodeRid === null ? undefined : replyRelayPendingKey(String(sourceNodeRid), ack);
|
|
3792
|
+
const exactKey = sourceNodeRid === null ? undefined : replyRelayPendingKey(sourceNodeRid, ack);
|
|
3472
3793
|
const pending = exactKey === undefined ? undefined : this.pendingReplyRelays.get(exactKey);
|
|
3473
3794
|
if (pending === undefined) {
|
|
3474
3795
|
const collision = [...this.pendingReplyRelays.values()].find(
|
|
@@ -3506,14 +3827,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3506
3827
|
signal?: AbortSignal
|
|
3507
3828
|
): Promise<void> {
|
|
3508
3829
|
const descriptor = (await this.options.liveDescriptors(meshName, signal)).find(
|
|
3509
|
-
(value) =>
|
|
3830
|
+
(value) => routingIdsEqual(value.rid, authenticatedRid) && value.state === 1
|
|
3510
3831
|
);
|
|
3511
3832
|
if (
|
|
3512
3833
|
descriptor === undefined ||
|
|
3513
3834
|
(expected !== undefined &&
|
|
3514
3835
|
(descriptor.ownerId !== expected.ownerId ||
|
|
3515
3836
|
descriptor.leaseGeneration !== expected.leaseGeneration ||
|
|
3516
|
-
|
|
3837
|
+
!routingIdsEqual(descriptor.rid, expected.nodeRid) ||
|
|
3517
3838
|
descriptor.lifecycleGeneration !== expected.nodeGeneration))
|
|
3518
3839
|
) {
|
|
3519
3840
|
throw new Error('Relocation reply relay peer is not the exact admitted source fence.');
|
|
@@ -3532,7 +3853,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3532
3853
|
owner === undefined ||
|
|
3533
3854
|
owner.ownerId !== target.ownerId ||
|
|
3534
3855
|
owner.leaseGeneration !== target.ownerLeaseGeneration ||
|
|
3535
|
-
|
|
3856
|
+
!routingIdsEqual(status.routingId, target.nodeRid) ||
|
|
3536
3857
|
status.lifecycleGeneration !== target.nodeGeneration
|
|
3537
3858
|
) {
|
|
3538
3859
|
throw new Error('Relocation reply relay target coordinator fence is stale.');
|
|
@@ -3540,42 +3861,47 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3540
3861
|
return {
|
|
3541
3862
|
ownerId: owner.ownerId,
|
|
3542
3863
|
leaseGeneration: owner.leaseGeneration,
|
|
3543
|
-
nodeRid:
|
|
3864
|
+
nodeRid: status.routingId,
|
|
3544
3865
|
nodeGeneration: status.lifecycleGeneration,
|
|
3545
3866
|
expectedAuthorityStoreVersion: authority.storeVersion.value
|
|
3546
3867
|
};
|
|
3547
3868
|
}
|
|
3548
3869
|
|
|
3549
|
-
private async
|
|
3870
|
+
private async relayTerminalReply(
|
|
3550
3871
|
meshName: string,
|
|
3551
3872
|
stage: LocalStage,
|
|
3552
3873
|
coordinator: ServiceWireRelocationCoordinatorFence,
|
|
3874
|
+
participantId: bigint,
|
|
3875
|
+
packet: ZLinkActorHandoffPacket,
|
|
3876
|
+
result: ZLinkActorHandoffResult,
|
|
3553
3877
|
signal?: AbortSignal
|
|
3554
3878
|
): Promise<void> {
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
)
|
|
3577
|
-
|
|
3578
|
-
|
|
3879
|
+
if (!packet.returnResponse || packet.source === undefined) return;
|
|
3880
|
+
if (result.index !== packet.index) {
|
|
3881
|
+
throw new Error('Relocation request result changed after target dispatch.');
|
|
3882
|
+
}
|
|
3883
|
+
const completion: RelocationTerminalCompletion = {
|
|
3884
|
+
index: packet.index,
|
|
3885
|
+
operationId: packet.messageFollowContext.operationId,
|
|
3886
|
+
source: packet.source,
|
|
3887
|
+
result
|
|
3888
|
+
};
|
|
3889
|
+
const source = handoffSourceFence(packet.source);
|
|
3890
|
+
await this.sendReplyRelay(
|
|
3891
|
+
meshName,
|
|
3892
|
+
source.nodeRid,
|
|
3893
|
+
maintenanceReplyRelay(
|
|
3894
|
+
stage,
|
|
3895
|
+
coordinator,
|
|
3896
|
+
participantId,
|
|
3897
|
+
completion,
|
|
3898
|
+
packet,
|
|
3899
|
+
this.options.registration.messageSerializers,
|
|
3900
|
+
this.options.flowCreationEnabled?.() ?? false
|
|
3901
|
+
),
|
|
3902
|
+
source,
|
|
3903
|
+
signal
|
|
3904
|
+
);
|
|
3579
3905
|
}
|
|
3580
3906
|
|
|
3581
3907
|
private async exactSourceLeaseExpired(
|
|
@@ -3611,70 +3937,39 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3611
3937
|
request: ZLinkServiceRelocationControlRequest,
|
|
3612
3938
|
signal?: AbortSignal,
|
|
3613
3939
|
deadlineAtMs: number = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS,
|
|
3614
|
-
|
|
3940
|
+
onSubmitted?: (signal: AbortSignal) => Promise<void>
|
|
3615
3941
|
): Promise<ReturnType<typeof decodeServiceRelocationControlResponse>> {
|
|
3616
3942
|
const key = controlAckKey(request);
|
|
3617
|
-
const frames =
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
3622
|
-
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
if (remainingMs <= 0) {
|
|
3626
|
-
throw createInternalFrameworkException(
|
|
3627
|
-
ZLinkFrameworkInternalErrorKind.DeadlineExceeded,
|
|
3628
|
-
`Relocation control ACK '${key}' timed out.`,
|
|
3629
|
-
true,
|
|
3630
|
-
lastRequestError
|
|
3631
|
-
);
|
|
3632
|
-
}
|
|
3633
|
-
const attemptTimeoutMs = Math.max(
|
|
3634
|
-
1,
|
|
3635
|
-
Math.min(RELOCATION_ATTEMPT_TIMEOUT_MS, Math.ceil(remainingMs))
|
|
3943
|
+
const frames = [encodeServiceRelocationControlRequest(request)];
|
|
3944
|
+
signal?.throwIfAborted();
|
|
3945
|
+
const remainingMs = Math.ceil(deadlineAtMs - performance.now());
|
|
3946
|
+
if (remainingMs <= 0) {
|
|
3947
|
+
throw createInternalFrameworkException(
|
|
3948
|
+
ZLinkFrameworkInternalErrorKind.DeadlineExceeded,
|
|
3949
|
+
`Relocation control ACK '${key}' timed out.`,
|
|
3950
|
+
true
|
|
3636
3951
|
);
|
|
3637
|
-
const startedAtMs = performance.now();
|
|
3638
|
-
try {
|
|
3639
|
-
const replyFrames = await this.requestInfrastructureControl(
|
|
3640
|
-
meshName,
|
|
3641
|
-
targetNodeRid,
|
|
3642
|
-
frames,
|
|
3643
|
-
attemptTimeoutMs
|
|
3644
|
-
);
|
|
3645
|
-
if (replyFrames.length !== 1) {
|
|
3646
|
-
throw new ServiceWireProtocolError(
|
|
3647
|
-
`Relocation control '${key}' reply must contain exactly one frame.`
|
|
3648
|
-
);
|
|
3649
|
-
}
|
|
3650
|
-
const response = decodeServiceRelocationControlResponse(Buffer.from(replyFrames[0]!));
|
|
3651
|
-
if (response.kind === 'failed') {
|
|
3652
|
-
validateControlFailureResponse(request, response);
|
|
3653
|
-
throw relocationFailureException(response);
|
|
3654
|
-
}
|
|
3655
|
-
validateControlResponse(request, response);
|
|
3656
|
-
return response;
|
|
3657
|
-
} catch (error) {
|
|
3658
|
-
if (error instanceof ServiceWireProtocolError || error instanceof ZLinkFrameworkException) {
|
|
3659
|
-
throw error;
|
|
3660
|
-
}
|
|
3661
|
-
lastRequestError = error;
|
|
3662
|
-
if (isRelocationDebugEnabled()) {
|
|
3663
|
-
relocationDebug('control.request_retry', {
|
|
3664
|
-
key,
|
|
3665
|
-
kind: request.kind,
|
|
3666
|
-
targetRid: String(targetNodeRid)
|
|
3667
|
-
});
|
|
3668
|
-
}
|
|
3669
|
-
const retryDelayMs = Math.min(
|
|
3670
|
-
Math.max(0, attemptTimeoutMs - (performance.now() - startedAtMs)),
|
|
3671
|
-
Math.max(0, deadlineAtMs - performance.now())
|
|
3672
|
-
);
|
|
3673
|
-
if (retryDelayMs > 0) {
|
|
3674
|
-
await waitForRelocationRetry(retryDelayMs, signal);
|
|
3675
|
-
}
|
|
3676
|
-
}
|
|
3677
3952
|
}
|
|
3953
|
+
const replyFrames = await this.requestInfrastructureControl(
|
|
3954
|
+
meshName,
|
|
3955
|
+
targetNodeRid,
|
|
3956
|
+
frames,
|
|
3957
|
+
remainingMs,
|
|
3958
|
+
signal,
|
|
3959
|
+
onSubmitted
|
|
3960
|
+
);
|
|
3961
|
+
if (replyFrames.length !== 1) {
|
|
3962
|
+
throw new ServiceWireProtocolError(
|
|
3963
|
+
`Relocation control '${key}' reply must contain exactly one frame.`
|
|
3964
|
+
);
|
|
3965
|
+
}
|
|
3966
|
+
const response = decodeServiceRelocationControlResponse(Buffer.from(replyFrames[0]!));
|
|
3967
|
+
if (response.kind === 'failed') {
|
|
3968
|
+
validateControlFailureResponse(request, response);
|
|
3969
|
+
throw relocationFailureException(response);
|
|
3970
|
+
}
|
|
3971
|
+
validateControlResponse(request, response);
|
|
3972
|
+
return response;
|
|
3678
3973
|
}
|
|
3679
3974
|
|
|
3680
3975
|
private async selectTarget(
|
|
@@ -3687,13 +3982,13 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3687
3982
|
const descriptors = await this.options.liveDescriptors(meshName, signal);
|
|
3688
3983
|
const local =
|
|
3689
3984
|
this.options.localDescriptor?.(meshName) ??
|
|
3690
|
-
descriptors.find((descriptor) =>
|
|
3985
|
+
descriptors.find((descriptor) => routingIdsEqual(descriptor.rid, localRid));
|
|
3691
3986
|
const requiredVersion = targetApplicationVersion ?? local?.applicationVersion;
|
|
3692
3987
|
const peers = this.requireMeshNode(meshName).peers();
|
|
3693
3988
|
const candidates = descriptors
|
|
3694
3989
|
.filter(
|
|
3695
3990
|
(descriptor) =>
|
|
3696
|
-
|
|
3991
|
+
!routingIdsEqual(descriptor.rid, localRid) &&
|
|
3697
3992
|
descriptor.state === ZLinkFrameworkRuntimeState.Serving &&
|
|
3698
3993
|
descriptor.objectRole === ZLinkObjectRole.Server &&
|
|
3699
3994
|
descriptor.placementWeight > 0 &&
|
|
@@ -3703,7 +3998,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3703
3998
|
peers.some(
|
|
3704
3999
|
(peer) =>
|
|
3705
4000
|
peer.routingId !== null &&
|
|
3706
|
-
|
|
4001
|
+
routingIdsEqual(peer.routingId, descriptor.rid) &&
|
|
3707
4002
|
peer.lifecycleGeneration === descriptor.lifecycleGeneration &&
|
|
3708
4003
|
peer.state === MeshPeerRuntimeState.Serving
|
|
3709
4004
|
) &&
|
|
@@ -3761,7 +4056,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3761
4056
|
const publication = relocationPublication(prepare, envelope);
|
|
3762
4057
|
const participants = envelope.participants.map((participant) => {
|
|
3763
4058
|
const expected = authorities.get(participant.key)!;
|
|
3764
|
-
const membership = actorMembershipTarget(
|
|
4059
|
+
const membership = actorMembershipTarget(
|
|
4060
|
+
envelope.memberships.find((value) => value.actorKey === participant.key),
|
|
4061
|
+
this.options.localDescriptor?.(meshName)
|
|
4062
|
+
);
|
|
3765
4063
|
const entrySpotId =
|
|
3766
4064
|
expected.allocation.objectKind === 'actor' && membership === undefined
|
|
3767
4065
|
? this.options.spotManager()?.entrySpotIdForMesh(meshName)
|
|
@@ -3788,18 +4086,14 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3788
4086
|
objectGeneration: expected.objectGeneration,
|
|
3789
4087
|
targetAttemptGeneration: prepare.targetAttemptGeneration,
|
|
3790
4088
|
coordinatorExpectedStoreVersion: expected.storeVersion.value,
|
|
3791
|
-
...(membership
|
|
3792
|
-
|
|
4089
|
+
...(membership ??
|
|
4090
|
+
(expected.allocation.objectKind === 'actor'
|
|
3793
4091
|
? {
|
|
3794
4092
|
actorSpotId: entrySpotId!,
|
|
3795
4093
|
actorSpotGeneration: prepare.target.nodeGeneration,
|
|
3796
4094
|
actorSpotKind: ZLinkSpotKind.Entry as const
|
|
3797
4095
|
}
|
|
3798
|
-
: {}
|
|
3799
|
-
: {
|
|
3800
|
-
...membership,
|
|
3801
|
-
actorSpotKind: ZLinkSpotKind.User as const
|
|
3802
|
-
})
|
|
4096
|
+
: {}))
|
|
3803
4097
|
}),
|
|
3804
4098
|
membershipMutation: encodeMembershipMutation(envelope.memberships, participant.key)
|
|
3805
4099
|
};
|
|
@@ -3934,6 +4228,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
3934
4228
|
failures.push(error);
|
|
3935
4229
|
}
|
|
3936
4230
|
stage.phase = 'failed';
|
|
4231
|
+
stage.releaseJoinAdmission();
|
|
3937
4232
|
if (failures.length === 1) throw failures[0];
|
|
3938
4233
|
if (failures.length > 1) {
|
|
3939
4234
|
throw new AggregateError(failures, 'Relocation target rollback was incomplete.');
|
|
@@ -4098,6 +4393,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4098
4393
|
|
|
4099
4394
|
/** Maps schema ordinals to the authoritative Location Store inventory. */
|
|
4100
4395
|
private async resolveTargetParticipantInventory(
|
|
4396
|
+
meshName: string,
|
|
4101
4397
|
request: ServiceMaintenanceRelocationPrepare,
|
|
4102
4398
|
envelope: ServiceRelocationEnvelope,
|
|
4103
4399
|
signal?: AbortSignal
|
|
@@ -4136,7 +4432,10 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4136
4432
|
candidate.snapshot.payload,
|
|
4137
4433
|
candidate.snapshot.objectGeneration
|
|
4138
4434
|
);
|
|
4139
|
-
if (
|
|
4435
|
+
if (
|
|
4436
|
+
actor?.spotId === request.object.spotId &&
|
|
4437
|
+
actor.spotGeneration === request.object.objectGeneration
|
|
4438
|
+
) {
|
|
4140
4439
|
entries.push(candidate);
|
|
4141
4440
|
}
|
|
4142
4441
|
}
|
|
@@ -4184,21 +4483,23 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4184
4483
|
authorityOwnerGeneration: snapshot.authorityOwnerGeneration
|
|
4185
4484
|
};
|
|
4186
4485
|
});
|
|
4187
|
-
//
|
|
4188
|
-
//
|
|
4189
|
-
// Location Store still names the source Spot until target cutover. Rebuild
|
|
4190
|
-
// the Actor lane from the already accepted ActorJoin admission, preserving
|
|
4191
|
-
// the target queue boundary for decoded saved-work.
|
|
4486
|
+
// Join recovery uses its accepted target membership. Maintenance Actor
|
|
4487
|
+
// roots restore into the target's startup Entry incarnation (§10.1).
|
|
4192
4488
|
const memberships =
|
|
4193
4489
|
request.object.kind === 'actor'
|
|
4194
|
-
?
|
|
4490
|
+
? actorRootTargetMembership(
|
|
4491
|
+
manager,
|
|
4492
|
+
envelope.aggregateId,
|
|
4493
|
+
participants,
|
|
4494
|
+
this.options.localDescriptor?.(meshName)
|
|
4495
|
+
)
|
|
4195
4496
|
: entries
|
|
4196
4497
|
.filter((entry) => entry.snapshot.allocation.objectKind === 'actor')
|
|
4197
4498
|
.map((entry) => ({
|
|
4198
4499
|
actorKey: entry.key.value,
|
|
4199
4500
|
spotKey: primaryKey.value,
|
|
4200
4501
|
spotObjectGeneration: primary.objectGeneration,
|
|
4201
|
-
membershipEpoch:
|
|
4502
|
+
membershipEpoch: INITIAL_MEMBERSHIP_EPOCH
|
|
4202
4503
|
}));
|
|
4203
4504
|
return {
|
|
4204
4505
|
...envelope,
|
|
@@ -4285,8 +4586,7 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4285
4586
|
meshName: string,
|
|
4286
4587
|
unitKind: 'actor' | 'user_spot' | 'instance_spot',
|
|
4287
4588
|
executionMode: 'spot_wide' | 'per_actor' | undefined,
|
|
4288
|
-
startedAt: number
|
|
4289
|
-
outcome: string
|
|
4589
|
+
startedAt: number
|
|
4290
4590
|
): void {
|
|
4291
4591
|
const seconds = Math.max(0, performance.now() - startedAt) / MILLISECONDS_PER_SECOND;
|
|
4292
4592
|
const attributes = {
|
|
@@ -4294,12 +4594,6 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4294
4594
|
...(executionMode === undefined ? {} : { execution_mode: executionMode })
|
|
4295
4595
|
};
|
|
4296
4596
|
this.options.metrics?.duration(METRIC_NAMES.RelocationInterruption, seconds, attributes);
|
|
4297
|
-
this.options.metrics?.count(METRIC_NAMES.RelocationCompleted, 1, {
|
|
4298
|
-
mesh_name: meshName,
|
|
4299
|
-
object_kind: unitKind,
|
|
4300
|
-
policy: unitKind === 'user_spot' && executionMode === 'per_actor' ? 'recreate' : 'configured',
|
|
4301
|
-
outcome
|
|
4302
|
-
});
|
|
4303
4597
|
if (seconds <= 1) return;
|
|
4304
4598
|
console.warn(
|
|
4305
4599
|
`[zlink.runtime.relocation.changed] ${unitKind} admission interruption ` +
|
|
@@ -4353,30 +4647,30 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4353
4647
|
private async commitSpotMessageFollow(
|
|
4354
4648
|
meshName: string,
|
|
4355
4649
|
seal: ServiceSpotMessageFollowSeal,
|
|
4356
|
-
|
|
4357
|
-
activation: ZLinkSpotActivation
|
|
4358
|
-
target: ZLinkMeshNodeDescriptor
|
|
4650
|
+
committed: ZLinkAuthoritySnapshot | undefined,
|
|
4651
|
+
activation: ZLinkSpotActivation
|
|
4359
4652
|
): Promise<void> {
|
|
4360
4653
|
const node = this.requireMeshNode(meshName);
|
|
4361
4654
|
const commit = node.commitSpotMessageFollowIngress;
|
|
4362
4655
|
if (commit === undefined) {
|
|
4363
4656
|
throw new Error('Spot Message Follow backend support is incomplete.');
|
|
4364
4657
|
}
|
|
4365
|
-
const committed = await requireAuthority(this.requireLocationStore(), authorityKey);
|
|
4366
4658
|
const accepted = await commit.call(
|
|
4367
4659
|
node,
|
|
4368
4660
|
seal,
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4661
|
+
committed === undefined
|
|
4662
|
+
? null
|
|
4663
|
+
: {
|
|
4664
|
+
spot: {
|
|
4665
|
+
spotId: String(activation.spotId),
|
|
4666
|
+
generation: committed.objectGeneration
|
|
4667
|
+
},
|
|
4668
|
+
targetNodeRid: String(committed.allocation.descriptor.rid),
|
|
4669
|
+
targetNodeGeneration: committed.allocation.descriptorLifecycleGeneration,
|
|
4670
|
+
authorityOwnerGeneration: committed.authorityOwnerGeneration,
|
|
4671
|
+
ownerLeaseGeneration: committed.ownerLeaseGeneration,
|
|
4672
|
+
storeVersion: committed.storeVersion.value
|
|
4673
|
+
},
|
|
4380
4674
|
this.relocationLimits().messageFollowDurationMs
|
|
4381
4675
|
);
|
|
4382
4676
|
if (!accepted) {
|
|
@@ -4482,17 +4776,40 @@ export class ZLinkHostServiceRelocationRuntime implements ZLinkActorJoinRelocati
|
|
|
4482
4776
|
meshName: string,
|
|
4483
4777
|
targetNodeRid: RoutingId,
|
|
4484
4778
|
frames: readonly Uint8Array[],
|
|
4485
|
-
timeoutMs: number
|
|
4779
|
+
timeoutMs: number,
|
|
4780
|
+
signal?: AbortSignal,
|
|
4781
|
+
onSubmitted?: (signal: AbortSignal) => Promise<void>
|
|
4486
4782
|
): Promise<readonly Uint8Array[]> {
|
|
4487
4783
|
const node = this.requireMeshNode(meshName);
|
|
4488
4784
|
const request = node.requestInfrastructureControlFrames;
|
|
4489
4785
|
if (request === undefined) {
|
|
4490
4786
|
throw new Error('RouteMesh backend cannot submit bare infrastructure requests.');
|
|
4491
4787
|
}
|
|
4492
|
-
return await request.call(node, targetNodeRid, frames, { timeoutMs });
|
|
4788
|
+
return await request.call(node, targetNodeRid, frames, { timeoutMs, signal, onSubmitted });
|
|
4493
4789
|
}
|
|
4494
4790
|
}
|
|
4495
4791
|
|
|
4792
|
+
function committedSpotRoute<TKind extends ZLinkSpotKind.User | ZLinkSpotKind.Instance>(
|
|
4793
|
+
meshName: string,
|
|
4794
|
+
spotId: RoutingId,
|
|
4795
|
+
spotKind: TKind,
|
|
4796
|
+
targetNodeRid: RoutingId,
|
|
4797
|
+
authority: ZLinkAuthoritySnapshot | undefined
|
|
4798
|
+
): ZLinkSpotRouteTarget & { readonly spotKind: TKind } {
|
|
4799
|
+
return {
|
|
4800
|
+
routerChannelId: meshName,
|
|
4801
|
+
targetNodeRid,
|
|
4802
|
+
spotId,
|
|
4803
|
+
spotKind,
|
|
4804
|
+
targetSpotGeneration: authority?.objectGeneration,
|
|
4805
|
+
targetNodeGeneration: authority?.allocation.descriptorLifecycleGeneration,
|
|
4806
|
+
authorityOwnerGeneration: authority?.authorityOwnerGeneration,
|
|
4807
|
+
targetOwnerId: authority?.ownerId,
|
|
4808
|
+
ownerLeaseGeneration: authority?.ownerLeaseGeneration,
|
|
4809
|
+
authorityStoreVersion: authority?.storeVersion.value
|
|
4810
|
+
};
|
|
4811
|
+
}
|
|
4812
|
+
|
|
4496
4813
|
function canonicalActorJoinRecovery(
|
|
4497
4814
|
participants: readonly ServiceRelocationParticipant[]
|
|
4498
4815
|
): CanonicalActorJoinRecovery | undefined {
|
|
@@ -4514,13 +4831,28 @@ function actorJoinAdmissionIdentity(envelope: ServiceRelocationEnvelope): string
|
|
|
4514
4831
|
);
|
|
4515
4832
|
}
|
|
4516
4833
|
|
|
4517
|
-
function
|
|
4834
|
+
function actorRootTargetMembership(
|
|
4518
4835
|
manager: DefaultZLinkSpotManager | undefined,
|
|
4519
4836
|
relocationId: string,
|
|
4520
|
-
participants: readonly ServiceRelocationParticipant[]
|
|
4837
|
+
participants: readonly ServiceRelocationParticipant[],
|
|
4838
|
+
localDescriptor: ZLinkMeshNodeDescriptor | undefined
|
|
4521
4839
|
): readonly ServiceRelocationMembership[] {
|
|
4522
4840
|
const recovery = canonicalActorJoinRecovery(participants);
|
|
4523
|
-
const
|
|
4841
|
+
const actor = participants.find((value) => value.objectKind === 'actor');
|
|
4842
|
+
if (recovery === undefined) {
|
|
4843
|
+
if (actor === undefined || localDescriptor?.entrySpotId === undefined) {
|
|
4844
|
+
throw new Error(`Actor relocation '${relocationId}' has no target Entry membership.`);
|
|
4845
|
+
}
|
|
4846
|
+
return [
|
|
4847
|
+
{
|
|
4848
|
+
actorKey: actor.key,
|
|
4849
|
+
spotKey: encodeAuthorityKey('user_spot', localDescriptor.entrySpotId).value,
|
|
4850
|
+
spotObjectGeneration: localDescriptor.lifecycleGeneration,
|
|
4851
|
+
membershipEpoch: INITIAL_MEMBERSHIP_EPOCH
|
|
4852
|
+
}
|
|
4853
|
+
];
|
|
4854
|
+
}
|
|
4855
|
+
const admissionId = recovery.request.handoffId;
|
|
4524
4856
|
const admission = (
|
|
4525
4857
|
manager as unknown as
|
|
4526
4858
|
| {
|
|
@@ -4538,7 +4870,6 @@ function actorJoinTargetMembership(
|
|
|
4538
4870
|
}
|
|
4539
4871
|
| undefined
|
|
4540
4872
|
)?.formalRemoteActorAdmissions?.get(admissionId);
|
|
4541
|
-
const actor = participants.find((value) => value.objectKind === 'actor');
|
|
4542
4873
|
if (admission === undefined || actor === undefined) {
|
|
4543
4874
|
throw new Error(`Actor relocation '${relocationId}' has no accepted target membership.`);
|
|
4544
4875
|
}
|
|
@@ -4565,23 +4896,27 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4565
4896
|
): Promise<LocalHidden> {
|
|
4566
4897
|
const identity = decodeAuthorityKey({ value: participant.key } as ZLinkAuthorityKey).globalId;
|
|
4567
4898
|
if (participant.objectKind === 'actor') {
|
|
4899
|
+
if (participant.timers.length !== 0) {
|
|
4900
|
+
throw new Error('Actor relocation cannot restore a Spot timer.');
|
|
4901
|
+
}
|
|
4568
4902
|
const membership = this.envelope.memberships.find(
|
|
4569
4903
|
(value) => value.actorKey === participant.key
|
|
4570
4904
|
);
|
|
4571
4905
|
if (membership === undefined)
|
|
4572
4906
|
throw new Error(`Actor '${identity}' relocation membership is missing.`);
|
|
4573
|
-
const
|
|
4574
|
-
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
const nativeEntrySpot = localDescriptor?.entrySpotId === spotIdentity;
|
|
4907
|
+
const targetMembership = actorMembershipTarget(
|
|
4908
|
+
membership,
|
|
4909
|
+
this.options.localDescriptor?.(this.meshName)
|
|
4910
|
+
)!;
|
|
4578
4911
|
const actor = await this.requireActorManager().prepareRelocationActor(
|
|
4579
4912
|
identity,
|
|
4580
4913
|
participant.stableType,
|
|
4581
4914
|
participant.objectGeneration,
|
|
4582
|
-
participant.authorityOwnerGeneration
|
|
4583
|
-
|
|
4584
|
-
|
|
4915
|
+
participant.authorityOwnerGeneration,
|
|
4916
|
+
targetMembership.actorSpotKind === ZLinkSpotKind.Entry
|
|
4917
|
+
? undefined
|
|
4918
|
+
: (targetMembership.actorSpotId as RoutingId),
|
|
4919
|
+
targetMembership.actorSpotGeneration,
|
|
4585
4920
|
membership.membershipEpoch,
|
|
4586
4921
|
signal
|
|
4587
4922
|
);
|
|
@@ -4590,8 +4925,6 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4590
4925
|
participant,
|
|
4591
4926
|
actor,
|
|
4592
4927
|
initialized: false,
|
|
4593
|
-
restoredTimers: [],
|
|
4594
|
-
replayResults: [],
|
|
4595
4928
|
replayPackets: []
|
|
4596
4929
|
};
|
|
4597
4930
|
}
|
|
@@ -4603,7 +4936,20 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4603
4936
|
registration.implementation as Type<ZLinkSpot | ZLinkInstanceSpot>,
|
|
4604
4937
|
identity as RoutingId,
|
|
4605
4938
|
participant.objectGeneration,
|
|
4606
|
-
participant.authorityOwnerGeneration
|
|
4939
|
+
participant.authorityOwnerGeneration,
|
|
4940
|
+
participant.timers.map((value) => ({
|
|
4941
|
+
name: value.timerId,
|
|
4942
|
+
handlerType: value.handlerType,
|
|
4943
|
+
periodMs: value.intervalMs,
|
|
4944
|
+
overrunPolicy: value.overrunPolicy as never,
|
|
4945
|
+
maxCatchUpTicks: value.maxCatchUpTicks,
|
|
4946
|
+
stopOnUnhandledException: value.stopOnUnhandledException,
|
|
4947
|
+
startedAtUnixMs: value.startedAtUnixMs,
|
|
4948
|
+
deliveryIndex: value.deliveryIndex,
|
|
4949
|
+
lastScheduledIndex: value.lastScheduledIndex,
|
|
4950
|
+
nextDueAtUnixMs: value.dueAtUnixMs,
|
|
4951
|
+
pendingTicks: value.pendingTicks
|
|
4952
|
+
})),
|
|
4607
4953
|
signal
|
|
4608
4954
|
);
|
|
4609
4955
|
return {
|
|
@@ -4611,8 +4957,6 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4611
4957
|
participant,
|
|
4612
4958
|
activation,
|
|
4613
4959
|
initialized: false,
|
|
4614
|
-
restoredTimers: [],
|
|
4615
|
-
replayResults: [],
|
|
4616
4960
|
replayPackets: []
|
|
4617
4961
|
};
|
|
4618
4962
|
}
|
|
@@ -4654,9 +4998,11 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4654
4998
|
): Promise<void> {
|
|
4655
4999
|
for (const membership of memberships) {
|
|
4656
5000
|
const actor = hidden.get(membership.actorKey)?.actor;
|
|
4657
|
-
const
|
|
4658
|
-
|
|
4659
|
-
|
|
5001
|
+
const targetMembership = actorMembershipTarget(
|
|
5002
|
+
membership,
|
|
5003
|
+
this.options.localDescriptor?.(this.meshName)
|
|
5004
|
+
)!;
|
|
5005
|
+
const spotIdentity = targetMembership.actorSpotId;
|
|
4660
5006
|
const spot =
|
|
4661
5007
|
hidden.get(membership.spotKey)?.activation ??
|
|
4662
5008
|
this.requireSpotManager().resolveRelocationActivation(
|
|
@@ -4665,9 +5011,7 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4665
5011
|
);
|
|
4666
5012
|
if (actor === undefined) throw new Error('Relocation membership Actor staging is missing.');
|
|
4667
5013
|
const state = this.requireActorManager().getState(actor.context.actorId)!;
|
|
4668
|
-
|
|
4669
|
-
const isEntryMembership = localDescriptor?.entrySpotId === spotIdentity;
|
|
4670
|
-
if (isEntryMembership) {
|
|
5014
|
+
if (targetMembership.actorSpotKind === ZLinkSpotKind.Entry) {
|
|
4671
5015
|
// Entry Spot identity is published as a Location membership key, but
|
|
4672
5016
|
// it is not a User Spot activation in the Actor runtime. The native
|
|
4673
5017
|
// restore already received the target MeshNode RID as its Entry
|
|
@@ -4675,7 +5019,12 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4675
5019
|
// local Entry Spot path after the target commit.
|
|
4676
5020
|
state.clearJoinedSpot();
|
|
4677
5021
|
} else {
|
|
4678
|
-
state.setJoinedSpot(
|
|
5022
|
+
state.setJoinedSpot(
|
|
5023
|
+
spotIdentity as RoutingId,
|
|
5024
|
+
spot?.spot,
|
|
5025
|
+
membership.membershipEpoch,
|
|
5026
|
+
membership.spotObjectGeneration
|
|
5027
|
+
);
|
|
4679
5028
|
}
|
|
4680
5029
|
spot?.commitActorJoin(actor);
|
|
4681
5030
|
}
|
|
@@ -4798,66 +5147,31 @@ class LocalTargetPort implements ServiceRelocationTargetObjectPort<LocalHidden>
|
|
|
4798
5147
|
}
|
|
4799
5148
|
|
|
4800
5149
|
async openAdmission(hidden: LocalHidden): Promise<void> {
|
|
4801
|
-
if (hidden.actor
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
if (state.spotId === undefined) {
|
|
4810
|
-
const runtime = this.options.spotNodeRuntime();
|
|
4811
|
-
if (runtime === undefined) {
|
|
4812
|
-
throw new Error('Relocated Entry Spot Actor dispatch is unavailable.');
|
|
4813
|
-
}
|
|
4814
|
-
return runtime.dispatchEntryActorPacket(
|
|
4815
|
-
'remote',
|
|
4816
|
-
hidden.actor!.context.actorId,
|
|
4817
|
-
parts,
|
|
4818
|
-
returnResponse,
|
|
4819
|
-
remoteBoundSessionTarget,
|
|
4820
|
-
fallbackActorRef
|
|
4821
|
-
);
|
|
4822
|
-
}
|
|
4823
|
-
return this.requireSpotManager().dispatchRoutedActorPacket(
|
|
4824
|
-
'remote',
|
|
4825
|
-
state.spotId,
|
|
4826
|
-
hidden.actor!.context.actorId,
|
|
4827
|
-
parts,
|
|
4828
|
-
returnResponse,
|
|
4829
|
-
remoteBoundSessionTarget,
|
|
4830
|
-
fallbackActorRef
|
|
4831
|
-
);
|
|
4832
|
-
}
|
|
4833
|
-
);
|
|
4834
|
-
hidden.replayResults.push(...results);
|
|
5150
|
+
if (hidden.actor !== undefined) {
|
|
5151
|
+
this.requireActorManager().publishRelocationActor(hidden.actor.context.actorId);
|
|
5152
|
+
this.requireSpotManager().openActorJoinRelocation(
|
|
5153
|
+
this.meshName,
|
|
5154
|
+
actorJoinAdmissionIdentity(this.envelope),
|
|
5155
|
+
hidden.actor
|
|
5156
|
+
);
|
|
5157
|
+
}
|
|
4835
5158
|
}
|
|
4836
5159
|
|
|
4837
|
-
async
|
|
4838
|
-
if (hidden.
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
|
|
4842
|
-
|
|
4843
|
-
hidden.
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
maxCatchUpTicks: value.maxCatchUpTicks,
|
|
4849
|
-
stopOnUnhandledException: value.stopOnUnhandledException,
|
|
4850
|
-
startedAtUnixMs: value.startedAtUnixMs,
|
|
4851
|
-
deliveryIndex: value.deliveryIndex,
|
|
4852
|
-
lastScheduledIndex: value.lastScheduledIndex,
|
|
4853
|
-
nextDueAtUnixMs: value.dueAtUnixMs,
|
|
4854
|
-
pendingTicks: value.pendingTicks
|
|
4855
|
-
}))
|
|
5160
|
+
async normalize(hidden: LocalHidden): Promise<void> {
|
|
5161
|
+
if (hidden.actor === undefined || hidden.replayPackets.length === 0) return;
|
|
5162
|
+
if (hidden.replayStart === undefined || hidden.onReplayResult === undefined) {
|
|
5163
|
+
throw new Error('Relocated Actor backlog has no target activation owner.');
|
|
5164
|
+
}
|
|
5165
|
+
this.options.actorTransfer.admitRelocationPrefix(
|
|
5166
|
+
hidden.actor,
|
|
5167
|
+
hidden.participant.objectGeneration,
|
|
5168
|
+
hidden.replayPackets,
|
|
5169
|
+
hidden.replayStart,
|
|
5170
|
+
hidden.onReplayResult
|
|
4856
5171
|
);
|
|
5172
|
+
hidden.replayPackets.length = 0;
|
|
4857
5173
|
}
|
|
4858
5174
|
|
|
4859
|
-
async normalize(_hidden: LocalHidden): Promise<void> {}
|
|
4860
|
-
|
|
4861
5175
|
async abort(hidden: LocalHidden): Promise<void> {
|
|
4862
5176
|
if (hidden.actor !== undefined) {
|
|
4863
5177
|
await this.requireActorManager().abortRelocationActor(hidden.actor.context.actorId);
|
|
@@ -4944,14 +5258,22 @@ function rewriteAuthorityApplicationRoute(
|
|
|
4944
5258
|
}
|
|
4945
5259
|
|
|
4946
5260
|
function actorMembershipTarget(
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
):
|
|
4950
|
-
|
|
5261
|
+
membership: ServiceRelocationMembership | undefined,
|
|
5262
|
+
localDescriptor: ZLinkMeshNodeDescriptor | undefined
|
|
5263
|
+
):
|
|
5264
|
+
| {
|
|
5265
|
+
readonly actorSpotId: string;
|
|
5266
|
+
readonly actorSpotGeneration: bigint;
|
|
5267
|
+
readonly actorSpotKind: ZLinkSpotKind.Entry | ZLinkSpotKind.User;
|
|
5268
|
+
}
|
|
5269
|
+
| undefined {
|
|
4951
5270
|
if (membership === undefined) return undefined;
|
|
5271
|
+
const spotId = decodeAuthorityKey({ value: membership.spotKey } as ZLinkAuthorityKey).globalId;
|
|
4952
5272
|
return {
|
|
4953
|
-
actorSpotId:
|
|
4954
|
-
actorSpotGeneration: membership.spotObjectGeneration
|
|
5273
|
+
actorSpotId: spotId,
|
|
5274
|
+
actorSpotGeneration: membership.spotObjectGeneration,
|
|
5275
|
+
actorSpotKind:
|
|
5276
|
+
spotId === localDescriptor?.entrySpotId ? ZLinkSpotKind.Entry : ZLinkSpotKind.User
|
|
4955
5277
|
};
|
|
4956
5278
|
}
|
|
4957
5279
|
|
|
@@ -5079,6 +5401,22 @@ export function decodeQueuedHandoffPacket(
|
|
|
5079
5401
|
...(messageFollowOrigin === undefined ? {} : { messageFollowOrigin })
|
|
5080
5402
|
}
|
|
5081
5403
|
])[0]!;
|
|
5404
|
+
const requestSource =
|
|
5405
|
+
decoded.source === undefined ? undefined : handoffSourceFence(decoded.source);
|
|
5406
|
+
if (
|
|
5407
|
+
messageFollowOrigin !== undefined &&
|
|
5408
|
+
requestSource !== undefined &&
|
|
5409
|
+
routingIdsEqual(messageFollowOrigin.sourceNodeRid, requestSource.nodeRid)
|
|
5410
|
+
) {
|
|
5411
|
+
return {
|
|
5412
|
+
...decoded,
|
|
5413
|
+
index: index as number,
|
|
5414
|
+
messageFollowOrigin: {
|
|
5415
|
+
...messageFollowOrigin,
|
|
5416
|
+
requestSource
|
|
5417
|
+
}
|
|
5418
|
+
};
|
|
5419
|
+
}
|
|
5082
5420
|
return { ...decoded, index: index as number };
|
|
5083
5421
|
}
|
|
5084
5422
|
|
|
@@ -5253,16 +5591,6 @@ export function createServiceRelocationId(
|
|
|
5253
5591
|
}
|
|
5254
5592
|
}
|
|
5255
5593
|
|
|
5256
|
-
function operationWireId(value: string): { readonly high: bigint; readonly low: bigint } {
|
|
5257
|
-
const match = /^(\d+):(\d+)$/.exec(value);
|
|
5258
|
-
if (match === null) throw new Error(`Actor handoff operation '${value}' is not canonical.`);
|
|
5259
|
-
const result = { high: BigInt(match[1]!), low: BigInt(match[2]!) };
|
|
5260
|
-
if (result.high === 0n && result.low === 0n) {
|
|
5261
|
-
throw new Error('Actor handoff operation identity must not be zero.');
|
|
5262
|
-
}
|
|
5263
|
-
return result;
|
|
5264
|
-
}
|
|
5265
|
-
|
|
5266
5594
|
function sameWireId(
|
|
5267
5595
|
left: { readonly high: bigint; readonly low: bigint },
|
|
5268
5596
|
right: { readonly high: bigint; readonly low: bigint }
|
|
@@ -5277,7 +5605,7 @@ function sameCoordinator(
|
|
|
5277
5605
|
return (
|
|
5278
5606
|
left.ownerId === right.ownerId &&
|
|
5279
5607
|
BigInt(left.leaseGeneration) === right.leaseGeneration &&
|
|
5280
|
-
left.nodeRid
|
|
5608
|
+
routingIdsEqual(left.nodeRid, right.nodeRid) &&
|
|
5281
5609
|
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
5282
5610
|
left.expectedAuthorityStoreVersion === right.expectedAuthorityStoreVersion
|
|
5283
5611
|
);
|
|
@@ -5288,7 +5616,7 @@ function sameRelocationTarget(
|
|
|
5288
5616
|
right: ServiceWireRelocationTarget
|
|
5289
5617
|
): boolean {
|
|
5290
5618
|
return (
|
|
5291
|
-
left.nodeRid
|
|
5619
|
+
routingIdsEqual(left.nodeRid, right.nodeRid) &&
|
|
5292
5620
|
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
5293
5621
|
left.ownerId === right.ownerId &&
|
|
5294
5622
|
BigInt(left.ownerLeaseGeneration) === right.ownerLeaseGeneration
|
|
@@ -5341,7 +5669,7 @@ function isSourceOwner(current: ZLinkAuthoritySnapshot, expected: ZLinkAuthority
|
|
|
5341
5669
|
current.allocation.state === expected.allocation.state &&
|
|
5342
5670
|
current.allocation.objectKind === expected.allocation.objectKind &&
|
|
5343
5671
|
current.allocation.stableType === expected.allocation.stableType &&
|
|
5344
|
-
|
|
5672
|
+
routingIdsEqual(current.allocation.descriptor.rid, expected.allocation.descriptor.rid) &&
|
|
5345
5673
|
current.allocation.descriptorLifecycleGeneration ===
|
|
5346
5674
|
expected.allocation.descriptorLifecycleGeneration
|
|
5347
5675
|
);
|
|
@@ -5358,7 +5686,7 @@ function isExactCommittedTarget(
|
|
|
5358
5686
|
current.authorityOwnerGeneration > expected.authorityOwnerGeneration &&
|
|
5359
5687
|
current.ownerId === target.ownerId &&
|
|
5360
5688
|
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
5361
|
-
|
|
5689
|
+
routingIdsEqual(current.allocation.descriptor.rid, target.nodeRid) &&
|
|
5362
5690
|
current.allocation.descriptorLifecycleGeneration === target.nodeGeneration &&
|
|
5363
5691
|
Buffer.from(current.payload).equals(Buffer.from(payload))
|
|
5364
5692
|
);
|
|
@@ -5387,7 +5715,7 @@ function sameRequestSource(
|
|
|
5387
5715
|
return (
|
|
5388
5716
|
left.ownerId === right.ownerId &&
|
|
5389
5717
|
left.leaseGeneration === right.leaseGeneration &&
|
|
5390
|
-
left.nodeRid
|
|
5718
|
+
routingIdsEqual(left.nodeRid, right.nodeRid) &&
|
|
5391
5719
|
left.nodeGeneration === right.nodeGeneration
|
|
5392
5720
|
);
|
|
5393
5721
|
}
|
|
@@ -5398,7 +5726,7 @@ function handoffSourceFence(
|
|
|
5398
5726
|
return {
|
|
5399
5727
|
ownerId: value.ownerId,
|
|
5400
5728
|
leaseGeneration: BigInt(value.ownerLeaseGeneration),
|
|
5401
|
-
nodeRid: value.nodeRid,
|
|
5729
|
+
nodeRid: decodeRoutingId(value.nodeRid, value.nodeRidHex),
|
|
5402
5730
|
nodeGeneration: BigInt(value.nodeGeneration)
|
|
5403
5731
|
};
|
|
5404
5732
|
}
|
|
@@ -5407,14 +5735,34 @@ function maintenanceReplyRelay(
|
|
|
5407
5735
|
stage: LocalStage,
|
|
5408
5736
|
coordinator: ServiceWireRelocationCoordinatorFence,
|
|
5409
5737
|
participantId: bigint,
|
|
5410
|
-
completion: RelocationTerminalCompletion
|
|
5738
|
+
completion: RelocationTerminalCompletion,
|
|
5739
|
+
packet: ZLinkActorHandoffPacket,
|
|
5740
|
+
serializers: Parameters<typeof encodeActorReply>[3],
|
|
5741
|
+
flowEnabled: boolean
|
|
5411
5742
|
): ServiceMaintenanceReplyRelay {
|
|
5412
5743
|
const value = completion.result.ok ? completion.result.response : undefined;
|
|
5744
|
+
const direct = packet.messageFollowOrigin?.requestSource !== undefined;
|
|
5745
|
+
const application =
|
|
5746
|
+
direct && completion.result.ok
|
|
5747
|
+
? decodeApplicationPayloadView(
|
|
5748
|
+
encodeMultipartApplicationPayload(
|
|
5749
|
+
encodeActorReply(
|
|
5750
|
+
Buffer.from(packet.header, 'base64'),
|
|
5751
|
+
ZLinkStreamMessageKind.Response,
|
|
5752
|
+
value,
|
|
5753
|
+
serializers,
|
|
5754
|
+
flowEnabled
|
|
5755
|
+
),
|
|
5756
|
+
SERVICE_FRAMEWORK_MULTIPART_PACKET_NAME,
|
|
5757
|
+
SERVICE_FRAMEWORK_MULTIPART_CONTENT_TYPE
|
|
5758
|
+
)
|
|
5759
|
+
)
|
|
5760
|
+
: undefined;
|
|
5413
5761
|
return {
|
|
5414
5762
|
relocation: relocationWireId(stage.staging.envelope.aggregateId),
|
|
5415
5763
|
targetAttemptGeneration: stage.offer.prepare.targetAttemptGeneration,
|
|
5416
5764
|
coordinator,
|
|
5417
|
-
operation:
|
|
5765
|
+
operation: parseHandoffOperationId(completion.operationId),
|
|
5418
5766
|
replyRouteId: BigInt(completion.source.replyRouteId),
|
|
5419
5767
|
participantId,
|
|
5420
5768
|
sequence: BigInt(completion.index + 1),
|
|
@@ -5422,15 +5770,23 @@ function maintenanceReplyRelay(
|
|
|
5422
5770
|
failureCode: completion.result.ok
|
|
5423
5771
|
? ServiceWireFrameworkErrorCode.none
|
|
5424
5772
|
: ServiceWireFrameworkErrorCode.requestFailed,
|
|
5425
|
-
...(
|
|
5426
|
-
? {
|
|
5427
|
-
: {
|
|
5773
|
+
...(application !== undefined
|
|
5774
|
+
? {
|
|
5428
5775
|
payload: {
|
|
5429
|
-
packetName:
|
|
5430
|
-
contentType:
|
|
5431
|
-
bytes:
|
|
5776
|
+
packetName: application.packetName,
|
|
5777
|
+
contentType: application.contentType,
|
|
5778
|
+
bytes: application.payload
|
|
5432
5779
|
}
|
|
5433
|
-
}
|
|
5780
|
+
}
|
|
5781
|
+
: value === undefined
|
|
5782
|
+
? {}
|
|
5783
|
+
: {
|
|
5784
|
+
payload: {
|
|
5785
|
+
packetName: SERVICE_RELOCATION_REPLY_PACKET_NAME,
|
|
5786
|
+
contentType: ZlinkStreamContentType.Json,
|
|
5787
|
+
bytes: Buffer.from(JSON.stringify(value), 'utf8')
|
|
5788
|
+
}
|
|
5789
|
+
})
|
|
5434
5790
|
};
|
|
5435
5791
|
}
|
|
5436
5792
|
|
|
@@ -5466,6 +5822,14 @@ function wireIdText(value: { readonly high: bigint; readonly low: bigint }): str
|
|
|
5466
5822
|
*/
|
|
5467
5823
|
export class ServiceRelocationDataLostError extends Error {}
|
|
5468
5824
|
|
|
5825
|
+
/** The local source lifecycle ended a pending relocation operation. */
|
|
5826
|
+
class ServiceRelocationRuntimeStoppedError extends Error {
|
|
5827
|
+
constructor() {
|
|
5828
|
+
super('Relocation runtime stopped.');
|
|
5829
|
+
this.name = 'AbortError';
|
|
5830
|
+
}
|
|
5831
|
+
}
|
|
5832
|
+
|
|
5469
5833
|
/** Classifies relocation failure with the error model §2.1 mapping. */
|
|
5470
5834
|
export function relocationFailedFailureCode(error: unknown): number {
|
|
5471
5835
|
if (error instanceof ServiceRelocationDataLostError) {
|
|
@@ -5600,7 +5964,7 @@ function validateTargetOneWayControl(
|
|
|
5600
5964
|
): void {
|
|
5601
5965
|
const prepare = stage.offer.prepare;
|
|
5602
5966
|
if (
|
|
5603
|
-
|
|
5967
|
+
!routingIdsEqual(sourceNodeRid, stage.offer.authenticatedSourceNodeRid) ||
|
|
5604
5968
|
!sameWireId(request.relocation, prepare.relocation) ||
|
|
5605
5969
|
request.targetAttemptGeneration !== prepare.targetAttemptGeneration ||
|
|
5606
5970
|
!sameCoordinator(request.coordinator, prepare.coordinator) ||
|
|
@@ -5678,12 +6042,7 @@ function canonicalQueuedFrozenRecord(
|
|
|
5678
6042
|
const source =
|
|
5679
6043
|
packet.source === undefined
|
|
5680
6044
|
? coordinatorSource(coordinator)
|
|
5681
|
-
:
|
|
5682
|
-
ownerId: packet.source.ownerId,
|
|
5683
|
-
leaseGeneration: BigInt(packet.source.ownerLeaseGeneration),
|
|
5684
|
-
nodeRid: packet.source.nodeRid,
|
|
5685
|
-
nodeGeneration: BigInt(packet.source.nodeGeneration)
|
|
5686
|
-
};
|
|
6045
|
+
: handoffSourceFence(packet.source);
|
|
5687
6046
|
const operationId = parseHandoffOperationId(packet.messageFollowContext.operationId);
|
|
5688
6047
|
if (packet.returnResponse && packet.source === undefined) {
|
|
5689
6048
|
throw new Error('Captured Actor request has no exact request-source fence.');
|
|
@@ -5705,7 +6064,7 @@ function canonicalQueuedFrozenRecord(
|
|
|
5705
6064
|
objectGeneration: participant.objectGeneration,
|
|
5706
6065
|
nodeRid: target.nodeRid,
|
|
5707
6066
|
nodeGeneration: target.nodeGeneration,
|
|
5708
|
-
authorityOwnerGeneration: participant.authorityOwnerGeneration
|
|
6067
|
+
authorityOwnerGeneration: participant.authorityOwnerGeneration,
|
|
5709
6068
|
ownerLeaseGeneration: target.ownerLeaseGeneration
|
|
5710
6069
|
},
|
|
5711
6070
|
operationId,
|
|
@@ -5821,17 +6180,13 @@ function sessionRelocationRouteSubmitKey(
|
|
|
5821
6180
|
request: ServiceSessionRelocationRoute,
|
|
5822
6181
|
targetNodeRid: RoutingId
|
|
5823
6182
|
): string {
|
|
5824
|
-
return `${sessionRelocationIdentityKey(request)}:${
|
|
6183
|
+
return `${sessionRelocationIdentityKey(request)}:${encodeRoutingIdStorageHex(targetNodeRid)}`;
|
|
5825
6184
|
}
|
|
5826
6185
|
|
|
5827
6186
|
function sessionRelocationResponseKey(response: ServiceSessionRelocationSealed): string {
|
|
5828
6187
|
return sessionRelocationIdentityKey(response);
|
|
5829
6188
|
}
|
|
5830
6189
|
|
|
5831
|
-
function sessionRelocationResponseFingerprint(response: ServiceSessionRelocationSealed): string {
|
|
5832
|
-
return encodeSessionRelocationSealed(response).toString('base64');
|
|
5833
|
-
}
|
|
5834
|
-
|
|
5835
6190
|
function sessionRelocationIdentityKey(
|
|
5836
6191
|
value:
|
|
5837
6192
|
ServiceSessionRelocationSeal | ServiceSessionRelocationSealed | ServiceSessionRelocationRoute
|
|
@@ -5846,16 +6201,16 @@ function validateSessionRelocationResponse(
|
|
|
5846
6201
|
const sameCoordinator =
|
|
5847
6202
|
request.coordinator.ownerId === response.coordinator.ownerId &&
|
|
5848
6203
|
request.coordinator.leaseGeneration === response.coordinator.leaseGeneration &&
|
|
5849
|
-
request.coordinator.nodeRid
|
|
6204
|
+
routingIdsEqual(request.coordinator.nodeRid, response.coordinator.nodeRid) &&
|
|
5850
6205
|
request.coordinator.nodeGeneration === response.coordinator.nodeGeneration &&
|
|
5851
6206
|
request.coordinator.expectedAuthorityStoreVersion ===
|
|
5852
6207
|
response.coordinator.expectedAuthorityStoreVersion;
|
|
5853
6208
|
const sameSession =
|
|
5854
|
-
request.session.sessionOwnerNodeRid
|
|
6209
|
+
routingIdsEqual(request.session.sessionOwnerNodeRid, response.session.sessionOwnerNodeRid) &&
|
|
5855
6210
|
request.session.sessionOwnerNodeGeneration === response.session.sessionOwnerNodeGeneration &&
|
|
5856
6211
|
request.session.sessionOwnerId === response.session.sessionOwnerId &&
|
|
5857
6212
|
request.session.sessionOwnerLeaseGeneration === response.session.sessionOwnerLeaseGeneration &&
|
|
5858
|
-
request.session.sessionRid
|
|
6213
|
+
routingIdsEqual(request.session.sessionRid, response.session.sessionRid) &&
|
|
5859
6214
|
request.session.bindingGeneration === response.session.bindingGeneration;
|
|
5860
6215
|
if (
|
|
5861
6216
|
request.relocation.high !== response.relocation.high ||
|
|
@@ -5868,7 +6223,7 @@ function validateSessionRelocationResponse(
|
|
|
5868
6223
|
if (
|
|
5869
6224
|
request.actor.actor.actorId !== response.actor.actor.actorId ||
|
|
5870
6225
|
request.actor.actor.generation !== response.actor.actor.generation ||
|
|
5871
|
-
request.actor.actor.nodeRid
|
|
6226
|
+
!routingIdsEqual(request.actor.actor.nodeRid, response.actor.actor.nodeRid) ||
|
|
5872
6227
|
request.actor.targetNodeGeneration !== response.actor.targetNodeGeneration ||
|
|
5873
6228
|
request.actor.authorityOwnerGeneration !== response.actor.authorityOwnerGeneration ||
|
|
5874
6229
|
request.actor.ownerLeaseGeneration !== response.actor.ownerLeaseGeneration
|
|
@@ -5891,27 +6246,20 @@ function replyRelayIdentityKey(value: {
|
|
|
5891
6246
|
value.relocation.low.toString(),
|
|
5892
6247
|
value.coordinator.ownerId,
|
|
5893
6248
|
value.coordinator.leaseGeneration.toString(),
|
|
5894
|
-
value.coordinator.nodeRid,
|
|
6249
|
+
encodeRoutingIdStorageHex(value.coordinator.nodeRid),
|
|
5895
6250
|
value.coordinator.nodeGeneration.toString(),
|
|
5896
6251
|
value.coordinator.expectedAuthorityStoreVersion
|
|
5897
6252
|
]);
|
|
5898
6253
|
}
|
|
5899
6254
|
|
|
5900
6255
|
function replyRelayPendingKey(
|
|
5901
|
-
ackTargetNodeRid:
|
|
6256
|
+
ackTargetNodeRid: RoutingId,
|
|
5902
6257
|
value: Parameters<typeof replyRelayIdentityKey>[0]
|
|
5903
6258
|
): string {
|
|
5904
|
-
return JSON.stringify([
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
return (
|
|
5909
|
-
payload.byteLength >= 5 &&
|
|
5910
|
-
payload[0] === 0x5a &&
|
|
5911
|
-
payload[1] === 0x4d &&
|
|
5912
|
-
payload[2] === 1 &&
|
|
5913
|
-
payload[3] === command
|
|
5914
|
-
);
|
|
6259
|
+
return JSON.stringify([
|
|
6260
|
+
encodeRoutingIdStorageHex(ackTargetNodeRid),
|
|
6261
|
+
replyRelayIdentityKey(value)
|
|
6262
|
+
]);
|
|
5915
6263
|
}
|
|
5916
6264
|
|
|
5917
6265
|
export function encodeHandoffQueuedMessages(
|
|
@@ -6000,97 +6348,58 @@ function toServiceTimer(
|
|
|
6000
6348
|
};
|
|
6001
6349
|
}
|
|
6002
6350
|
|
|
6003
|
-
const PREPARE_SIDEBAND_MAGIC = 'ZLNI';
|
|
6004
|
-
const PREPARE_SIDEBAND_VERSION = 1;
|
|
6005
|
-
const PREPARE_SIDEBAND_HEADER_BYTES = 11;
|
|
6006
|
-
const PREPARE_SIDEBAND_VERSION_OFFSET = 4;
|
|
6007
|
-
const PREPARE_SIDEBAND_KEY_LENGTH_OFFSET = 5;
|
|
6008
|
-
const PREPARE_SIDEBAND_PAYLOAD_LENGTH_OFFSET = 7;
|
|
6009
6351
|
const ACTOR_SESSION_JOURNAL_VERSION = 1;
|
|
6010
6352
|
const SERVICE_RELOCATION_REPLY_PACKET_NAME = 'zlink.relocation.reply';
|
|
6011
6353
|
|
|
6012
|
-
|
|
6013
|
-
* NODE-INTERNAL Prepare sideband. Frame zero remains the exact canonical
|
|
6014
|
-
* command-40 schema record; each following ZLNI frame carries one sealed
|
|
6015
|
-
* actor-session journal keyed by its canonical participant authority key.
|
|
6016
|
-
* Sessionless relocation remains a single-frame, wire-identical Prepare.
|
|
6017
|
-
*/
|
|
6018
|
-
function encodePrepareSideband(envelope: ServiceRelocationEnvelope): readonly Buffer[] {
|
|
6019
|
-
return envelope.participants
|
|
6020
|
-
.filter((participant) => participant.boundSessionState.byteLength !== 0)
|
|
6021
|
-
.map((participant) => {
|
|
6022
|
-
const key = Buffer.from(participant.key, 'utf8');
|
|
6023
|
-
const payload = Buffer.from(participant.boundSessionState);
|
|
6024
|
-
if (key.byteLength === 0 || key.byteLength > 0xffff || payload.byteLength > 0xffff_ffff) {
|
|
6025
|
-
throw new TypeError('Bound-session relocation sideband is out of bounds.');
|
|
6026
|
-
}
|
|
6027
|
-
const header = Buffer.allocUnsafe(PREPARE_SIDEBAND_HEADER_BYTES);
|
|
6028
|
-
header.write(PREPARE_SIDEBAND_MAGIC, 0, 'ascii');
|
|
6029
|
-
header[PREPARE_SIDEBAND_VERSION_OFFSET] = PREPARE_SIDEBAND_VERSION;
|
|
6030
|
-
header.writeUInt16BE(key.byteLength, PREPARE_SIDEBAND_KEY_LENGTH_OFFSET);
|
|
6031
|
-
header.writeUInt32BE(payload.byteLength, PREPARE_SIDEBAND_PAYLOAD_LENGTH_OFFSET);
|
|
6032
|
-
return Buffer.concat([header, key, payload]);
|
|
6033
|
-
});
|
|
6034
|
-
}
|
|
6035
|
-
|
|
6036
|
-
function decodePrepareSideband(frames: readonly Uint8Array[]):
|
|
6037
|
-
| readonly {
|
|
6038
|
-
readonly participantKey: string;
|
|
6039
|
-
readonly payload: Buffer;
|
|
6040
|
-
}[]
|
|
6041
|
-
| undefined {
|
|
6042
|
-
const result: Array<{ readonly participantKey: string; readonly payload: Buffer }> = [];
|
|
6043
|
-
const keys = new Set<string>();
|
|
6044
|
-
for (const frame of frames) {
|
|
6045
|
-
const bytes = Buffer.from(frame);
|
|
6046
|
-
if (
|
|
6047
|
-
bytes.byteLength < PREPARE_SIDEBAND_HEADER_BYTES ||
|
|
6048
|
-
bytes.subarray(0, PREPARE_SIDEBAND_VERSION_OFFSET).toString('ascii') !==
|
|
6049
|
-
PREPARE_SIDEBAND_MAGIC ||
|
|
6050
|
-
bytes[PREPARE_SIDEBAND_VERSION_OFFSET] !== PREPARE_SIDEBAND_VERSION
|
|
6051
|
-
) {
|
|
6052
|
-
return undefined;
|
|
6053
|
-
}
|
|
6054
|
-
const keyLength = bytes.readUInt16BE(PREPARE_SIDEBAND_KEY_LENGTH_OFFSET);
|
|
6055
|
-
const payloadLength = bytes.readUInt32BE(PREPARE_SIDEBAND_PAYLOAD_LENGTH_OFFSET);
|
|
6056
|
-
if (
|
|
6057
|
-
PREPARE_SIDEBAND_HEADER_BYTES + keyLength + payloadLength !== bytes.byteLength ||
|
|
6058
|
-
keyLength === 0
|
|
6059
|
-
)
|
|
6060
|
-
return undefined;
|
|
6061
|
-
const participantKey = bytes
|
|
6062
|
-
.subarray(PREPARE_SIDEBAND_HEADER_BYTES, PREPARE_SIDEBAND_HEADER_BYTES + keyLength)
|
|
6063
|
-
.toString('utf8');
|
|
6064
|
-
if (keys.has(participantKey)) return undefined;
|
|
6065
|
-
keys.add(participantKey);
|
|
6066
|
-
result.push({
|
|
6067
|
-
participantKey,
|
|
6068
|
-
payload: Buffer.from(bytes.subarray(PREPARE_SIDEBAND_HEADER_BYTES + keyLength))
|
|
6069
|
-
});
|
|
6070
|
-
}
|
|
6071
|
-
return result;
|
|
6072
|
-
}
|
|
6073
|
-
|
|
6074
|
-
function attachPrepareBoundSessions(
|
|
6354
|
+
function attachCanonicalBoundSessions(
|
|
6075
6355
|
envelope: ServiceRelocationEnvelope,
|
|
6076
|
-
|
|
6356
|
+
request: ServiceMaintenanceRelocationPrepare,
|
|
6357
|
+
meshName: string
|
|
6077
6358
|
): ServiceRelocationEnvelope {
|
|
6078
|
-
|
|
6079
|
-
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6090
|
-
|
|
6091
|
-
|
|
6092
|
-
|
|
6093
|
-
|
|
6359
|
+
return {
|
|
6360
|
+
...envelope,
|
|
6361
|
+
participants: envelope.participants.map((participant) => {
|
|
6362
|
+
const binding = participant.sessionBinding;
|
|
6363
|
+
if (binding === undefined) return participant;
|
|
6364
|
+
if (participant.objectKind !== 'actor')
|
|
6365
|
+
throw new TypeError('Session binding names a non-Actor participant.');
|
|
6366
|
+
const sessionNodeRid = decodeRoutingId(
|
|
6367
|
+
'',
|
|
6368
|
+
Buffer.from(binding.sessionOwnerNodeRid).toString('hex')
|
|
6369
|
+
);
|
|
6370
|
+
const sessionRid = decodeRoutingId('', Buffer.from(binding.sessionRid).toString('hex'));
|
|
6371
|
+
const session = {
|
|
6372
|
+
sessionOwnerNodeRid: sessionNodeRid as string,
|
|
6373
|
+
sessionOwnerNodeGeneration: binding.sessionOwnerNodeGeneration,
|
|
6374
|
+
sessionOwnerId: binding.sessionOwnerId,
|
|
6375
|
+
sessionOwnerLeaseGeneration: binding.sessionOwnerLeaseGeneration,
|
|
6376
|
+
sessionRid: sessionRid as string,
|
|
6377
|
+
bindingGeneration: binding.bindingGeneration
|
|
6378
|
+
};
|
|
6379
|
+
const target: ZLinkRemoteBoundSessionTarget = {
|
|
6380
|
+
routerChannelId: meshName,
|
|
6381
|
+
targetNodeRid: sessionNodeRid,
|
|
6382
|
+
spotId:
|
|
6383
|
+
request.object.kind === 'actor'
|
|
6384
|
+
? ((participant.rootSpotId ?? '') as RoutingId)
|
|
6385
|
+
: (request.object.spotId as RoutingId),
|
|
6386
|
+
sessionNodeRid,
|
|
6387
|
+
sessionRid,
|
|
6388
|
+
sessionOwnerNodeGeneration: binding.sessionOwnerNodeGeneration,
|
|
6389
|
+
sessionOwnerId: binding.sessionOwnerId,
|
|
6390
|
+
sessionOwnerLeaseGeneration: binding.sessionOwnerLeaseGeneration,
|
|
6391
|
+
bindingGeneration: binding.bindingGeneration,
|
|
6392
|
+
previousAuthorityOwnerGeneration: participant.authorityOwnerGeneration,
|
|
6393
|
+
relocationSealId: envelope.aggregateId,
|
|
6394
|
+
serviceWireRelocation: {
|
|
6395
|
+
relocation: request.relocation,
|
|
6396
|
+
coordinator: request.coordinator,
|
|
6397
|
+
session
|
|
6398
|
+
}
|
|
6399
|
+
};
|
|
6400
|
+
return { ...participant, boundSessionState: encodeActorSession(target) };
|
|
6401
|
+
})
|
|
6402
|
+
};
|
|
6094
6403
|
}
|
|
6095
6404
|
|
|
6096
6405
|
function encodeActorSession(target: ZLinkRemoteBoundSessionTarget | undefined): Buffer {
|
|
@@ -6100,10 +6409,17 @@ function encodeActorSession(target: ZLinkRemoteBoundSessionTarget | undefined):
|
|
|
6100
6409
|
version: ACTOR_SESSION_JOURNAL_VERSION,
|
|
6101
6410
|
routerChannelId: target.routerChannelId,
|
|
6102
6411
|
targetNodeRid: String(target.targetNodeRid),
|
|
6412
|
+
targetNodeRidHex: encodeRoutingIdStorageHex(target.targetNodeRid),
|
|
6103
6413
|
spotId: String(target.spotId),
|
|
6104
6414
|
sessionNodeRid:
|
|
6105
6415
|
target.sessionNodeRid === undefined ? undefined : String(target.sessionNodeRid),
|
|
6416
|
+
sessionNodeRidHex:
|
|
6417
|
+
target.sessionNodeRid === undefined
|
|
6418
|
+
? undefined
|
|
6419
|
+
: encodeRoutingIdStorageHex(target.sessionNodeRid),
|
|
6106
6420
|
sessionRid: target.sessionRid === undefined ? undefined : String(target.sessionRid),
|
|
6421
|
+
sessionRidHex:
|
|
6422
|
+
target.sessionRid === undefined ? undefined : encodeRoutingIdStorageHex(target.sessionRid),
|
|
6107
6423
|
sessionOwnerNodeGeneration: target.sessionOwnerNodeGeneration?.toString(),
|
|
6108
6424
|
sessionOwnerId: target.sessionOwnerId,
|
|
6109
6425
|
sessionOwnerLeaseGeneration: target.sessionOwnerLeaseGeneration?.toString(),
|
|
@@ -6120,18 +6436,27 @@ function encodeActorSession(target: ZLinkRemoteBoundSessionTarget | undefined):
|
|
|
6120
6436
|
coordinatorOwnerId: target.serviceWireRelocation.coordinator.ownerId,
|
|
6121
6437
|
coordinatorLeaseGeneration:
|
|
6122
6438
|
target.serviceWireRelocation.coordinator.leaseGeneration.toString(),
|
|
6123
|
-
coordinatorNodeRid: target.serviceWireRelocation.coordinator.nodeRid,
|
|
6439
|
+
coordinatorNodeRid: String(target.serviceWireRelocation.coordinator.nodeRid),
|
|
6440
|
+
coordinatorNodeRidHex: encodeRoutingIdStorageHex(
|
|
6441
|
+
target.serviceWireRelocation.coordinator.nodeRid
|
|
6442
|
+
),
|
|
6124
6443
|
coordinatorNodeGeneration:
|
|
6125
6444
|
target.serviceWireRelocation.coordinator.nodeGeneration.toString(),
|
|
6126
6445
|
expectedAuthorityStoreVersion:
|
|
6127
6446
|
target.serviceWireRelocation.coordinator.expectedAuthorityStoreVersion,
|
|
6128
|
-
sessionOwnerNodeRid: target.serviceWireRelocation.session.sessionOwnerNodeRid,
|
|
6447
|
+
sessionOwnerNodeRid: String(target.serviceWireRelocation.session.sessionOwnerNodeRid),
|
|
6448
|
+
sessionOwnerNodeRidHex: encodeRoutingIdStorageHex(
|
|
6449
|
+
target.serviceWireRelocation.session.sessionOwnerNodeRid as RoutingId
|
|
6450
|
+
),
|
|
6129
6451
|
sessionOwnerNodeGeneration:
|
|
6130
6452
|
target.serviceWireRelocation.session.sessionOwnerNodeGeneration.toString(),
|
|
6131
6453
|
sessionOwnerId: target.serviceWireRelocation.session.sessionOwnerId,
|
|
6132
6454
|
sessionOwnerLeaseGeneration:
|
|
6133
6455
|
target.serviceWireRelocation.session.sessionOwnerLeaseGeneration.toString(),
|
|
6134
|
-
sessionRid: target.serviceWireRelocation.session.sessionRid,
|
|
6456
|
+
sessionRid: String(target.serviceWireRelocation.session.sessionRid),
|
|
6457
|
+
sessionRidHex: encodeRoutingIdStorageHex(
|
|
6458
|
+
target.serviceWireRelocation.session.sessionRid as RoutingId
|
|
6459
|
+
),
|
|
6135
6460
|
bindingGeneration: target.serviceWireRelocation.session.bindingGeneration.toString()
|
|
6136
6461
|
}
|
|
6137
6462
|
}),
|
|
@@ -6155,12 +6480,14 @@ function decodeActorSession(payload: Uint8Array): ZLinkRemoteBoundSessionTarget
|
|
|
6155
6480
|
const serviceWire = decodeActorSessionServiceWireFence(value.serviceWireRelocation);
|
|
6156
6481
|
return {
|
|
6157
6482
|
routerChannelId: value.routerChannelId,
|
|
6158
|
-
targetNodeRid: value.targetNodeRid
|
|
6483
|
+
targetNodeRid: decodeRoutingId(value.targetNodeRid, value.targetNodeRidHex),
|
|
6159
6484
|
spotId: value.spotId as RoutingId,
|
|
6160
6485
|
...(typeof value.sessionNodeRid === 'string'
|
|
6161
|
-
? { sessionNodeRid: value.sessionNodeRid
|
|
6486
|
+
? { sessionNodeRid: decodeRoutingId(value.sessionNodeRid, value.sessionNodeRidHex) }
|
|
6487
|
+
: {}),
|
|
6488
|
+
...(typeof value.sessionRid === 'string'
|
|
6489
|
+
? { sessionRid: decodeRoutingId(value.sessionRid, value.sessionRidHex) }
|
|
6162
6490
|
: {}),
|
|
6163
|
-
...(typeof value.sessionRid === 'string' ? { sessionRid: value.sessionRid as RoutingId } : {}),
|
|
6164
6491
|
...(optionalBigInt('sessionOwnerNodeGeneration') === undefined
|
|
6165
6492
|
? {}
|
|
6166
6493
|
: { sessionOwnerNodeGeneration: optionalBigInt('sessionOwnerNodeGeneration') }),
|
|
@@ -6224,16 +6551,22 @@ function decodeActorSessionServiceWireFence(
|
|
|
6224
6551
|
coordinator: {
|
|
6225
6552
|
ownerId: text('coordinatorOwnerId'),
|
|
6226
6553
|
leaseGeneration: positive('coordinatorLeaseGeneration'),
|
|
6227
|
-
nodeRid: text('coordinatorNodeRid'),
|
|
6554
|
+
nodeRid: decodeRoutingId(text('coordinatorNodeRid'), value.coordinatorNodeRidHex),
|
|
6228
6555
|
nodeGeneration: positive('coordinatorNodeGeneration'),
|
|
6229
6556
|
expectedAuthorityStoreVersion: text('expectedAuthorityStoreVersion')
|
|
6230
6557
|
},
|
|
6231
6558
|
session: {
|
|
6232
|
-
sessionOwnerNodeRid:
|
|
6559
|
+
sessionOwnerNodeRid: decodeRoutingId(
|
|
6560
|
+
typeof value.sessionOwnerNodeRid === 'string' ? value.sessionOwnerNodeRid : '',
|
|
6561
|
+
value.sessionOwnerNodeRidHex
|
|
6562
|
+
) as string,
|
|
6233
6563
|
sessionOwnerNodeGeneration: positive('sessionOwnerNodeGeneration'),
|
|
6234
6564
|
sessionOwnerId: text('sessionOwnerId'),
|
|
6235
6565
|
sessionOwnerLeaseGeneration: positive('sessionOwnerLeaseGeneration'),
|
|
6236
|
-
sessionRid:
|
|
6566
|
+
sessionRid: decodeRoutingId(
|
|
6567
|
+
typeof value.sessionRid === 'string' ? value.sessionRid : '',
|
|
6568
|
+
value.sessionRidHex
|
|
6569
|
+
) as string,
|
|
6237
6570
|
bindingGeneration: positive('bindingGeneration')
|
|
6238
6571
|
}
|
|
6239
6572
|
};
|