@zlink-systems/framework 0.30.0 → 0.30.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +150 -68
- package/dist/.tsbuildinfo +1 -1
- package/dist/contracts/Actors/ZLinkActorClient.d.ts +2 -1
- package/dist/contracts/Channels/Calls.d.ts +2 -1
- package/dist/contracts/Common/ZLinkEncodedPayload.d.ts +6 -0
- package/dist/contracts/Common/ZLinkEncodedPayload.js +18 -3
- package/dist/contracts/Common/ZLinkMessageMetadata.js +11 -5
- package/dist/contracts/Common/encoded-payload-storage.d.ts +1 -1
- package/dist/contracts/Common/encoded-payload-storage.js +5 -2
- package/dist/contracts/Configuration/Builders.d.ts +1 -1
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.d.ts +2 -1
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.js +5 -1
- package/dist/contracts/Configuration/RegistrationBuilders.js +2 -2
- package/dist/contracts/Configuration/RegistrationTypes.d.ts +1 -0
- package/dist/contracts/Dispatch/ZLinkDispatchOptions.d.ts +2 -2
- package/dist/contracts/Handlers/Attributes.d.ts +5 -13
- package/dist/contracts/Handlers/Attributes.js +7 -11
- package/dist/contracts/Handlers/JsonContract.d.ts +2 -46
- package/dist/contracts/Handlers/JsonContract.js +4 -153
- package/dist/contracts/Locations/Rows.d.ts +2 -0
- package/dist/contracts/Spots/Contracts.d.ts +6 -1
- package/dist/contracts/Spots/ZLinkSpot.d.ts +3 -3
- package/dist/nest-integration.d.ts +1 -1
- package/dist/nest-integration.js +2 -1
- package/dist/runtime/abort.d.ts +5 -0
- package/dist/runtime/abort.js +10 -0
- package/dist/runtime/actors/actor-authority-publication.js +3 -2
- package/dist/runtime/actors/actor-client.d.ts +1 -2
- package/dist/runtime/actors/actor-client.js +52 -57
- package/dist/runtime/actors/actor-context.d.ts +2 -0
- package/dist/runtime/actors/actor-context.js +46 -9
- package/dist/runtime/actors/actor-handoff.d.ts +8 -4
- package/dist/runtime/actors/actor-handoff.js +219 -92
- package/dist/runtime/actors/actor-join-relocation.d.ts +3 -0
- package/dist/runtime/actors/actor-local-native-join.js +13 -12
- package/dist/runtime/actors/actor-mailbox.js +27 -1
- package/dist/runtime/actors/actor-remote-joiner.js +3 -1
- package/dist/runtime/actors/actor-remote-wire.js +15 -0
- package/dist/runtime/actors/actor-reply.d.ts +4 -0
- package/dist/runtime/actors/actor-reply.js +29 -0
- package/dist/runtime/actors/actor-routed-join-transport.d.ts +6 -2
- package/dist/runtime/actors/actor-routed-json-request.d.ts +2 -0
- package/dist/runtime/actors/actor-runtime-contracts.d.ts +2 -0
- package/dist/runtime/actors/actor-runtime-state.js +2 -20
- package/dist/runtime/actors/actor-source-transfer.d.ts +1 -17
- package/dist/runtime/actors/index.d.ts +8 -3
- package/dist/runtime/actors/index.js +55 -27
- package/dist/runtime/actors/session-disconnected-wire.d.ts +8 -0
- package/dist/runtime/actors/session-disconnected-wire.js +48 -0
- package/dist/runtime/actors/spot-actor-dispatch.js +2 -2
- package/dist/runtime/admission.d.ts +9 -1
- package/dist/runtime/admission.js +56 -23
- package/dist/runtime/application-jobs/application-ingress-record-owner.d.ts +6 -8
- package/dist/runtime/application-jobs/application-ingress-record-owner.js +39 -22
- package/dist/runtime/application-jobs/application-job-queue-scope.d.ts +15 -1
- package/dist/runtime/application-jobs/application-job-queue-scope.js +61 -20
- package/dist/runtime/application-jobs/contracts.d.ts +12 -1
- package/dist/runtime/backend/contracts/index.d.ts +37 -12
- package/dist/runtime/backend/mesh-actor-session-node-adapter.js +2 -2
- package/dist/runtime/backend/mesh-completion-table.d.ts +9 -1
- package/dist/runtime/backend/mesh-completion-table.js +154 -49
- package/dist/runtime/backend/mesh-dispatch-pump.d.ts +28 -1
- package/dist/runtime/backend/mesh-dispatch-pump.js +208 -101
- package/dist/runtime/backend/node/node-backend-adapter-support.d.ts +2 -2
- package/dist/runtime/backend/node/node-backend-adapter-support.js +16 -9
- package/dist/runtime/backend/node/node-dealer-channel-client-transport.d.ts +1 -1
- package/dist/runtime/backend/node/node-dealer-channel-client-transport.js +5 -2
- package/dist/runtime/backend/node/node-raw-binding-port.js +124 -72
- package/dist/runtime/backend/node/node-raw-mesh-backend.d.ts +46 -13
- package/dist/runtime/backend/node/node-raw-mesh-backend.js +181 -164
- package/dist/runtime/backend/node/node-socket-backend-adapter.d.ts +1 -1
- package/dist/runtime/backend/node/node-socket-backend-adapter.js +11 -11
- package/dist/runtime/backend/raw-binding-port.d.ts +2 -2
- package/dist/runtime/backend/runtime-message.d.ts +1 -0
- package/dist/runtime/backend/runtime-message.js +5 -0
- package/dist/runtime/backend/runtime-values.d.ts +1 -0
- package/dist/runtime/backend/runtime-values.js +2 -1
- package/dist/runtime/channels/channel-clients.d.ts +2 -0
- package/dist/runtime/channels/channel-clients.js +75 -23
- package/dist/runtime/channels/channel-dispatch-pipeline.d.ts +25 -5
- package/dist/runtime/channels/channel-dispatch-pipeline.js +111 -56
- package/dist/runtime/channels/channel-dispatch-services.d.ts +1 -0
- package/dist/runtime/channels/channel-dispatch-services.js +45 -0
- package/dist/runtime/channels/channel-dispatchers.d.ts +4 -3
- package/dist/runtime/channels/channel-dispatchers.js +62 -54
- package/dist/runtime/channels/channel-envelope-inspection.d.ts +1 -2
- package/dist/runtime/channels/channel-envelope-inspection.js +1 -16
- package/dist/runtime/channels/channel-envelope.d.ts +2 -0
- package/dist/runtime/channels/channel-envelope.js +146 -28
- package/dist/runtime/channels/channel-outbound-operations.d.ts +2 -2
- package/dist/runtime/channels/channel-outbound-operations.js +8 -8
- package/dist/runtime/channels/channel-runtime-lifecycle.d.ts +4 -3
- package/dist/runtime/channels/channel-runtime-lifecycle.js +19 -24
- package/dist/runtime/channels/channel-runtime-manager.d.ts +9 -8
- package/dist/runtime/channels/channel-runtime-manager.js +16 -16
- package/dist/runtime/channels/channel-transports.d.ts +18 -16
- package/dist/runtime/channels/channel-transports.js +77 -58
- package/dist/runtime/channels/client-server-location-runtime.js +5 -4
- package/dist/runtime/channels/dispatch-error-reporter.d.ts +2 -1
- package/dist/runtime/channels/dispatch-error-reporter.js +2 -1
- package/dist/runtime/channels/fanout-location-runtime.d.ts +1 -1
- package/dist/runtime/channels/fanout-location-runtime.js +3 -3
- package/dist/runtime/channels/local-spot-route-transport.d.ts +1 -1
- package/dist/runtime/channels/local-spot-route-transport.js +2 -1
- package/dist/runtime/channels/router-socket-spot-route-transport.d.ts +2 -2
- package/dist/runtime/channels/router-socket-spot-route-transport.js +4 -4
- package/dist/runtime/channels/source-spot-router.d.ts +2 -2
- package/dist/runtime/channels/source-spot-router.js +4 -4
- package/dist/runtime/channels/spot-node-route-transport.d.ts +2 -2
- package/dist/runtime/channels/spot-node-route-transport.js +4 -4
- package/dist/runtime/channels/spot-route-bridge-transport.d.ts +2 -2
- package/dist/runtime/channels/spot-route-bridge-transport.js +4 -4
- package/dist/runtime/channels/spot-route-dispatch-strategy.d.ts +5 -4
- package/dist/runtime/channels/spot-route-dispatch-strategy.js +18 -18
- package/dist/runtime/diagnostics/flow-context.js +19 -7
- package/dist/runtime/diagnostics/index.js +4 -9
- package/dist/runtime/diagnostics/message-flow.d.ts +8 -0
- package/dist/runtime/diagnostics/message-flow.js +35 -5
- package/dist/runtime/diagnostics/runtime-metrics.d.ts +2 -2
- package/dist/runtime/diagnostics/runtime-metrics.js +10 -8
- package/dist/runtime/execution/index.d.ts +4 -2
- package/dist/runtime/execution/index.js +33 -10
- package/dist/runtime/execution/serial-execution-queue.d.ts +5 -0
- package/dist/runtime/execution/serial-execution-queue.js +22 -2
- package/dist/runtime/execution/state-lane.d.ts +2 -0
- package/dist/runtime/execution/state-lane.js +5 -0
- package/dist/runtime/foundation/actor-join-recovery-codec.js +2 -2
- package/dist/runtime/foundation/durable-service-request.d.ts +14 -0
- package/dist/runtime/foundation/durable-service-request.js +98 -0
- package/dist/runtime/foundation/operation-identity.d.ts +1 -0
- package/dist/runtime/foundation/operation-identity.js +5 -1
- package/dist/runtime/foundation/operation-registry.d.ts +25 -1
- package/dist/runtime/foundation/operation-registry.js +86 -18
- package/dist/runtime/foundation/raw-service-mesh-runtime.d.ts +61 -8
- package/dist/runtime/foundation/raw-service-mesh-runtime.js +486 -266
- package/dist/runtime/foundation/service-liveness-registry.d.ts +12 -1
- package/dist/runtime/foundation/service-liveness-registry.js +38 -0
- package/dist/runtime/foundation/service-mailbox.d.ts +7 -0
- package/dist/runtime/foundation/service-maintenance-runtime.js +1 -6
- package/dist/runtime/foundation/service-relocation-object-owner.d.ts +3 -3
- package/dist/runtime/foundation/service-relocation-object-owner.js +48 -34
- package/dist/runtime/foundation/service-relocation-runtime.d.ts +2 -0
- package/dist/runtime/foundation/service-relocation-runtime.js +20 -4
- package/dist/runtime/foundation/service-runtime-contracts.d.ts +10 -1
- package/dist/runtime/foundation/service-session-binding-ingress-port.d.ts +2 -1
- package/dist/runtime/foundation/service-stateful-registry.d.ts +5 -4
- package/dist/runtime/foundation/service-stateful-registry.js +24 -26
- package/dist/runtime/foundation/service-stateful-runtime.d.ts +75 -9
- package/dist/runtime/foundation/service-stateful-runtime.js +486 -218
- package/dist/runtime/foundation/service-stateful-wire-codec.d.ts +14 -6
- package/dist/runtime/foundation/service-stateful-wire-codec.js +48 -27
- package/dist/runtime/foundation/service-topology-registry.d.ts +12 -2
- package/dist/runtime/foundation/service-topology-registry.js +61 -27
- package/dist/runtime/foundation/service-wire-binary-primitives.d.ts +6 -1
- package/dist/runtime/foundation/service-wire-binary-primitives.js +27 -11
- package/dist/runtime/foundation/service-wire-constants.generated.d.ts +1 -0
- package/dist/runtime/foundation/service-wire-constants.generated.js +1 -0
- package/dist/runtime/foundation/service-wire-m6a-codec.d.ts +13 -3
- package/dist/runtime/foundation/service-wire-m6a-codec.js +189 -78
- package/dist/runtime/handlers/handler-instance-scope.d.ts +8 -0
- package/dist/runtime/handlers/handler-instance-scope.js +54 -11
- package/dist/runtime/host/actor-packet-relay.d.ts +6 -4
- package/dist/runtime/host/actor-packet-relay.js +59 -72
- package/dist/runtime/host/actor-placement-coordinator.js +25 -8
- package/dist/runtime/host/actor-runtime-options-factory.js +2 -2
- package/dist/runtime/host/actor-transfer-runtime.d.ts +7 -31
- package/dist/runtime/host/actor-transfer-runtime.js +85 -331
- package/dist/runtime/host/application-job-queue.d.ts +40 -7
- package/dist/runtime/host/application-job-queue.js +137 -42
- package/dist/runtime/host/entry-actor-runtime.d.ts +1 -0
- package/dist/runtime/host/entry-actor-runtime.js +3 -0
- package/dist/runtime/host/index.d.ts +12 -2
- package/dist/runtime/host/index.js +300 -202
- package/dist/runtime/host/location-runtime-owner.js +2 -6
- package/dist/runtime/host/mesh-router-resolver.d.ts +4 -0
- package/dist/runtime/host/mesh-router-resolver.js +14 -6
- package/dist/runtime/host/remote-bound-session-relay.d.ts +2 -4
- package/dist/runtime/host/remote-bound-session-relay.js +73 -208
- package/dist/runtime/host/route-mesh-runtime.d.ts +2 -2
- package/dist/runtime/host/route-mesh-runtime.js +6 -4
- package/dist/runtime/host/runtime-shutdown.d.ts +1 -0
- package/dist/runtime/host/runtime-shutdown.js +3 -2
- package/dist/runtime/host/service-relocation-control.d.ts +1 -0
- package/dist/runtime/host/service-relocation-control.js +10 -4
- package/dist/runtime/host/service-relocation-host-runtime.d.ts +22 -14
- package/dist/runtime/host/service-relocation-host-runtime.js +754 -599
- package/dist/runtime/host/spot-address-transport.js +3 -2
- package/dist/runtime/host/spot-runtime-options-factory.js +16 -7
- package/dist/runtime/host/user-spot-creation-coordinator.js +33 -84
- package/dist/runtime/locations/actor-location-claims.d.ts +2 -2
- package/dist/runtime/locations/actor-location-claims.js +34 -19
- package/dist/runtime/locations/auto-connect-loop.d.ts +2 -0
- package/dist/runtime/locations/auto-connect-loop.js +22 -14
- package/dist/runtime/locations/auto-connect-reconciler.js +0 -6
- package/dist/runtime/locations/auto-connect-types.d.ts +0 -1
- package/dist/runtime/locations/descriptor-publication.d.ts +4 -0
- package/dist/runtime/locations/descriptor-publication.js +11 -0
- package/dist/runtime/locations/domain-store-contract.d.ts +1 -0
- package/dist/runtime/locations/in-memory-authority-store.d.ts +7 -1
- package/dist/runtime/locations/in-memory-authority-store.js +56 -19
- package/dist/runtime/locations/in-memory-location-store.js +1 -10
- package/dist/runtime/locations/index.d.ts +1 -0
- package/dist/runtime/locations/index.js +1 -0
- package/dist/runtime/locations/lease-tracker.js +1 -6
- package/dist/runtime/locations/lifecycle.d.ts +1 -2
- package/dist/runtime/locations/lifecycle.js +2 -5
- package/dist/runtime/locations/location-store-repository.d.ts +51 -1
- package/dist/runtime/locations/location-store-repository.js +363 -139
- package/dist/runtime/locations/relocation-blob.d.ts +18 -1
- package/dist/runtime/locations/relocation-blob.js +52 -1
- package/dist/runtime/locations/request-source-resolver.d.ts +37 -0
- package/dist/runtime/locations/request-source-resolver.js +80 -0
- package/dist/runtime/locations/resolvers.js +1 -1
- package/dist/runtime/locations/runtime.d.ts +7 -3
- package/dist/runtime/locations/runtime.js +61 -53
- package/dist/runtime/messaging/framework-json-v1.d.ts +4 -6
- package/dist/runtime/messaging/framework-json-v1.js +38 -294
- package/dist/runtime/messaging/packet-name.d.ts +1 -1
- package/dist/runtime/messaging/packet-name.js +8 -24
- package/dist/runtime/messaging/payload-codec.d.ts +7 -1
- package/dist/runtime/messaging/payload-codec.js +7 -6
- package/dist/runtime/messaging/request-result.d.ts +2 -0
- package/dist/runtime/messaging/request-result.js +57 -0
- package/dist/runtime/messaging/submission-result.d.ts +1 -0
- package/dist/runtime/messaging/submission-result.js +5 -2
- package/dist/runtime/protocol/service_wire_codec.generated.d.ts +34 -1
- package/dist/runtime/protocol/service_wire_codec.generated.js +229 -0
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.d.ts +16 -1
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.js +37 -8
- package/dist/runtime/spots/formal-remote-actor-admission-registry.d.ts +7 -2
- package/dist/runtime/spots/formal-remote-actor-admission-registry.js +36 -21
- package/dist/runtime/spots/index.d.ts +21 -11
- package/dist/runtime/spots/index.js +243 -178
- package/dist/runtime/spots/spot-activation-registry.d.ts +4 -2
- package/dist/runtime/spots/spot-activation-registry.js +21 -11
- package/dist/runtime/spots/spot-activation-state.d.ts +1 -1
- package/dist/runtime/spots/spot-activation-state.js +7 -7
- package/dist/runtime/spots/spot-activation.d.ts +7 -3
- package/dist/runtime/spots/spot-activation.js +140 -52
- package/dist/runtime/spots/spot-actor-admission-coordinator.d.ts +1 -0
- package/dist/runtime/spots/spot-actor-admission-coordinator.js +13 -22
- package/dist/runtime/spots/spot-actor-membership.d.ts +0 -1
- package/dist/runtime/spots/spot-actor-membership.js +0 -8
- package/dist/runtime/spots/spot-actor-packet-dispatch.js +33 -8
- package/dist/runtime/spots/spot-actor-packet-drain.js +5 -33
- package/dist/runtime/spots/spot-entry-activation.d.ts +2 -0
- package/dist/runtime/spots/spot-entry-activation.js +20 -11
- package/dist/runtime/spots/spot-handler-registry.d.ts +7 -4
- package/dist/runtime/spots/spot-handler-registry.js +24 -10
- package/dist/runtime/spots/spot-manager-internal-contracts.d.ts +2 -2
- package/dist/runtime/spots/spot-manager-public.js +2 -2
- package/dist/runtime/spots/spot-node-autoconnect.d.ts +2 -1
- package/dist/runtime/spots/spot-node-autoconnect.js +22 -42
- package/dist/runtime/spots/spot-node-runtime-manager.d.ts +19 -2
- package/dist/runtime/spots/spot-node-runtime-manager.js +115 -50
- package/dist/runtime/spots/spot-outbound.d.ts +3 -0
- package/dist/runtime/spots/spot-outbound.js +27 -18
- package/dist/runtime/spots/spot-remote-codec.js +5 -1
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.d.ts +1 -0
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.js +1 -1
- package/dist/runtime/spots/spot-runtime-ports.d.ts +1 -1
- package/dist/runtime/spots/spot-serial-executor.d.ts +10 -0
- package/dist/runtime/spots/spot-serial-executor.js +61 -8
- package/dist/runtime/spots/spot-serial-turn-executor.js +19 -2
- package/dist/runtime/spots/spot-subscription-dispatch.js +1 -4
- package/dist/runtime/spots/spot-timer.d.ts +57 -5
- package/dist/runtime/spots/spot-timer.js +288 -103
- package/dist/runtime/streams/actor-session-binding-registry.d.ts +42 -11
- package/dist/runtime/streams/actor-session-binding-registry.js +165 -48
- package/dist/runtime/streams/actor-session-binding-runtime-owner.d.ts +8 -4
- package/dist/runtime/streams/bound-actor-relay-sender.d.ts +2 -1
- package/dist/runtime/streams/bound-actor-relay-sender.js +15 -8
- package/dist/runtime/streams/bound-session-service.d.ts +1 -1
- package/dist/runtime/streams/bound-session-service.js +6 -11
- package/dist/runtime/streams/index.d.ts +3 -2
- package/dist/runtime/streams/index.js +6 -4
- package/dist/runtime/streams/managed-stream.js +3 -2
- package/dist/runtime/streams/native-fallback-bound-session.js +8 -73
- package/dist/runtime/streams/session-actor-coordinator.d.ts +2 -8
- package/dist/runtime/streams/session-actor-coordinator.js +15 -32
- package/dist/runtime/streams/stream-binding-runtime-ports.d.ts +3 -10
- package/dist/runtime/streams/stream-session-runtime.d.ts +1 -1
- package/dist/runtime/streams/stream-session-runtime.js +14 -31
- package/dist/runtime/workers/index.js +1 -1
- package/package.json +3 -3
- package/src/contracts/Actors/ZLinkActorClient.ts +2 -1
- package/src/contracts/Channels/Calls.ts +2 -1
- package/src/contracts/Common/ZLinkEncodedPayload.ts +24 -4
- package/src/contracts/Common/ZLinkMessageMetadata.ts +13 -8
- package/src/contracts/Common/encoded-payload-storage.ts +6 -3
- package/src/contracts/Configuration/Builders.ts +4 -1
- package/src/contracts/Configuration/RegistrationBuilderPolicy.ts +7 -2
- package/src/contracts/Configuration/RegistrationBuilders.ts +6 -2
- package/src/contracts/Configuration/RegistrationTypes.ts +1 -0
- package/src/contracts/Dispatch/ZLinkDispatchOptions.ts +2 -1
- package/src/contracts/Handlers/Attributes.ts +14 -21
- package/src/contracts/Handlers/JsonContract.ts +8 -216
- package/src/contracts/Locations/Rows.ts +2 -0
- package/src/contracts/Spots/Contracts.ts +10 -1
- package/src/contracts/Spots/ZLinkSpot.ts +3 -3
- package/src/nest-integration.ts +1 -0
- package/src/runtime/abort.ts +12 -0
- package/src/runtime/actors/actor-authority-publication.ts +3 -2
- package/src/runtime/actors/actor-client.ts +100 -85
- package/src/runtime/actors/actor-context.ts +49 -9
- package/src/runtime/actors/actor-handoff.ts +291 -110
- package/src/runtime/actors/actor-join-relocation.ts +3 -0
- package/src/runtime/actors/actor-local-native-join.ts +21 -26
- package/src/runtime/actors/actor-mailbox.ts +40 -2
- package/src/runtime/actors/actor-remote-joiner.ts +6 -1
- package/src/runtime/actors/actor-remote-wire.ts +17 -0
- package/src/runtime/actors/actor-reply.ts +42 -0
- package/src/runtime/actors/actor-routed-join-transport.ts +14 -4
- package/src/runtime/actors/actor-routed-json-request.ts +10 -2
- package/src/runtime/actors/actor-runtime-contracts.ts +2 -0
- package/src/runtime/actors/actor-runtime-state.ts +2 -20
- package/src/runtime/actors/actor-source-transfer.ts +1 -34
- package/src/runtime/actors/index.ts +60 -32
- package/src/runtime/actors/session-disconnected-wire.ts +66 -0
- package/src/runtime/actors/spot-actor-dispatch.ts +4 -2
- package/src/runtime/admission.ts +69 -21
- package/src/runtime/application-jobs/application-ingress-record-owner.ts +41 -25
- package/src/runtime/application-jobs/application-job-queue-scope.ts +83 -22
- package/src/runtime/application-jobs/contracts.ts +16 -1
- package/src/runtime/backend/contracts/index.ts +92 -18
- package/src/runtime/backend/mesh-actor-session-node-adapter.ts +5 -3
- package/src/runtime/backend/mesh-completion-table.ts +160 -57
- package/src/runtime/backend/mesh-dispatch-pump.ts +237 -108
- package/src/runtime/backend/node/node-backend-adapter-support.ts +17 -8
- package/src/runtime/backend/node/node-dealer-channel-client-transport.ts +12 -2
- package/src/runtime/backend/node/node-raw-binding-port.ts +148 -85
- package/src/runtime/backend/node/node-raw-mesh-backend.ts +315 -201
- package/src/runtime/backend/node/node-socket-backend-adapter.ts +26 -11
- package/src/runtime/backend/raw-binding-port.ts +7 -2
- package/src/runtime/backend/runtime-message.ts +6 -0
- package/src/runtime/backend/runtime-values.ts +1 -0
- package/src/runtime/channels/channel-clients.ts +132 -58
- package/src/runtime/channels/channel-dispatch-pipeline.ts +198 -114
- package/src/runtime/channels/channel-dispatch-services.ts +69 -1
- package/src/runtime/channels/channel-dispatchers.ts +136 -90
- package/src/runtime/channels/channel-envelope-inspection.ts +2 -16
- package/src/runtime/channels/channel-envelope.ts +225 -31
- package/src/runtime/channels/channel-outbound-operations.ts +22 -8
- package/src/runtime/channels/channel-runtime-lifecycle.ts +13 -13
- package/src/runtime/channels/channel-runtime-manager.ts +41 -17
- package/src/runtime/channels/channel-transports.ts +171 -112
- package/src/runtime/channels/client-server-location-runtime.ts +11 -10
- package/src/runtime/channels/dispatch-error-reporter.ts +4 -2
- package/src/runtime/channels/fanout-location-runtime.ts +17 -9
- package/src/runtime/channels/local-spot-route-transport.ts +3 -1
- package/src/runtime/channels/router-socket-spot-route-transport.ts +13 -4
- package/src/runtime/channels/source-spot-router.ts +8 -4
- package/src/runtime/channels/spot-node-route-transport.ts +8 -4
- package/src/runtime/channels/spot-route-bridge-transport.ts +6 -4
- package/src/runtime/channels/spot-route-dispatch-strategy.ts +47 -18
- package/src/runtime/diagnostics/flow-context.ts +20 -7
- package/src/runtime/diagnostics/index.ts +1 -7
- package/src/runtime/diagnostics/message-flow.ts +47 -9
- package/src/runtime/diagnostics/runtime-metrics.ts +12 -10
- package/src/runtime/execution/index.ts +40 -14
- package/src/runtime/execution/serial-execution-queue.ts +33 -4
- package/src/runtime/execution/state-lane.ts +7 -0
- package/src/runtime/foundation/actor-join-recovery-codec.ts +2 -2
- package/src/runtime/foundation/durable-service-request.ts +137 -0
- package/src/runtime/foundation/operation-identity.ts +5 -1
- package/src/runtime/foundation/operation-registry.ts +98 -20
- package/src/runtime/foundation/raw-service-mesh-runtime.ts +704 -336
- package/src/runtime/foundation/service-liveness-registry.ts +39 -0
- package/src/runtime/foundation/service-mailbox.ts +8 -0
- package/src/runtime/foundation/service-maintenance-runtime.ts +1 -8
- package/src/runtime/foundation/service-relocation-object-owner.ts +55 -36
- package/src/runtime/foundation/service-relocation-runtime.ts +25 -4
- package/src/runtime/foundation/service-runtime-contracts.ts +19 -2
- package/src/runtime/foundation/service-session-binding-ingress-port.ts +2 -1
- package/src/runtime/foundation/service-stateful-registry.ts +38 -35
- package/src/runtime/foundation/service-stateful-runtime.ts +732 -275
- package/src/runtime/foundation/service-stateful-wire-codec.ts +49 -16
- package/src/runtime/foundation/service-topology-registry.ts +91 -36
- package/src/runtime/foundation/service-wire-binary-primitives.ts +35 -12
- package/src/runtime/foundation/service-wire-constants.generated.ts +1 -0
- package/src/runtime/foundation/service-wire-m6a-codec.ts +231 -76
- package/src/runtime/handlers/handler-instance-scope.ts +67 -16
- package/src/runtime/host/actor-packet-relay.ts +113 -94
- package/src/runtime/host/actor-placement-coordinator.ts +31 -8
- package/src/runtime/host/actor-runtime-options-factory.ts +13 -2
- package/src/runtime/host/actor-transfer-runtime.ts +104 -430
- package/src/runtime/host/application-job-queue.ts +160 -41
- package/src/runtime/host/entry-actor-runtime.ts +4 -0
- package/src/runtime/host/index.ts +473 -271
- package/src/runtime/host/location-runtime-owner.ts +2 -6
- package/src/runtime/host/mesh-router-resolver.ts +13 -5
- package/src/runtime/host/remote-bound-session-relay.ts +88 -271
- package/src/runtime/host/route-mesh-runtime.ts +12 -6
- package/src/runtime/host/runtime-shutdown.ts +15 -4
- package/src/runtime/host/service-relocation-control.ts +11 -5
- package/src/runtime/host/service-relocation-host-runtime.ts +1092 -759
- package/src/runtime/host/spot-address-transport.ts +4 -2
- package/src/runtime/host/spot-runtime-options-factory.ts +20 -7
- package/src/runtime/host/user-spot-creation-coordinator.ts +41 -100
- package/src/runtime/locations/actor-location-claims.ts +52 -30
- package/src/runtime/locations/auto-connect-loop.ts +22 -14
- package/src/runtime/locations/auto-connect-reconciler.ts +0 -6
- package/src/runtime/locations/auto-connect-types.ts +0 -1
- package/src/runtime/locations/descriptor-publication.ts +7 -0
- package/src/runtime/locations/domain-store-contract.ts +5 -0
- package/src/runtime/locations/in-memory-authority-store.ts +85 -23
- package/src/runtime/locations/in-memory-location-store.ts +1 -12
- package/src/runtime/locations/index.ts +1 -0
- package/src/runtime/locations/lease-tracker.ts +2 -9
- package/src/runtime/locations/lifecycle.ts +8 -26
- package/src/runtime/locations/location-store-repository.ts +628 -152
- package/src/runtime/locations/relocation-blob.ts +75 -1
- package/src/runtime/locations/request-source-resolver.ts +138 -0
- package/src/runtime/locations/resolvers.ts +2 -2
- package/src/runtime/locations/runtime.ts +98 -64
- package/src/runtime/messaging/framework-json-v1.ts +5 -325
- package/src/runtime/messaging/packet-name.ts +5 -25
- package/src/runtime/messaging/payload-codec.ts +8 -11
- package/src/runtime/messaging/request-result.ts +62 -0
- package/src/runtime/messaging/submission-result.ts +5 -1
- package/src/runtime/protocol/service_wire_codec.generated.ts +218 -5
- package/src/runtime/protocol/service_wire_pilot_codec.generated.ts +16 -13
- package/src/runtime/spots/formal-remote-actor-admission-registry.ts +46 -25
- package/src/runtime/spots/index.ts +389 -283
- package/src/runtime/spots/spot-activation-registry.ts +25 -13
- package/src/runtime/spots/spot-activation-state.ts +12 -19
- package/src/runtime/spots/spot-activation.ts +198 -96
- package/src/runtime/spots/spot-actor-admission-coordinator.ts +25 -55
- package/src/runtime/spots/spot-actor-membership.ts +0 -15
- package/src/runtime/spots/spot-actor-packet-dispatch.ts +56 -11
- package/src/runtime/spots/spot-actor-packet-drain.ts +9 -52
- package/src/runtime/spots/spot-entry-activation.ts +33 -21
- package/src/runtime/spots/spot-handler-registry.ts +56 -12
- package/src/runtime/spots/spot-manager-internal-contracts.ts +2 -2
- package/src/runtime/spots/spot-manager-public.ts +2 -2
- package/src/runtime/spots/spot-node-autoconnect.ts +28 -47
- package/src/runtime/spots/spot-node-runtime-manager.ts +197 -67
- package/src/runtime/spots/spot-outbound.ts +39 -19
- package/src/runtime/spots/spot-remote-codec.ts +9 -2
- package/src/runtime/spots/spot-routed-spot-packet-dispatch.ts +2 -1
- package/src/runtime/spots/spot-runtime-ports.ts +1 -1
- package/src/runtime/spots/spot-serial-executor.ts +97 -25
- package/src/runtime/spots/spot-serial-turn-executor.ts +22 -3
- package/src/runtime/spots/spot-subscription-dispatch.ts +1 -4
- package/src/runtime/spots/spot-timer.ts +418 -158
- package/src/runtime/streams/actor-session-binding-registry.ts +236 -64
- package/src/runtime/streams/actor-session-binding-runtime-owner.ts +16 -8
- package/src/runtime/streams/bound-actor-relay-sender.ts +23 -9
- package/src/runtime/streams/bound-session-service.ts +7 -10
- package/src/runtime/streams/index.ts +12 -6
- package/src/runtime/streams/managed-stream.ts +4 -2
- package/src/runtime/streams/native-fallback-bound-session.ts +14 -103
- package/src/runtime/streams/session-actor-coordinator.ts +39 -88
- package/src/runtime/streams/stream-binding-runtime-ports.ts +3 -9
- package/src/runtime/streams/stream-session-runtime.ts +13 -34
- package/src/runtime/workers/index.ts +1 -1
|
@@ -5,18 +5,26 @@ exports.decodeQueuedHandoffPacket = decodeQueuedHandoffPacket;
|
|
|
5
5
|
exports.createServiceRelocationId = createServiceRelocationId;
|
|
6
6
|
exports.relocationFailedFailureCode = relocationFailedFailureCode;
|
|
7
7
|
exports.encodeHandoffQueuedMessages = encodeHandoffQueuedMessages;
|
|
8
|
+
const actor_reply_1 = require("../actors/actor-reply");
|
|
9
|
+
const protocol_1 = require("../streams/protocol");
|
|
10
|
+
const service_wire_m6a_codec_1 = require("../foundation/service-wire-m6a-codec");
|
|
11
|
+
const service_wire_constants_generated_1 = require("../foundation/service-wire-constants.generated");
|
|
12
|
+
const operation_registry_1 = require("../foundation/operation-registry");
|
|
13
|
+
const durable_service_request_1 = require("../foundation/durable-service-request");
|
|
8
14
|
const stream_wire_1 = require("@zlink-systems/stream-wire");
|
|
9
15
|
const node_crypto_1 = require("node:crypto");
|
|
10
16
|
const contracts_1 = require("../../contracts");
|
|
17
|
+
const ZLinkDispatchOptions_1 = require("../../contracts/Dispatch/ZLinkDispatchOptions");
|
|
11
18
|
const Options_1 = require("../../contracts/Locations/Options");
|
|
12
19
|
const Stores_1 = require("../../contracts/Locations/Stores");
|
|
13
20
|
const actor_authority_publication_1 = require("../actors/actor-authority-publication");
|
|
14
|
-
const actor_handoff_1 = require("../actors/actor-handoff");
|
|
15
21
|
const actor_remote_wire_1 = require("../actors/actor-remote-wire");
|
|
16
22
|
const actor_runtime_state_1 = require("../actors/actor-runtime-state");
|
|
17
23
|
const runtime_values_1 = require("../backend/runtime-values");
|
|
18
24
|
const diagnostics_1 = require("../diagnostics");
|
|
19
25
|
const runtime_metrics_1 = require("../diagnostics/runtime-metrics");
|
|
26
|
+
const message_flow_1 = require("../diagnostics/message-flow");
|
|
27
|
+
const dispatch_error_details_1 = require("../diagnostics/dispatch-error-details");
|
|
20
28
|
const state_lane_1 = require("../execution/state-lane");
|
|
21
29
|
const actor_join_recovery_codec_1 = require("../foundation/actor-join-recovery-codec");
|
|
22
30
|
const operation_identity_1 = require("../foundation/operation-identity");
|
|
@@ -28,8 +36,8 @@ const service_relocation_object_owner_1 = require("../foundation/service-relocat
|
|
|
28
36
|
const service_relocation_runtime_1 = require("../foundation/service-relocation-runtime");
|
|
29
37
|
const service_runtime_contracts_1 = require("../foundation/service-runtime-contracts");
|
|
30
38
|
const service_stateful_wire_codec_1 = require("../foundation/service-stateful-wire-codec");
|
|
31
|
-
const
|
|
32
|
-
const
|
|
39
|
+
const service_wire_constants_generated_2 = require("../foundation/service-wire-constants.generated");
|
|
40
|
+
const service_wire_m6a_codec_2 = require("../foundation/service-wire-m6a-codec");
|
|
33
41
|
const framework_errors_internal_1 = require("../framework-errors-internal");
|
|
34
42
|
const authority_key_codec_1 = require("../locations/authority-key-codec");
|
|
35
43
|
const location_store_failure_1 = require("../locations/location-store-failure");
|
|
@@ -42,7 +50,6 @@ const relocation_direct_transfer_1 = require("./relocation-direct-transfer");
|
|
|
42
50
|
const relocation_integrity_fault_gate_1 = require("./relocation-integrity-fault-gate");
|
|
43
51
|
const relocation_state_adapter_1 = require("./relocation-state-adapter");
|
|
44
52
|
const service_relocation_control_1 = require("./service-relocation-control");
|
|
45
|
-
const RELOCATION_ATTEMPT_TIMEOUT_MS = 250;
|
|
46
53
|
const RELOCATION_AUTHORITY_RETRY_DELAY_MS = 25;
|
|
47
54
|
const RELOCATION_MAX_SEND_ATTEMPTS = 120;
|
|
48
55
|
const RELOCATION_AUTHORITY_POLL_INTERVAL_MS = 10;
|
|
@@ -60,12 +67,12 @@ const RELOCATION_PARTICIPANT_STATE_LIMIT_BYTES = 64 * 1024 * 1024;
|
|
|
60
67
|
const RELOCATION_TARGET_TOMBSTONE_LIMIT = 1024;
|
|
61
68
|
const RELOCATION_OPERATION_RETENTION_MS = 5 * 60_000;
|
|
62
69
|
const SERVICE_CONTROL_TERMINAL_CAPACITY = 4096;
|
|
70
|
+
const INITIAL_MEMBERSHIP_EPOCH = 1n;
|
|
63
71
|
/** Production host bridge from Retire inventory to remote RouteMesh owners. */
|
|
64
72
|
class ZLinkHostServiceRelocationRuntime {
|
|
65
73
|
options;
|
|
66
74
|
targetStages = new Map();
|
|
67
75
|
terminalTargets = new bounded_replay_map_1.BoundedReplayMap(RELOCATION_TARGET_TOMBSTONE_LIMIT);
|
|
68
|
-
relocationAuthorityKeys = new Map();
|
|
69
76
|
targetPrepareOperations = new Map();
|
|
70
77
|
pendingReplyRelays = new Map();
|
|
71
78
|
pendingSessionRelocations = new Map();
|
|
@@ -103,6 +110,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
103
110
|
* checksum integrity failure — it must not be reported as DataLost(35).
|
|
104
111
|
*/
|
|
105
112
|
disposed = false;
|
|
113
|
+
targetPreparationStop = new AbortController();
|
|
106
114
|
constructor(options) {
|
|
107
115
|
this.options = options;
|
|
108
116
|
}
|
|
@@ -130,6 +138,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
130
138
|
reject(failure.cause);
|
|
131
139
|
};
|
|
132
140
|
});
|
|
141
|
+
let acceptRelay;
|
|
142
|
+
const relayReady = new Promise((resolve) => {
|
|
143
|
+
acceptRelay = resolve;
|
|
144
|
+
});
|
|
133
145
|
this.sourceActorJoinProfiles.set(input.relocationId, {
|
|
134
146
|
actor: input.actor,
|
|
135
147
|
state: input.state,
|
|
@@ -143,32 +155,43 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
143
155
|
: actor_join_recovery_codec_1.RECREATE_RELOCATION_CONTENT_TYPE,
|
|
144
156
|
completionOperationId: input.completionOperationId,
|
|
145
157
|
canonicalRecovery: input.canonicalRecovery,
|
|
158
|
+
relayReady: acceptRelay,
|
|
146
159
|
ready,
|
|
147
160
|
resolveReady
|
|
148
161
|
});
|
|
149
162
|
const membershipEpoch = input.state.spotMembershipEpoch > 0n ? input.state.spotMembershipEpoch : 1n;
|
|
150
163
|
const spotGeneration = input.target.targetSpotGeneration ?? target.lifecycleGeneration;
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
catch (error) {
|
|
162
|
-
if (error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) {
|
|
163
|
-
resolveReady(error);
|
|
164
|
-
this.completeActorJoinSourceCleanup(input.state.actorId);
|
|
164
|
+
const cleanup = (async () => {
|
|
165
|
+
try {
|
|
166
|
+
await this.relocateStandaloneActor(input.meshName, input.state, target, target.applicationVersion, input.signal, {
|
|
167
|
+
...input.target,
|
|
168
|
+
spotKind: input.target.spotKind === contracts_1.ZLinkSpotKind.Entry
|
|
169
|
+
? contracts_1.ZLinkSpotKind.Entry
|
|
170
|
+
: contracts_1.ZLinkSpotKind.User,
|
|
171
|
+
spotObjectGeneration: spotGeneration
|
|
172
|
+
}, input.relocationId, true, input.advertisedReceiveChunkLimitBytes, input.deadlineAtMs);
|
|
173
|
+
resolveReady();
|
|
165
174
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
175
|
+
catch (error) {
|
|
176
|
+
if (error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) {
|
|
177
|
+
resolveReady(error);
|
|
178
|
+
this.completeActorJoinSourceCleanup(input.state.actorId);
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
this.sourceActorJoinProfiles.delete(input.relocationId);
|
|
182
|
+
resolveReady();
|
|
183
|
+
throw error;
|
|
184
|
+
}
|
|
169
185
|
}
|
|
170
|
-
}
|
|
186
|
+
})();
|
|
187
|
+
const settledCleanup = Promise.allSettled([cleanup]);
|
|
188
|
+
await Promise.race([relayReady, cleanup]);
|
|
171
189
|
return {
|
|
190
|
+
sourceCleanup: async () => {
|
|
191
|
+
const [outcome] = await settledCleanup;
|
|
192
|
+
if (outcome.status === 'rejected')
|
|
193
|
+
throw outcome.reason;
|
|
194
|
+
},
|
|
172
195
|
actorRef: {
|
|
173
196
|
actorId: sourceRef.actorId,
|
|
174
197
|
generation: sourceRef.generation,
|
|
@@ -199,9 +222,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
199
222
|
if (window.followTimer !== undefined)
|
|
200
223
|
clearTimeout(window.followTimer);
|
|
201
224
|
}
|
|
225
|
+
const hadSourceWindows = this.sourceCutoverWindows.size !== 0;
|
|
202
226
|
this.sourceCutoverWindows.clear();
|
|
203
|
-
|
|
204
|
-
|
|
227
|
+
if (hadSourceWindows)
|
|
228
|
+
this.options.sourceQuiescenceChanged?.();
|
|
229
|
+
const stopped = new ServiceRelocationRuntimeStoppedError();
|
|
205
230
|
this.targetPrepareOperations.clear();
|
|
206
231
|
for (const pending of this.pendingReplyRelays.values()) {
|
|
207
232
|
if (pending.timer !== undefined)
|
|
@@ -219,10 +244,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
219
244
|
this.pendingSessionRelocations.clear();
|
|
220
245
|
this.terminalSessionRelocations.clear();
|
|
221
246
|
this.failedSessionRouteSubmits.clear();
|
|
222
|
-
this.
|
|
247
|
+
await this.drainActorJoinSources();
|
|
223
248
|
this.terminalActorJoinSourceLeaves.clear();
|
|
224
|
-
this.options.boundSessionRelocation?.clear?.();
|
|
225
|
-
this.relocationAuthorityKeys.clear();
|
|
249
|
+
await this.options.boundSessionRelocation?.clear?.();
|
|
226
250
|
if (errors.length === 1)
|
|
227
251
|
throw errors[0];
|
|
228
252
|
if (errors.length > 1)
|
|
@@ -230,6 +254,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
230
254
|
}
|
|
231
255
|
/** Host shutdown waits for the existing attempt terminal before releasing its lease. */
|
|
232
256
|
async drainTargetAttempts() {
|
|
257
|
+
this.targetPreparationStop.abort(new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.ShuttingDown, 'Relocation target is shutting down.'));
|
|
258
|
+
for (const pending of this.targetAssemblies.values()) {
|
|
259
|
+
pending.assembly.fail('Relocation target is shutting down.');
|
|
260
|
+
}
|
|
261
|
+
await Promise.allSettled([...this.targetPrepareOperations.values()].map((value) => value.promise));
|
|
233
262
|
const results = await Promise.allSettled([...this.targetStages.entries()].map(([stagingId, stage]) => {
|
|
234
263
|
if (stage.fallback !== undefined)
|
|
235
264
|
clearTimeout(stage.fallback);
|
|
@@ -252,18 +281,34 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
252
281
|
}
|
|
253
282
|
/** Spec 28 §4.4: the target owner alone orders shutdown seal and verified cutover. */
|
|
254
283
|
targetStageSealed(stage) {
|
|
255
|
-
return
|
|
284
|
+
return !stage.cutoverReceived && this.targetAdmissionSealed();
|
|
285
|
+
}
|
|
286
|
+
/** The Shutdown admission seal: closed once the target owner starts shutdown. */
|
|
287
|
+
targetAdmissionSealed() {
|
|
288
|
+
return this.disposed || this.options.targetAdmissionSealed?.() === true;
|
|
256
289
|
}
|
|
257
290
|
completeActorJoinSourceCleanup(actorId) {
|
|
258
291
|
for (const [relocationId, profile] of this.sourceActorJoinProfiles) {
|
|
259
292
|
if (profile.state.actorId !== actorId)
|
|
260
293
|
continue;
|
|
261
|
-
this.
|
|
262
|
-
this.terminalActorJoinSourceLeaves.remember(relocationId, actorId);
|
|
263
|
-
void this.finalizeActorJoinSource(profile);
|
|
294
|
+
void this.retireActorJoinSource(relocationId, profile);
|
|
264
295
|
}
|
|
265
296
|
}
|
|
266
|
-
|
|
297
|
+
/** Drains the exact source scopes before their Spot lifecycle owners stop. */
|
|
298
|
+
async drainActorJoinSources(meshName) {
|
|
299
|
+
await Promise.all([...this.sourceActorJoinProfiles].map(([relocationId, profile]) => meshName === undefined || profile.state.meshName === meshName
|
|
300
|
+
? this.retireActorJoinSource(relocationId, profile)
|
|
301
|
+
: undefined));
|
|
302
|
+
}
|
|
303
|
+
retireActorJoinSource(relocationId, profile) {
|
|
304
|
+
return (profile.retirement ??= this.finalizeActorJoinSource(profile, relocationId).finally(() => {
|
|
305
|
+
if (this.sourceActorJoinProfiles.get(relocationId) === profile) {
|
|
306
|
+
this.sourceActorJoinProfiles.delete(relocationId);
|
|
307
|
+
this.terminalActorJoinSourceLeaves.remember(relocationId, profile.state.actorId);
|
|
308
|
+
}
|
|
309
|
+
}));
|
|
310
|
+
}
|
|
311
|
+
async finalizeActorJoinSource(profile, relocationId) {
|
|
267
312
|
const failures = [];
|
|
268
313
|
try {
|
|
269
314
|
await profile.ready;
|
|
@@ -271,6 +316,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
271
316
|
catch (error) {
|
|
272
317
|
failures.push(error);
|
|
273
318
|
}
|
|
319
|
+
// A pre-commit failure removed this attempt and restored the source.
|
|
320
|
+
if (this.sourceActorJoinProfiles.get(relocationId) !== profile)
|
|
321
|
+
return;
|
|
274
322
|
try {
|
|
275
323
|
await this.options.actorTransfer.completeRelocationSourceLeave(profile.actor, profile.sourceSpotId);
|
|
276
324
|
}
|
|
@@ -280,7 +328,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
280
328
|
try {
|
|
281
329
|
await this.options
|
|
282
330
|
.actorManager()
|
|
283
|
-
?.
|
|
331
|
+
?.completeCoreRelocationSource(profile.state.actorId, profile.state);
|
|
284
332
|
}
|
|
285
333
|
catch (error) {
|
|
286
334
|
failures.push(error);
|
|
@@ -326,10 +374,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
326
374
|
id: `actor:${state.actorId}`,
|
|
327
375
|
relocate: async (relocationSignal) => {
|
|
328
376
|
await relocateShell(relocationSignal);
|
|
377
|
+
const spotAuthority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(activation.spotId)), relocationSignal);
|
|
329
378
|
await this.relocateStandaloneActor(meshName, state, undefined, targetApplicationVersion, relocationSignal, {
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
spotObjectGeneration: (await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(activation.spotId)), relocationSignal)).objectGeneration
|
|
379
|
+
...committedSpotRoute(meshName, activation.spotId, contracts_1.ZLinkSpotKind.User, spotAuthority.allocation.descriptor.rid, spotAuthority),
|
|
380
|
+
spotObjectGeneration: spotAuthority.objectGeneration
|
|
333
381
|
});
|
|
334
382
|
}
|
|
335
383
|
});
|
|
@@ -402,18 +450,27 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
402
450
|
}
|
|
403
451
|
throw failure;
|
|
404
452
|
}
|
|
405
|
-
|
|
453
|
+
tryHandleControl(meshName, record, signal) {
|
|
406
454
|
if (record.parts.length === 0)
|
|
407
455
|
return false;
|
|
408
456
|
const payload = record.parts[0].data();
|
|
409
|
-
|
|
457
|
+
const command = (0, service_relocation_control_1.serviceRelocationControlCommand)(payload);
|
|
458
|
+
if (command === undefined) {
|
|
459
|
+
if (record.kind !== service_runtime_contracts_1.ReceiveKind.NodeSend || record.parts.length !== 1)
|
|
460
|
+
return false;
|
|
461
|
+
const sourceLeave = (0, actor_remote_wire_1.decodeRemoteActorSourceLeaveTerminal)(payload);
|
|
462
|
+
if (sourceLeave === undefined)
|
|
463
|
+
return false;
|
|
464
|
+
return this.acceptActorJoinSourceLeave(sourceLeave);
|
|
465
|
+
}
|
|
466
|
+
return this.handleControl(meshName, record, payload, command, signal);
|
|
467
|
+
}
|
|
468
|
+
async handleControl(meshName, record, payload, command, signal) {
|
|
469
|
+
if (command === service_stateful_wire_codec_1.M6bServiceWireCommand.replyRelay) {
|
|
410
470
|
await this.handleReplyRelay(meshName, (0, service_stateful_wire_codec_1.decodeMaintenanceReplyRelay)(record.parts.map((part) => part.data())), record.sourceNodeRid, signal);
|
|
411
471
|
return true;
|
|
412
472
|
}
|
|
413
|
-
|
|
414
|
-
? undefined
|
|
415
|
-
: decodePrepareSideband(record.parts.slice(1).map((part) => part.data()));
|
|
416
|
-
if (record.parts.length > 1 && sideband === undefined)
|
|
473
|
+
if (record.parts.length !== 1)
|
|
417
474
|
return false;
|
|
418
475
|
if (record.kind === service_runtime_contracts_1.ReceiveKind.NodeSend) {
|
|
419
476
|
const sourceLeave = (0, actor_remote_wire_1.decodeRemoteActorSourceLeaveTerminal)(payload);
|
|
@@ -421,33 +478,57 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
421
478
|
return this.acceptActorJoinSourceLeave(sourceLeave);
|
|
422
479
|
}
|
|
423
480
|
}
|
|
424
|
-
if (
|
|
481
|
+
if (command === service_stateful_wire_codec_1.M6bServiceWireCommand.sessionRelocationSealed) {
|
|
425
482
|
this.acceptSessionRelocationResponse((0, service_stateful_wire_codec_1.decodeSessionRelocationSealed)(payload), record.sourceNodeRid);
|
|
426
483
|
return true;
|
|
427
484
|
}
|
|
428
|
-
if (
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
485
|
+
if (command === service_stateful_wire_codec_1.M6bServiceWireCommand.sessionRelocationSeal) {
|
|
486
|
+
let request;
|
|
487
|
+
try {
|
|
488
|
+
request = (0, service_stateful_wire_codec_1.decodeSessionRelocationSeal)(payload);
|
|
489
|
+
}
|
|
490
|
+
catch (error) {
|
|
491
|
+
const outcome = ZLinkDispatchOptions_1.ZLinkRuntimeMessageFlowOutcome.Error;
|
|
492
|
+
(0, message_flow_1.flowIfEnabled)(this.options.messageFlow?.(), outcome)?.trace({
|
|
493
|
+
outcome,
|
|
494
|
+
surface: ZLinkDispatchOptions_1.ZLinkDispatchErrorSurface.StreamSession,
|
|
495
|
+
messageKind: ZLinkDispatchOptions_1.ZLinkDispatchMessageKind.Control,
|
|
496
|
+
packetName: 'sessionRelocationSeal',
|
|
497
|
+
sourceRid: record.sourceNodeRid ?? undefined,
|
|
498
|
+
errorReason: 'invalid_frame',
|
|
499
|
+
errorAction: ZLinkDispatchOptions_1.ZLinkRuntimeDispatchErrorAction.Drop,
|
|
500
|
+
...(0, dispatch_error_details_1.dispatchErrorDetails)(error)
|
|
501
|
+
});
|
|
502
|
+
throw error;
|
|
503
|
+
}
|
|
504
|
+
let response;
|
|
505
|
+
try {
|
|
506
|
+
response = await this.handleSessionRelocationSeal(meshName, request, record.sourceNodeRid, signal);
|
|
507
|
+
}
|
|
508
|
+
catch (error) {
|
|
509
|
+
(0, message_flow_1.traceSessionBindingControlDrop)(this.options.messageFlow?.(), record.sourceNodeRid ?? '', request.actor.actor.actorId, 'sessionRelocationSeal', error instanceof contracts_1.ZLinkFrameworkException &&
|
|
510
|
+
error.kind === contracts_1.ZLinkFrameworkErrorKind.Unavailable
|
|
511
|
+
? 'location_unavailable'
|
|
512
|
+
: 'stale_target', error);
|
|
513
|
+
throw error;
|
|
514
|
+
}
|
|
515
|
+
if (response !== undefined) {
|
|
516
|
+
await this.sendSessionRelocationResponse(meshName, record.sourceNodeRid, (0, service_stateful_wire_codec_1.encodeSessionRelocationSealed)(response));
|
|
517
|
+
}
|
|
432
518
|
return true;
|
|
433
519
|
}
|
|
434
|
-
if (
|
|
520
|
+
if (command === service_stateful_wire_codec_1.M6bServiceWireCommand.sessionRelocationRoute) {
|
|
435
521
|
const request = (0, service_stateful_wire_codec_1.decodeSessionRelocationRoute)(payload);
|
|
436
522
|
await this.handleSessionRelocationRoute(meshName, request, record.sourceNodeRid, signal);
|
|
437
523
|
return true;
|
|
438
524
|
}
|
|
439
|
-
if (
|
|
525
|
+
if (command === service_stateful_wire_codec_1.M6bServiceWireCommand.replyRelayAck) {
|
|
440
526
|
await this.acceptReplyRelayAck(meshName, (0, service_stateful_wire_codec_1.decodeMaintenanceReplyRelayAck)(payload), record.sourceNodeRid, signal);
|
|
441
527
|
return true;
|
|
442
528
|
}
|
|
443
529
|
let request = (0, service_relocation_control_1.decodeServiceRelocationControlRequest)(payload);
|
|
444
530
|
if (request === undefined)
|
|
445
531
|
return false;
|
|
446
|
-
if (sideband !== undefined) {
|
|
447
|
-
if (request.kind !== 'prepare')
|
|
448
|
-
return false;
|
|
449
|
-
request = { ...request, nodeInternalBoundSessions: sideband };
|
|
450
|
-
}
|
|
451
532
|
// Ready(30) and Failed(53) are replies to Prepare(40), never independent
|
|
452
533
|
// inbound controls. The source consumes them on Core's request reply leg.
|
|
453
534
|
if (request.kind === 'ready' || request.kind === 'failed')
|
|
@@ -461,7 +542,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
461
542
|
throw new Error('Relocation prepare has no authenticated source node.');
|
|
462
543
|
}
|
|
463
544
|
if (record.kind !== service_runtime_contracts_1.ReceiveKind.NodeRequest) {
|
|
464
|
-
throw new
|
|
545
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Relocation prepare requires a Core request leg.');
|
|
465
546
|
}
|
|
466
547
|
const operationKey = relocationStagingId(request);
|
|
467
548
|
const fingerprint = stringifyWire(request);
|
|
@@ -474,12 +555,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
474
555
|
this.targetAssemblies.get(operationKey)?.fingerprint ??
|
|
475
556
|
this.targetStages.get(operationKey)?.offer.prepareFingerprint ??
|
|
476
557
|
this.terminalTargets.get(operationKey);
|
|
477
|
-
if (request.sourceNodeRid
|
|
558
|
+
if (!(0, routing_id_1.routingIdsEqual)(request.sourceNodeRid, record.sourceNodeRid))
|
|
478
559
|
return true;
|
|
479
560
|
if (acceptedFingerprint !== undefined && acceptedFingerprint !== fingerprint) {
|
|
480
561
|
if (!samePrepareIdentity(acceptedFingerprint, request))
|
|
481
562
|
return true;
|
|
482
|
-
this.replyPrepare(record, relocationFailed(request,
|
|
563
|
+
this.replyPrepare(record, relocationFailed(request, service_wire_constants_generated_2.ServiceWireFrameworkErrorCode.relocationDataLost));
|
|
483
564
|
return true;
|
|
484
565
|
}
|
|
485
566
|
// A restore that failed outright is terminal: the source observes the
|
|
@@ -503,7 +584,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
503
584
|
// finish the restore and the Ready reply asynchronously.
|
|
504
585
|
let promise;
|
|
505
586
|
try {
|
|
506
|
-
this.registerTargetAssembly(operationKey, request,
|
|
587
|
+
this.registerTargetAssembly(operationKey, request, record.sourceNodeRid);
|
|
507
588
|
promise = this.handlePrepareControl(meshName, operationKey, request, record.sourceNodeRid, signal);
|
|
508
589
|
}
|
|
509
590
|
catch (error) {
|
|
@@ -614,7 +695,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
614
695
|
const pending = this.targetAssemblies.get(operationKey);
|
|
615
696
|
if (pending === undefined ||
|
|
616
697
|
sourceNodeRid === null ||
|
|
617
|
-
|
|
698
|
+
!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, pending.authenticatedSourceNodeRid) ||
|
|
618
699
|
request.senderRole !== 'source' ||
|
|
619
700
|
!sameCoordinator(request.coordinator, pending.prepare.coordinator) ||
|
|
620
701
|
!sameRelocationObject(request.object, pending.prepare.object)) {
|
|
@@ -666,7 +747,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
666
747
|
// to itself; RouteMesh has no self connection (NotConnected), so the
|
|
667
748
|
// control dispatches through the same inbound handler locally.
|
|
668
749
|
const localRid = this.requireMeshNode(meshName).status().routingId;
|
|
669
|
-
if (
|
|
750
|
+
if ((0, routing_id_1.routingIdsEqual)(localRid, targetNodeRid)) {
|
|
670
751
|
await this.handleSessionRelocationRoute(meshName, request, localRid, signal);
|
|
671
752
|
return;
|
|
672
753
|
}
|
|
@@ -686,20 +767,21 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
686
767
|
const key = sessionRelocationPendingKey(request);
|
|
687
768
|
const bytes = (0, service_stateful_wire_codec_1.encodeSessionRelocationSeal)(request);
|
|
688
769
|
const requestFingerprint = bytes.toString('base64');
|
|
689
|
-
const expectedTarget =
|
|
770
|
+
const expectedTarget = (0, routing_id_1.encodeRoutingIdStorageHex)(targetNodeRid);
|
|
690
771
|
const terminal = this.terminalSessionRelocations.get(key);
|
|
691
772
|
if (terminal !== undefined) {
|
|
692
773
|
if (terminal.targetNodeRid !== expectedTarget ||
|
|
693
774
|
terminal.requestFingerprint !== requestFingerprint) {
|
|
694
|
-
return Promise.reject(new
|
|
775
|
+
return Promise.reject(new service_wire_m6a_codec_2.ServiceWireProtocolError(`Session relocation control '${key}' repeated with different bytes or target.`));
|
|
695
776
|
}
|
|
696
777
|
this.terminalSessionRelocations.touch(key);
|
|
778
|
+
return Promise.resolve(terminal.response);
|
|
697
779
|
}
|
|
698
780
|
const existing = this.pendingSessionRelocations.get(key);
|
|
699
781
|
if (existing !== undefined) {
|
|
700
782
|
if (existing.targetNodeRid !== expectedTarget ||
|
|
701
783
|
existing.requestFingerprint !== requestFingerprint) {
|
|
702
|
-
return Promise.reject(new
|
|
784
|
+
return Promise.reject(new service_wire_m6a_codec_2.ServiceWireProtocolError(`Session relocation control '${key}' repeated with different bytes or target.`));
|
|
703
785
|
}
|
|
704
786
|
return existing.promise;
|
|
705
787
|
}
|
|
@@ -752,10 +834,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
752
834
|
// connection (NotConnected); dispatch through the inbound handler
|
|
753
835
|
// locally instead of the wire.
|
|
754
836
|
const localRid = this.requireMeshNode(meshName).status().routingId;
|
|
755
|
-
if (
|
|
837
|
+
if ((0, routing_id_1.encodeRoutingIdStorageHex)(localRid) === expectedTarget) {
|
|
756
838
|
try {
|
|
757
839
|
const response = await this.handleSessionRelocationSeal(meshName, request, localRid, signal);
|
|
758
|
-
|
|
840
|
+
if (response !== undefined)
|
|
841
|
+
pending.resolve(response);
|
|
759
842
|
}
|
|
760
843
|
catch (error) {
|
|
761
844
|
pending.reject(error);
|
|
@@ -788,9 +871,8 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
788
871
|
resolve: (response) => finish(() => {
|
|
789
872
|
this.terminalSessionRelocations.remember(key, {
|
|
790
873
|
targetNodeRid: expectedTarget,
|
|
791
|
-
request,
|
|
792
874
|
requestFingerprint,
|
|
793
|
-
|
|
875
|
+
response
|
|
794
876
|
});
|
|
795
877
|
resolvePromise(response);
|
|
796
878
|
}),
|
|
@@ -816,18 +898,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
816
898
|
const terminal = this.terminalSessionRelocations.get(key);
|
|
817
899
|
if (terminal === undefined)
|
|
818
900
|
return;
|
|
819
|
-
|
|
820
|
-
throw new service_wire_m6a_codec_1.ServiceWireProtocolError(`Session relocation ACK '${key}' source node changed.`);
|
|
821
|
-
}
|
|
822
|
-
validateSessionRelocationResponse(terminal.request, response);
|
|
823
|
-
if (sessionRelocationResponseFingerprint(response) !== terminal.responseFingerprint) {
|
|
824
|
-
throw new service_wire_m6a_codec_1.ServiceWireProtocolError(`Session relocation ACK '${key}' repeated with different bytes.`);
|
|
825
|
-
}
|
|
901
|
+
// Session–Actor binding §8.3: the first command 43 is final for this participant.
|
|
826
902
|
this.terminalSessionRelocations.touch(key);
|
|
827
903
|
return;
|
|
828
904
|
}
|
|
829
|
-
if (sourceNodeRid === null ||
|
|
830
|
-
|
|
905
|
+
if (sourceNodeRid === null ||
|
|
906
|
+
(0, routing_id_1.encodeRoutingIdStorageHex)(sourceNodeRid) !== pending.targetNodeRid) {
|
|
907
|
+
pending.reject(new service_wire_m6a_codec_2.ServiceWireProtocolError(`Session relocation ACK '${key}' source node changed.`));
|
|
831
908
|
return;
|
|
832
909
|
}
|
|
833
910
|
try {
|
|
@@ -846,44 +923,52 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
846
923
|
return undefined;
|
|
847
924
|
const node = this.requireMeshNode(meshName);
|
|
848
925
|
const status = node.status();
|
|
849
|
-
if (
|
|
926
|
+
if ((0, routing_id_1.routingIdsEqual)(status.routingId, sourceNodeRid)) {
|
|
850
927
|
return status.lifecycleGeneration;
|
|
851
928
|
}
|
|
852
929
|
return node
|
|
853
930
|
.peers()
|
|
854
|
-
.find((peer) => peer.routingId !== null &&
|
|
931
|
+
.find((peer) => peer.routingId !== null && (0, routing_id_1.routingIdsEqual)(peer.routingId, sourceNodeRid))
|
|
855
932
|
?.lifecycleGeneration;
|
|
856
933
|
}
|
|
857
934
|
async handleSessionRelocationSeal(meshName, request, sourceNodeRid, signal) {
|
|
858
935
|
if (sourceNodeRid === null ||
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
throw new
|
|
936
|
+
!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, request.coordinator.nodeRid) ||
|
|
937
|
+
!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, request.actor.actor.nodeRid)) {
|
|
938
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation seal source does not match its coordinator and Actor fence.');
|
|
862
939
|
}
|
|
863
940
|
this.validateSessionOwnerFence(meshName, request.session);
|
|
864
941
|
const sealSourceGeneration = this.sourceLifecycleGeneration(meshName, sourceNodeRid);
|
|
865
942
|
if (sealSourceGeneration !== request.actor.targetNodeGeneration ||
|
|
866
943
|
sealSourceGeneration !== request.coordinator.nodeGeneration) {
|
|
867
|
-
throw new
|
|
944
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation seal source lifecycle generation is stale.');
|
|
868
945
|
}
|
|
869
946
|
const handler = this.options.boundSessionRelocation?.receiveSeal;
|
|
870
947
|
if (handler === undefined) {
|
|
871
948
|
throw new Error('Session relocation seal ingress is not configured.');
|
|
872
949
|
}
|
|
873
|
-
return await handler(request, signal)
|
|
950
|
+
return await handler(request, signal, () => {
|
|
951
|
+
const node = this.requireMeshNode(meshName);
|
|
952
|
+
if ((0, routing_id_1.routingIdsEqual)(node.status().routingId, request.targetNodeRid) ||
|
|
953
|
+
node.isPeerRouteReady?.(request.targetNodeRid, request.targetNodeGeneration) === true)
|
|
954
|
+
return;
|
|
955
|
+
void this.options
|
|
956
|
+
.requestPeerReconcile?.(meshName, request.targetNodeRid)
|
|
957
|
+
.catch((error) => console.error('[zlink.runtime.relocation.session_route_reconcile_failed]', error));
|
|
958
|
+
});
|
|
874
959
|
}
|
|
875
960
|
async handleSessionRelocationRoute(meshName, request, sourceNodeRid, _signal) {
|
|
876
961
|
this.validateSessionOwnerFence(meshName, request.session);
|
|
877
962
|
const expectedSource = request.route.action === 'commit' ? request.route.targetNodeRid : request.coordinator.nodeRid;
|
|
878
|
-
if (sourceNodeRid === null ||
|
|
879
|
-
throw new
|
|
963
|
+
if (sourceNodeRid === null || !(0, routing_id_1.routingIdsEqual)(sourceNodeRid, expectedSource)) {
|
|
964
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation route source does not match its action fence.');
|
|
880
965
|
}
|
|
881
966
|
const routeSourceGeneration = this.sourceLifecycleGeneration(meshName, sourceNodeRid);
|
|
882
967
|
const expectedSourceGeneration = request.route.action === 'commit'
|
|
883
968
|
? request.route.targetNodeGeneration
|
|
884
969
|
: request.coordinator.nodeGeneration;
|
|
885
970
|
if (routeSourceGeneration !== expectedSourceGeneration) {
|
|
886
|
-
throw new
|
|
971
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation route source lifecycle generation is stale.');
|
|
887
972
|
}
|
|
888
973
|
const handler = this.options.boundSessionRelocation?.receiveRoute;
|
|
889
974
|
if (handler === undefined) {
|
|
@@ -894,16 +979,20 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
894
979
|
validateSessionOwnerFence(meshName, session) {
|
|
895
980
|
const status = this.requireMeshNode(meshName).status();
|
|
896
981
|
const owner = this.options.currentOwner();
|
|
897
|
-
|
|
982
|
+
// Session-Actor binding §4: without this host's own descriptor the command is Unavailable.
|
|
983
|
+
if (owner === undefined) {
|
|
984
|
+
throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected, 'Session owner has no live MeshNode descriptor.');
|
|
985
|
+
}
|
|
986
|
+
if (!(0, routing_id_1.routingIdsEqual)(status.routingId, session.sessionOwnerNodeRid) ||
|
|
898
987
|
status.lifecycleGeneration !== session.sessionOwnerNodeGeneration ||
|
|
899
|
-
owner
|
|
988
|
+
owner.ownerId !== session.sessionOwnerId ||
|
|
900
989
|
owner.leaseGeneration !== session.sessionOwnerLeaseGeneration) {
|
|
901
|
-
throw new
|
|
990
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation command does not match the local Session owner fence.');
|
|
902
991
|
}
|
|
903
992
|
}
|
|
904
993
|
async sendSessionRelocationResponse(meshName, targetNodeRid, bytes) {
|
|
905
994
|
if (targetNodeRid === null) {
|
|
906
|
-
throw new
|
|
995
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation command has no authenticated source.');
|
|
907
996
|
}
|
|
908
997
|
const submitted = await this.sendInfrastructureControl(meshName, targetNodeRid, bytes);
|
|
909
998
|
if (submitted !== runtime_values_1.SubmitResult.Ok) {
|
|
@@ -928,7 +1017,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
928
1017
|
let readinessBoundaryConsumed = false;
|
|
929
1018
|
let interruptionStartedAt;
|
|
930
1019
|
const spotRegistration = this.spotRegistration(meshName, kind, spotAuthority.allocation.stableType);
|
|
931
|
-
target ??= await this.selectTarget(meshName,
|
|
1020
|
+
target ??= await this.selectTarget(meshName, this.requireMeshNode(meshName).status().routingId, targetApplicationVersion, relocationTargetRequirements([
|
|
932
1021
|
{
|
|
933
1022
|
objectKind: kind,
|
|
934
1023
|
stableType: spotAuthority.allocation.stableType,
|
|
@@ -953,23 +1042,32 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
953
1042
|
objectGeneration: spotAuthority.objectGeneration,
|
|
954
1043
|
authorityOwnerGeneration: spotAuthority.authorityOwnerGeneration,
|
|
955
1044
|
seal: async (captureSignal) => {
|
|
1045
|
+
// Relocation 04 §4.1: every bound member's Session is sealed before
|
|
1046
|
+
// the Spot capture stops member dispatch.
|
|
1047
|
+
const preparations = await Promise.allSettled(actorStates.map(async (state) => ({
|
|
1048
|
+
state,
|
|
1049
|
+
actor: state.actor,
|
|
1050
|
+
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, captureSignal, false, relocationWireId(aggregateId), target)
|
|
1051
|
+
})));
|
|
1052
|
+
const failures = [];
|
|
1053
|
+
for (const preparation of preparations) {
|
|
1054
|
+
if (preparation.status === 'fulfilled')
|
|
1055
|
+
sessions.push(preparation.value);
|
|
1056
|
+
else
|
|
1057
|
+
failures.push(preparation.reason);
|
|
1058
|
+
}
|
|
1059
|
+
if (failures.length === 1)
|
|
1060
|
+
throw failures[0];
|
|
1061
|
+
if (failures.length > 1) {
|
|
1062
|
+
throw new AggregateError(failures, 'Spot relocation source preparation failed.');
|
|
1063
|
+
}
|
|
956
1064
|
const { spotCaptureOperation } = await this.afterSpotCloseDecision(meshName, activation.spotId, () => {
|
|
957
1065
|
interruptionStartedAt = performance.now();
|
|
958
1066
|
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
959
|
-
// Both seals run in the same event-loop turn after the Close decision.
|
|
1067
|
+
// Both Spot seals run in the same event-loop turn after the Close decision.
|
|
960
1068
|
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
961
1069
|
});
|
|
962
|
-
const preparedSessions = Promise.all(actorStates.map(async (state) => ({
|
|
963
|
-
state,
|
|
964
|
-
actor: state.actor,
|
|
965
|
-
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, captureSignal, false, relocationWireId(aggregateId))
|
|
966
|
-
})));
|
|
967
|
-
const preparedSessionsOutcome = preparedSessions.then((value) => ({ value }), (error) => ({ error }));
|
|
968
1070
|
spotCapture = await spotCaptureOperation;
|
|
969
|
-
const preparation = await preparedSessionsOutcome;
|
|
970
|
-
if ('error' in preparation)
|
|
971
|
-
throw preparation.error;
|
|
972
|
-
sessions.push(...preparation.value);
|
|
973
1071
|
return {
|
|
974
1072
|
boundSessionState: Buffer.alloc(0),
|
|
975
1073
|
queuedMessages: [],
|
|
@@ -977,7 +1075,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
977
1075
|
};
|
|
978
1076
|
},
|
|
979
1077
|
captureApplicationState: (captureSignal) => this.captureApplication(spotRegistration.relocation, activation.spot, captureSignal),
|
|
980
|
-
commitSeal: async () => {
|
|
1078
|
+
commitSeal: async (followAuthorities) => {
|
|
981
1079
|
const failures = [];
|
|
982
1080
|
try {
|
|
983
1081
|
if (!spotSealCommitted) {
|
|
@@ -987,7 +1085,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
987
1085
|
spotSealCommitted = true;
|
|
988
1086
|
}
|
|
989
1087
|
if (spotMessageFollowSeal !== undefined) {
|
|
990
|
-
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, spotKey, activation
|
|
1088
|
+
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, followAuthorities.get(spotKey.value), activation);
|
|
991
1089
|
}
|
|
992
1090
|
}
|
|
993
1091
|
catch (error) {
|
|
@@ -995,20 +1093,17 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
995
1093
|
}
|
|
996
1094
|
for (const session of sessions) {
|
|
997
1095
|
try {
|
|
998
|
-
const committedAuthority =
|
|
1096
|
+
const committedAuthority = followAuthorities.get((0, authority_key_codec_1.encodeAuthorityKey)('actor', session.state.actorId).value);
|
|
1097
|
+
const committedSpot = followAuthorities.get(spotKey.value);
|
|
999
1098
|
const targetActorRef = {
|
|
1000
1099
|
actorId: session.state.actorId,
|
|
1001
1100
|
objectGeneration: actorAuthorities.get(session.state.actorId).objectGeneration,
|
|
1002
1101
|
meshName,
|
|
1003
1102
|
nodeRid: target.rid
|
|
1004
1103
|
};
|
|
1005
|
-
await session.prepared.commit(
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
spotId: activation.spotId,
|
|
1009
|
-
spotKind: kind === 'user_spot' ? contracts_1.ZLinkSpotKind.User : contracts_1.ZLinkSpotKind.Instance,
|
|
1010
|
-
authorityOwnerGeneration: committedAuthority.authorityOwnerGeneration
|
|
1011
|
-
}, targetActorRef, (0, actor_transfer_runtime_1.committedActorOwnerFence)(session.state.actorId, targetActorRef, committedAuthority));
|
|
1104
|
+
await session.prepared.commit(committedSpotRoute(meshName, activation.spotId, kind === 'user_spot' ? contracts_1.ZLinkSpotKind.User : contracts_1.ZLinkSpotKind.Instance, target.rid, committedSpot), targetActorRef, committedAuthority === undefined || committedSpot === undefined
|
|
1105
|
+
? null
|
|
1106
|
+
: (0, actor_transfer_runtime_1.committedActorOwnerFence)(session.state.actorId, targetActorRef, committedAuthority));
|
|
1012
1107
|
}
|
|
1013
1108
|
catch (error) {
|
|
1014
1109
|
failures.push(error);
|
|
@@ -1040,6 +1135,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1040
1135
|
abortSeal: async () => {
|
|
1041
1136
|
const failures = [];
|
|
1042
1137
|
if (spotCapture !== undefined) {
|
|
1138
|
+
// Spot model §5.1: Continued completes while the source seal still holds jobs.
|
|
1139
|
+
if (readinessBoundaryConsumed) {
|
|
1140
|
+
try {
|
|
1141
|
+
await activation.completeConsumedRelocationBoundary(contracts_1.ZLinkSpotRelocationReadyOutcome.Continued);
|
|
1142
|
+
}
|
|
1143
|
+
catch (error) {
|
|
1144
|
+
failures.push(error);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1043
1147
|
try {
|
|
1044
1148
|
activation.abortRelocation(spotCapture);
|
|
1045
1149
|
}
|
|
@@ -1079,7 +1183,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1079
1183
|
spotObjectGeneration: spotAuthority.objectGeneration,
|
|
1080
1184
|
membershipEpoch: state.spotMembershipEpoch > 0n ? state.spotMembershipEpoch : 1n
|
|
1081
1185
|
}));
|
|
1082
|
-
|
|
1186
|
+
const relocationMetric = spotRegistration.relocation.kind === 'disabled'
|
|
1187
|
+
? undefined
|
|
1188
|
+
: this.options.metrics?.startRelocation(meshName, kind, spotRegistration.relocation.kind);
|
|
1083
1189
|
try {
|
|
1084
1190
|
const waitForRelocationBoundary = activation.waitForRelocationBoundary;
|
|
1085
1191
|
readinessBoundaryConsumed =
|
|
@@ -1099,10 +1205,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1099
1205
|
(0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId).value,
|
|
1100
1206
|
actorAuthorities.get(state.actorId)
|
|
1101
1207
|
])
|
|
1102
|
-
]), signal, sessions);
|
|
1208
|
+
]), signal, sessions, undefined, undefined, relocationMetric);
|
|
1103
1209
|
}
|
|
1104
1210
|
catch (error) {
|
|
1105
|
-
outcome = 'failed';
|
|
1106
1211
|
if (readinessBoundaryConsumed &&
|
|
1107
1212
|
!(error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) &&
|
|
1108
1213
|
!(error instanceof service_relocation_coordinator_1.ServiceRelocationAuthorityError)) {
|
|
@@ -1113,34 +1218,33 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1113
1218
|
throw error;
|
|
1114
1219
|
}
|
|
1115
1220
|
finally {
|
|
1221
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1116
1222
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1117
1223
|
if (interruptionStartedAt !== undefined) {
|
|
1118
|
-
this.recordRelocationInterruption(meshName, kind, kind === 'user_spot' ? 'spot_wide' : undefined, interruptionStartedAt
|
|
1224
|
+
this.recordRelocationInterruption(meshName, kind, kind === 'user_spot' ? 'spot_wide' : undefined, interruptionStartedAt);
|
|
1119
1225
|
}
|
|
1120
1226
|
}
|
|
1121
1227
|
}
|
|
1122
1228
|
async relocateStandaloneActor(meshName, state, target, targetApplicationVersion, signal, targetMembership, aggregateIdOverride, retainSourceForActorJoin = false,
|
|
1123
1229
|
/** Target's advertised relocation state chunk cap (Actor Join only; spec 15 §4.2). */
|
|
1124
|
-
advertisedReceiveChunkLimitBytes) {
|
|
1230
|
+
advertisedReceiveChunkLimitBytes, deadlineAtMs) {
|
|
1125
1231
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1126
1232
|
(0, diagnostics_1.relocationDebug)('standalone_actor.begin', { actorId: state.actorId });
|
|
1127
1233
|
}
|
|
1128
1234
|
const authority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId), signal);
|
|
1129
|
-
|
|
1235
|
+
const actorPolicy = this.actorRegistration(state.meshName ?? meshName, authority.allocation.stableType).relocation.kind;
|
|
1236
|
+
target ??= await this.selectTarget(meshName, this.requireMeshNode(meshName).status().routingId, targetApplicationVersion, [
|
|
1130
1237
|
{
|
|
1131
1238
|
objectKind: 'actor',
|
|
1132
1239
|
stableType: authority.allocation.stableType,
|
|
1133
|
-
policy:
|
|
1240
|
+
policy: actorPolicy,
|
|
1134
1241
|
count: 1
|
|
1135
1242
|
}
|
|
1136
1243
|
], signal);
|
|
1137
1244
|
const sessions = [];
|
|
1138
1245
|
const membershipSpotId = targetMembership?.spotId ?? (target.entrySpotId ?? String(target.rid));
|
|
1139
1246
|
const aggregateId = aggregateIdOverride ?? this.reserveRelocationId();
|
|
1140
|
-
const unit = this.actorCaptureUnit(state, authority, sessions, true, target, meshName,
|
|
1141
|
-
spotId: membershipSpotId,
|
|
1142
|
-
spotKind: targetMembership?.spotKind ?? contracts_1.ZLinkSpotKind.Entry
|
|
1143
|
-
}, relocationWireId(aggregateId), retainSourceForActorJoin);
|
|
1247
|
+
const unit = this.actorCaptureUnit(state, authority, sessions, true, target, meshName, targetMembership, relocationWireId(aggregateId), retainSourceForActorJoin);
|
|
1144
1248
|
const membership = {
|
|
1145
1249
|
actorKey: (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId).value,
|
|
1146
1250
|
spotKey: (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(membershipSpotId)).value,
|
|
@@ -1155,7 +1259,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1155
1259
|
return unit.seal(captureSignal);
|
|
1156
1260
|
}
|
|
1157
1261
|
};
|
|
1158
|
-
|
|
1262
|
+
const relocationMetric = actorPolicy === 'disabled'
|
|
1263
|
+
? undefined
|
|
1264
|
+
: this.options.metrics?.startRelocation(meshName, 'actor', actorPolicy);
|
|
1159
1265
|
try {
|
|
1160
1266
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1161
1267
|
(0, diagnostics_1.relocationDebug)('standalone_actor.wait_budget', {
|
|
@@ -1171,16 +1277,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1171
1277
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1172
1278
|
(0, diagnostics_1.relocationDebug)('standalone_actor.capture_complete', { actorId: state.actorId });
|
|
1173
1279
|
}
|
|
1174
|
-
await this.runCoordinator(meshName, target, authority, captured, new Map([[measuredUnit.authorityKey, authority]]), signal, sessions, advertisedReceiveChunkLimitBytes);
|
|
1175
|
-
}
|
|
1176
|
-
catch (error) {
|
|
1177
|
-
outcome = 'failed';
|
|
1178
|
-
throw error;
|
|
1280
|
+
await this.runCoordinator(meshName, target, authority, captured, new Map([[measuredUnit.authorityKey, authority]]), signal, sessions, advertisedReceiveChunkLimitBytes, deadlineAtMs, relocationMetric);
|
|
1179
1281
|
}
|
|
1180
1282
|
finally {
|
|
1283
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1181
1284
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1182
1285
|
if (interruptionStartedAt !== undefined) {
|
|
1183
|
-
this.recordRelocationInterruption(meshName, 'actor', undefined, interruptionStartedAt
|
|
1286
|
+
this.recordRelocationInterruption(meshName, 'actor', undefined, interruptionStartedAt);
|
|
1184
1287
|
}
|
|
1185
1288
|
}
|
|
1186
1289
|
}
|
|
@@ -1191,7 +1294,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1191
1294
|
const spotKey = (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(activation.spotId));
|
|
1192
1295
|
const spotAuthority = await requireAuthority(this.requireLocationStore(), spotKey, signal);
|
|
1193
1296
|
const spotRegistration = this.spotRegistration(meshName, 'user_spot', spotAuthority.allocation.stableType);
|
|
1194
|
-
target ??= await this.selectTarget(meshName,
|
|
1297
|
+
target ??= await this.selectTarget(meshName, this.requireMeshNode(meshName).status().routingId, targetApplicationVersion, [
|
|
1195
1298
|
{
|
|
1196
1299
|
objectKind: 'user_spot',
|
|
1197
1300
|
stableType: spotAuthority.allocation.stableType,
|
|
@@ -1225,7 +1328,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1225
1328
|
// PerActor User Spots are stateless shells. Application state and the
|
|
1226
1329
|
// Spot relocation adapter are intentionally not part of this unit.
|
|
1227
1330
|
captureApplicationState: async () => Buffer.alloc(0),
|
|
1228
|
-
commitSeal: async () => {
|
|
1331
|
+
commitSeal: async (followAuthorities) => {
|
|
1229
1332
|
if (!sealCommitted) {
|
|
1230
1333
|
if (spotCapture === undefined || !(await activation.commitRelocation(spotCapture))) {
|
|
1231
1334
|
throw new Error(`PerActor Spot '${String(activation.spotId)}' relocation seal became stale.`);
|
|
@@ -1233,7 +1336,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1233
1336
|
sealCommitted = true;
|
|
1234
1337
|
}
|
|
1235
1338
|
if (spotMessageFollowSeal !== undefined) {
|
|
1236
|
-
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, spotKey, activation
|
|
1339
|
+
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, followAuthorities.get(spotKey.value), activation);
|
|
1237
1340
|
}
|
|
1238
1341
|
await this.requireSpotManager().completeRelocationSource(activation);
|
|
1239
1342
|
},
|
|
@@ -1250,20 +1353,19 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1250
1353
|
}
|
|
1251
1354
|
};
|
|
1252
1355
|
const aggregateId = this.reserveRelocationId();
|
|
1253
|
-
|
|
1356
|
+
const relocationMetric = spotRegistration.relocation.kind === 'disabled'
|
|
1357
|
+
? undefined
|
|
1358
|
+
: this.options.metrics?.startRelocation(meshName, 'user_spot', spotRegistration.relocation.kind);
|
|
1254
1359
|
try {
|
|
1255
1360
|
await this.waitForRelocationBudgetHeadroom(String(target.rid), signal);
|
|
1256
1361
|
const captured = await new service_relocation_object_owner_1.ServiceRelocationObjectCaptureOwner().captureUserSpotAggregate(aggregateId, 1n, spotUnit, [], [], signal);
|
|
1257
|
-
await this.runCoordinator(meshName, target, spotAuthority, captured, new Map([[spotKey.value, spotAuthority]]), signal, []);
|
|
1258
|
-
}
|
|
1259
|
-
catch (error) {
|
|
1260
|
-
outcome = 'failed';
|
|
1261
|
-
throw error;
|
|
1362
|
+
await this.runCoordinator(meshName, target, spotAuthority, captured, new Map([[spotKey.value, spotAuthority]]), signal, [], undefined, undefined, relocationMetric);
|
|
1262
1363
|
}
|
|
1263
1364
|
finally {
|
|
1365
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1264
1366
|
this.sourceRelocationIds.delete(aggregateId);
|
|
1265
1367
|
if (interruptionStartedAt !== undefined) {
|
|
1266
|
-
this.recordRelocationInterruption(meshName, 'user_spot', 'per_actor', interruptionStartedAt
|
|
1368
|
+
this.recordRelocationInterruption(meshName, 'user_spot', 'per_actor', interruptionStartedAt);
|
|
1267
1369
|
}
|
|
1268
1370
|
}
|
|
1269
1371
|
}
|
|
@@ -1281,22 +1383,35 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1281
1383
|
ownSession = {
|
|
1282
1384
|
state,
|
|
1283
1385
|
actor: state.actor,
|
|
1284
|
-
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, signal, true, sessionRelocation)
|
|
1386
|
+
prepared: await this.options.actorTransfer.prepareMaintenanceSession(state.actor, state, signal, true, sessionRelocation, target)
|
|
1285
1387
|
};
|
|
1286
1388
|
sessions.push(ownSession);
|
|
1287
1389
|
}
|
|
1288
1390
|
const prepared = standalone ? ownSession : sessions.find((value) => value.state === state);
|
|
1289
1391
|
return {
|
|
1290
|
-
boundSessionState:
|
|
1392
|
+
boundSessionState: Buffer.alloc(0),
|
|
1393
|
+
...(prepared?.prepared.target?.serviceWireRelocation === undefined
|
|
1394
|
+
? {}
|
|
1395
|
+
: {
|
|
1396
|
+
sessionBinding: {
|
|
1397
|
+
participantId: 1n,
|
|
1398
|
+
sessionOwnerNodeRid: Buffer.from((0, routing_id_1.encodeRoutingIdStorageHex)(prepared.prepared.target.sessionNodeRid), 'hex'),
|
|
1399
|
+
sessionOwnerNodeGeneration: prepared.prepared.target.serviceWireRelocation.session
|
|
1400
|
+
.sessionOwnerNodeGeneration,
|
|
1401
|
+
sessionOwnerId: prepared.prepared.target.serviceWireRelocation.session.sessionOwnerId,
|
|
1402
|
+
sessionOwnerLeaseGeneration: prepared.prepared.target.serviceWireRelocation.session
|
|
1403
|
+
.sessionOwnerLeaseGeneration,
|
|
1404
|
+
sessionRid: Buffer.from((0, routing_id_1.encodeRoutingIdStorageHex)(prepared.prepared.target.sessionRid), 'hex'),
|
|
1405
|
+
bindingGeneration: prepared.prepared.target.serviceWireRelocation.session.bindingGeneration
|
|
1406
|
+
}
|
|
1407
|
+
}),
|
|
1291
1408
|
queuedMessages: encodeHandoffQueuedMessages(prepared?.prepared.handoffBacklog ?? []),
|
|
1292
1409
|
timers: []
|
|
1293
1410
|
};
|
|
1294
1411
|
},
|
|
1295
|
-
// The
|
|
1296
|
-
// state. The sealed bound-session target remains in sealed work and is
|
|
1297
|
-
// transported only by Prepare's NODE-INTERNAL sideband frames.
|
|
1412
|
+
// The sealed Session identity is part of the canonical participant vector.
|
|
1298
1413
|
captureApplicationState: (signal) => this.captureApplication(registration.relocation, state.actor, signal),
|
|
1299
|
-
commitSeal: async () => {
|
|
1414
|
+
commitSeal: async (followAuthorities) => {
|
|
1300
1415
|
if (!standalone ||
|
|
1301
1416
|
ownSession === undefined ||
|
|
1302
1417
|
target === undefined ||
|
|
@@ -1304,33 +1419,36 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1304
1419
|
return;
|
|
1305
1420
|
const failures = [];
|
|
1306
1421
|
try {
|
|
1307
|
-
const
|
|
1308
|
-
const committedAuthority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId));
|
|
1422
|
+
const committedAuthority = followAuthorities.get((0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId).value);
|
|
1309
1423
|
const targetActorRef = {
|
|
1310
1424
|
actorId: state.actorId,
|
|
1311
1425
|
objectGeneration: authority.objectGeneration,
|
|
1312
1426
|
meshName,
|
|
1313
1427
|
nodeRid: target.rid
|
|
1314
1428
|
};
|
|
1315
|
-
await ownSession.prepared.commit({
|
|
1429
|
+
await ownSession.prepared.commit(targetMembership ?? {
|
|
1316
1430
|
routerChannelId: meshName,
|
|
1317
1431
|
targetNodeRid: target.rid,
|
|
1318
|
-
spotId:
|
|
1319
|
-
spotKind:
|
|
1320
|
-
|
|
1321
|
-
|
|
1432
|
+
spotId: (target.entrySpotId ?? String(target.rid)),
|
|
1433
|
+
spotKind: contracts_1.ZLinkSpotKind.Entry
|
|
1434
|
+
}, targetActorRef, committedAuthority === undefined
|
|
1435
|
+
? null
|
|
1436
|
+
: (0, actor_transfer_runtime_1.committedActorOwnerFence)(state.actorId, targetActorRef, committedAuthority));
|
|
1322
1437
|
}
|
|
1323
1438
|
catch (error) {
|
|
1324
1439
|
failures.push(error);
|
|
1325
1440
|
}
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1441
|
+
try {
|
|
1442
|
+
if (retainSourceForActorJoin) {
|
|
1443
|
+
this.requireActorManager().detachRelocationSource(state);
|
|
1329
1444
|
}
|
|
1330
|
-
|
|
1331
|
-
|
|
1445
|
+
else {
|
|
1446
|
+
await this.requireActorManager().completeRelocationSource(state.actorId);
|
|
1332
1447
|
}
|
|
1333
1448
|
}
|
|
1449
|
+
catch (error) {
|
|
1450
|
+
failures.push(error);
|
|
1451
|
+
}
|
|
1334
1452
|
if (failures.length > 0)
|
|
1335
1453
|
throw failures.length === 1
|
|
1336
1454
|
? failures[0]
|
|
@@ -1367,7 +1485,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1367
1485
|
}
|
|
1368
1486
|
async runCoordinator(meshName, target, primary, captured, _authorities, signal, sessions = [],
|
|
1369
1487
|
/** Target's advertised relocation state chunk cap (Actor Join only; spec 15 §4.2). */
|
|
1370
|
-
advertisedReceiveChunkLimitBytes) {
|
|
1488
|
+
advertisedReceiveChunkLimitBytes, controlDeadlineAtMs = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS, relocationMetric) {
|
|
1371
1489
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1372
1490
|
(0, diagnostics_1.relocationDebug)('coordinator.begin', {
|
|
1373
1491
|
aggregateId: captured.envelope.aggregateId,
|
|
@@ -1375,18 +1493,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1375
1493
|
});
|
|
1376
1494
|
}
|
|
1377
1495
|
const localStatus = this.requireMeshNode(meshName).status();
|
|
1378
|
-
const controlDeadlineAtMs = signal === undefined
|
|
1379
|
-
? performance.now() + RELOCATION_OPERATION_TIMEOUT_MS
|
|
1380
|
-
: Number.MAX_SAFE_INTEGER;
|
|
1381
1496
|
const coordinator = {
|
|
1382
1497
|
ownerId: primary.ownerId,
|
|
1383
1498
|
leaseGeneration: primary.ownerLeaseGeneration,
|
|
1384
|
-
nodeRid:
|
|
1499
|
+
nodeRid: localStatus.routingId,
|
|
1385
1500
|
nodeGeneration: localStatus.lifecycleGeneration,
|
|
1386
1501
|
expectedAuthorityStoreVersion: primary.storeVersion.value
|
|
1387
1502
|
};
|
|
1388
1503
|
const targetFence = {
|
|
1389
|
-
nodeRid:
|
|
1504
|
+
nodeRid: target.rid,
|
|
1390
1505
|
nodeGeneration: target.lifecycleGeneration,
|
|
1391
1506
|
ownerId: target.ownerId,
|
|
1392
1507
|
ownerLeaseGeneration: target.leaseGeneration
|
|
@@ -1399,11 +1514,6 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1399
1514
|
// attempt even when it reuses the same routing id.
|
|
1400
1515
|
const targetAttemptGeneration = target.lifecycleGeneration;
|
|
1401
1516
|
const relayAuthorityId = wireIdText(relocationWireId(captured.envelope.aggregateId));
|
|
1402
|
-
const relayAuthorityKey = primaryKey(captured.envelope);
|
|
1403
|
-
if (this.relocationAuthorityKeys.has(relayAuthorityId)) {
|
|
1404
|
-
throw new Error(`Relocation reply relay authority '${relayAuthorityId}' is already registered.`);
|
|
1405
|
-
}
|
|
1406
|
-
this.relocationAuthorityKeys.set(relayAuthorityId, relayAuthorityKey);
|
|
1407
1517
|
// The captured payload stays in source memory: this copy is the only
|
|
1408
1518
|
// handoff original until the cutover submit terminal and the
|
|
1409
1519
|
// retransmission window end (spec 28 §4.2).
|
|
@@ -1413,6 +1523,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1413
1523
|
? canonicalEnvelope
|
|
1414
1524
|
: appendCanonicalActorJoinRecovery(canonicalEnvelope, actorJoinProfile, coordinator, primary);
|
|
1415
1525
|
const encoded = (0, service_relocation_runtime_1.encodeServiceRelocationEnvelope)(transportEnvelope, target.applicationVersion);
|
|
1526
|
+
relocationMetric?.recordBytes(encoded.byteLength);
|
|
1416
1527
|
const limits = this.relocationLimits();
|
|
1417
1528
|
// Actor Join threads the target's admission-time advertised cap in;
|
|
1418
1529
|
// other relocation paths have no such negotiation and fall back to the
|
|
@@ -1440,29 +1551,24 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1440
1551
|
payloadTotalLength: BigInt(plan.totalLength),
|
|
1441
1552
|
payloadChunkCount: plan.chunkCount,
|
|
1442
1553
|
payloadChecksumCrc32c: plan.checksumCrc32c,
|
|
1443
|
-
applicationVersion: target.applicationVersion
|
|
1554
|
+
applicationVersion: target.applicationVersion,
|
|
1555
|
+
deadlineUnixMs: BigInt(Date.now() + Math.max(0, Math.ceil(controlDeadlineAtMs - performance.now())))
|
|
1444
1556
|
};
|
|
1445
1557
|
// Start the Restore request first (it also registers the temporary
|
|
1446
1558
|
// queue and the assembly on the target), then stream the payload chunks
|
|
1447
1559
|
// on the same ordered connection. The Ready reply arrives only after
|
|
1448
1560
|
// the target assembled, verified, and restored the payload.
|
|
1449
|
-
const
|
|
1450
|
-
|
|
1451
|
-
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1561
|
+
const ready = await this.sendControl(meshName, target.rid, prepare, signal, controlDeadlineAtMs, async (attemptSignal) => {
|
|
1562
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)())
|
|
1452
1563
|
(0, diagnostics_1.relocationDebug)('coordinator.state_transfer_begin', {
|
|
1453
1564
|
aggregateId: captured.envelope.aggregateId
|
|
1454
1565
|
});
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
}
|
|
1462
|
-
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1463
|
-
(0, diagnostics_1.relocationDebug)('coordinator.await_ready', { aggregateId: captured.envelope.aggregateId });
|
|
1464
|
-
}
|
|
1465
|
-
const ready = await readyOperation;
|
|
1566
|
+
await this.sendRelocationStateChunks(meshName, target, prepare, encoded, plan, attemptSignal);
|
|
1567
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)())
|
|
1568
|
+
(0, diagnostics_1.relocationDebug)('coordinator.await_ready', {
|
|
1569
|
+
aggregateId: captured.envelope.aggregateId
|
|
1570
|
+
});
|
|
1571
|
+
});
|
|
1466
1572
|
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1467
1573
|
(0, diagnostics_1.relocationDebug)('coordinator.ready_received', {
|
|
1468
1574
|
aggregateId: captured.envelope.aggregateId
|
|
@@ -1473,6 +1579,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1473
1579
|
// forward the source can no longer reopen dispatch without risking two
|
|
1474
1580
|
// owners.
|
|
1475
1581
|
readyReceived = true;
|
|
1582
|
+
actorJoinProfile?.relayReady();
|
|
1476
1583
|
const boundaryBatch = [];
|
|
1477
1584
|
for (const session of sessions) {
|
|
1478
1585
|
const participant = captured.envelope.participants.find((value) => value.objectKind === 'actor' &&
|
|
@@ -1511,7 +1618,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1511
1618
|
// The cutover submit result is not the authority result. The memory
|
|
1512
1619
|
// copy of the boundary batch and cutover stays until the authority
|
|
1513
1620
|
// settlement below (spec 28 §4.4).
|
|
1514
|
-
|
|
1621
|
+
const submittedRouteGeneration = this.requireMeshNode(meshName).observePeerRoute?.(target.rid).generation;
|
|
1515
1622
|
try {
|
|
1516
1623
|
for (const record of boundaryBatch) {
|
|
1517
1624
|
await this.submitControlFrame(meshName, target.rid, record.frame, 'relay');
|
|
@@ -1519,12 +1626,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1519
1626
|
await this.submitControlFrame(meshName, target.rid, cutoverFrame, 'cutover');
|
|
1520
1627
|
}
|
|
1521
1628
|
catch (error) {
|
|
1522
|
-
|
|
1629
|
+
if (!(0, durable_service_request_1.durableRequestCanReplay)(error))
|
|
1630
|
+
throw error;
|
|
1523
1631
|
console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
|
|
1524
1632
|
}
|
|
1525
|
-
const window = this.beginSourceCutoverWindow(meshName, target.rid, relayAuthorityId, [...boundaryBatch.map((value) => value.frame), cutoverFrame],
|
|
1633
|
+
const window = this.beginSourceCutoverWindow(meshName, target.rid, relayAuthorityId, [...boundaryBatch.map((value) => value.frame), cutoverFrame], submittedRouteGeneration, limits);
|
|
1526
1634
|
try {
|
|
1527
|
-
settlement = await this.settleSourceAuthority(captured.envelope, primary, targetFence, restoreDeadlineReached, window);
|
|
1635
|
+
settlement = await this.settleSourceAuthority(captured.envelope, primary, targetFence, restoreDeadlineReached, window, relocationMetric);
|
|
1528
1636
|
}
|
|
1529
1637
|
finally {
|
|
1530
1638
|
this.closeSourceCutoverWindow(relayAuthorityId, window);
|
|
@@ -1539,22 +1647,31 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1539
1647
|
// Preserve won: the captured queue, timers and held records return to
|
|
1540
1648
|
// the source queue in their accepted order, and dispatch reopens.
|
|
1541
1649
|
await captured.abortSource();
|
|
1650
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
1542
1651
|
throw new Error(`Relocation '${relayAuthorityId}' ended by the source Preserve fence; the source kept the authority.`);
|
|
1543
1652
|
}
|
|
1544
1653
|
if (settlement === 'lost') {
|
|
1654
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
1545
1655
|
// Expired owner: dispatch and Store changes stop for good and every
|
|
1546
1656
|
// pending request without a terminal gets one Unavailable.
|
|
1547
|
-
const unavailable = (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected, `Relocation '${relayAuthorityId}' source owner lease
|
|
1657
|
+
const unavailable = (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected, `Relocation '${relayAuthorityId}' source authority ended (source owner lease expired, authority Missing, or owner changed).`, true);
|
|
1548
1658
|
await captured.discardSource(unavailable);
|
|
1549
1659
|
throw new service_relocation_coordinator_1.ServiceRelocationAuthorityError(unavailable.message, { cause: unavailable });
|
|
1550
1660
|
}
|
|
1661
|
+
this.completeSourceRelocationMetric(relocationMetric, settlement);
|
|
1551
1662
|
for (const session of sessions)
|
|
1552
1663
|
session.prepared.setReplayResults([]);
|
|
1553
|
-
await captured.commitSource();
|
|
1664
|
+
await captured.commitSource(settlement.followAuthorities);
|
|
1554
1665
|
await this.options.reconcileStatefulAuthorityRoutes?.(signal);
|
|
1555
1666
|
}
|
|
1556
1667
|
catch (error) {
|
|
1557
|
-
if (
|
|
1668
|
+
if (this.disposed &&
|
|
1669
|
+
(error instanceof ServiceRelocationRuntimeStoppedError ||
|
|
1670
|
+
(error instanceof runtime_values_1.ZLinkBackendResultError &&
|
|
1671
|
+
error.result ===
|
|
1672
|
+
(error.operation === 'request' ? runtime_values_1.RequestResult.Terminated : runtime_values_1.SubmitResult.Terminated))))
|
|
1673
|
+
this.completeSourceRelocationMetric(relocationMetric, undefined, true);
|
|
1674
|
+
if (settlement === undefined || typeof settlement === 'string')
|
|
1558
1675
|
throw error;
|
|
1559
1676
|
// Target commit is confirmed; later source cleanup failures cannot move
|
|
1560
1677
|
// the authority back (spec 28 §10).
|
|
@@ -1562,22 +1679,32 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1562
1679
|
throw new service_relocation_coordinator_1.ServiceRelocationPostCommitError(authority, { id: `${captured.envelope.aggregateId}:${captured.envelope.aggregateGeneration}` }, error);
|
|
1563
1680
|
}
|
|
1564
1681
|
finally {
|
|
1682
|
+
this.completeSourceRelocationMetric(relocationMetric);
|
|
1565
1683
|
if (!readyReceived) {
|
|
1566
1684
|
// Only an explicit failure before the Ready reply becomes accepted
|
|
1567
1685
|
// restores the source from the memory payload (spec 28 §9).
|
|
1568
1686
|
await captured.abortSource().catch(() => undefined);
|
|
1569
1687
|
}
|
|
1570
|
-
if (this.relocationAuthorityKeys.get(relayAuthorityId) === relayAuthorityKey) {
|
|
1571
|
-
this.relocationAuthorityKeys.delete(relayAuthorityId);
|
|
1572
|
-
}
|
|
1573
1688
|
}
|
|
1574
1689
|
}
|
|
1690
|
+
completeSourceRelocationMetric(metric, settlement, sourceStopped = false) {
|
|
1691
|
+
metric?.complete(settlement === 'source'
|
|
1692
|
+
? 'aborted'
|
|
1693
|
+
: settlement === 'lost'
|
|
1694
|
+
? 'failed'
|
|
1695
|
+
: settlement !== undefined
|
|
1696
|
+
? 'completed'
|
|
1697
|
+
: sourceStopped
|
|
1698
|
+
? 'shutdown'
|
|
1699
|
+
: 'failed');
|
|
1700
|
+
}
|
|
1575
1701
|
/** Streams the payload chunks of one stage under the effective in-flight budgets. */
|
|
1576
1702
|
async sendRelocationStateChunks(meshName, target, identity, encoded, plan, signal) {
|
|
1577
1703
|
for (let ordinal = 0; ordinal < plan.chunkCount; ordinal += 1) {
|
|
1578
1704
|
const chunkData = (0, relocation_direct_transfer_1.relocationChunkAt)(encoded, plan, ordinal);
|
|
1579
1705
|
const releases = await this.acquireRelocationBudget(String(target.rid), chunkData.byteLength, signal);
|
|
1580
1706
|
try {
|
|
1707
|
+
signal?.throwIfAborted();
|
|
1581
1708
|
await this.sendControlOneWay(meshName, target.rid, {
|
|
1582
1709
|
kind: 'state',
|
|
1583
1710
|
relocation: identity.relocation,
|
|
@@ -1638,18 +1765,21 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1638
1765
|
* window. S4 — the Message Follow route removal point — closes the unit for
|
|
1639
1766
|
* SafeToShutdown.
|
|
1640
1767
|
*/
|
|
1641
|
-
beginSourceCutoverWindow(meshName, targetNodeRid, relocationKey, frames,
|
|
1768
|
+
beginSourceCutoverWindow(meshName, targetNodeRid, relocationKey, frames, submittedRouteGeneration, limits) {
|
|
1642
1769
|
const submittedAt = performance.now();
|
|
1643
1770
|
const window = {
|
|
1644
1771
|
meshName,
|
|
1645
1772
|
targetNodeRid,
|
|
1646
1773
|
frames,
|
|
1647
|
-
|
|
1774
|
+
submittedRouteGeneration,
|
|
1648
1775
|
cutoverSubmittedAtMs: submittedAt,
|
|
1649
1776
|
windowClosed: false,
|
|
1650
1777
|
followExpired: false
|
|
1651
1778
|
};
|
|
1779
|
+
const wasQuiescent = this.isSafeToShutdown();
|
|
1652
1780
|
this.sourceCutoverWindows.set(relocationKey, window);
|
|
1781
|
+
if (wasQuiescent)
|
|
1782
|
+
this.options.sourceQuiescenceChanged?.();
|
|
1653
1783
|
window.followTimer = setTimeout(() => {
|
|
1654
1784
|
window.followExpired = true;
|
|
1655
1785
|
// S1→S4 route convergence: the span the source must keep its Message
|
|
@@ -1671,33 +1801,27 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1671
1801
|
window.followExpired &&
|
|
1672
1802
|
this.sourceCutoverWindows.get(relocationKey) === window) {
|
|
1673
1803
|
this.sourceCutoverWindows.delete(relocationKey);
|
|
1804
|
+
if (this.isSafeToShutdown())
|
|
1805
|
+
this.options.sourceQuiescenceChanged?.();
|
|
1674
1806
|
}
|
|
1675
1807
|
}
|
|
1676
|
-
/**
|
|
1677
|
-
* Sends the whole boundary batch and cutover again after a failed submit
|
|
1678
|
-
* or an observed loss of the target connection (spec 28 §4.4). The target
|
|
1679
|
-
* replaces its partial span with the whole batch.
|
|
1680
|
-
*/
|
|
1808
|
+
/** Relocation flow §4.4: replay the whole batch on a different ready route. */
|
|
1681
1809
|
async resendSourceCutover(window) {
|
|
1682
1810
|
const frames = window.frames;
|
|
1683
1811
|
if (frames === undefined)
|
|
1684
1812
|
return;
|
|
1685
|
-
const
|
|
1686
|
-
|
|
1687
|
-
if (peer !== undefined && peer.state !== service_runtime_contracts_1.MeshPeerRuntimeState.Serving) {
|
|
1688
|
-
window.resendPending = true;
|
|
1689
|
-
return;
|
|
1690
|
-
}
|
|
1691
|
-
if (!window.resendPending)
|
|
1813
|
+
const route = this.requireMeshNode(window.meshName).observePeerRoute?.(window.targetNodeRid);
|
|
1814
|
+
if (route?.ready !== true || route.generation === window.submittedRouteGeneration)
|
|
1692
1815
|
return;
|
|
1816
|
+
window.submittedRouteGeneration = route.generation;
|
|
1693
1817
|
try {
|
|
1694
1818
|
for (const frame of frames) {
|
|
1695
1819
|
await this.submitControlFrame(window.meshName, window.targetNodeRid, frame, 'cutover retransmission');
|
|
1696
1820
|
}
|
|
1697
|
-
window.resendPending = false;
|
|
1698
1821
|
}
|
|
1699
|
-
catch {
|
|
1700
|
-
|
|
1822
|
+
catch (error) {
|
|
1823
|
+
if (!(0, durable_service_request_1.durableRequestCanReplay)(error))
|
|
1824
|
+
throw error;
|
|
1701
1825
|
}
|
|
1702
1826
|
}
|
|
1703
1827
|
/**
|
|
@@ -1708,15 +1832,18 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1708
1832
|
* excludes a late target commit. `target`: the
|
|
1709
1833
|
* target committed. `source`: Preserve fenced the target out while the
|
|
1710
1834
|
* source owner lease was valid. `lost`: the source owner lease ended first
|
|
1711
|
-
* or
|
|
1712
|
-
*
|
|
1835
|
+
* or authority is Missing or fails the same-object, advanced-generation fence.
|
|
1836
|
+
* A committed later owner has no Follow snapshot (Location runtime §10). An uncertain
|
|
1837
|
+
* Store or lease result keeps the work retained and repeats the same decision.
|
|
1713
1838
|
*/
|
|
1714
|
-
async settleSourceAuthority(envelope, primary, target, restoreDeadlineReached, window) {
|
|
1839
|
+
async settleSourceAuthority(envelope, primary, target, restoreDeadlineReached, window, relocationMetric) {
|
|
1715
1840
|
const owner = { ownerId: primary.ownerId, leaseGeneration: primary.ownerLeaseGeneration };
|
|
1716
1841
|
const store = this.requireLocationStore();
|
|
1717
1842
|
for (;;) {
|
|
1718
|
-
if (this.disposed)
|
|
1719
|
-
|
|
1843
|
+
if (this.disposed) {
|
|
1844
|
+
this.completeSourceRelocationMetric(relocationMetric, undefined, true);
|
|
1845
|
+
throw new ServiceRelocationRuntimeStoppedError();
|
|
1846
|
+
}
|
|
1720
1847
|
await this.resendSourceCutover(window);
|
|
1721
1848
|
const observed = await this.readSourceSettlement(envelope, primary, target).catch((error) => {
|
|
1722
1849
|
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error))
|
|
@@ -1724,7 +1851,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1724
1851
|
return undefined;
|
|
1725
1852
|
});
|
|
1726
1853
|
if (observed?.kind === 'target')
|
|
1727
|
-
return
|
|
1854
|
+
return observed;
|
|
1728
1855
|
if (observed?.kind === 'other')
|
|
1729
1856
|
return 'lost';
|
|
1730
1857
|
if (restoreDeadlineReached() && observed?.kind === 'source') {
|
|
@@ -1759,6 +1886,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1759
1886
|
async readSourceSettlement(envelope, primary, target) {
|
|
1760
1887
|
const primaryAuthorityKey = primaryKey(envelope);
|
|
1761
1888
|
let committed = 0;
|
|
1889
|
+
const followAuthorities = new Map();
|
|
1762
1890
|
let primaryCurrent;
|
|
1763
1891
|
for (const participant of envelope.participants) {
|
|
1764
1892
|
const current = await this.requireLocationStore().readAuthority({
|
|
@@ -1767,18 +1895,22 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1767
1895
|
if (current.kind !== 'snapshot')
|
|
1768
1896
|
return { kind: 'other' };
|
|
1769
1897
|
if (current.objectGeneration === participant.objectGeneration &&
|
|
1770
|
-
current.ownerId === target.ownerId &&
|
|
1771
|
-
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
1772
1898
|
current.authorityOwnerGeneration > participant.authorityOwnerGeneration &&
|
|
1773
|
-
|
|
1774
|
-
|
|
1899
|
+
// Location runtime §6: an owner is the (OwnerId, OwnerLeaseGeneration) pair;
|
|
1900
|
+
// a restarted node keeps its OwnerId under a new lease generation.
|
|
1901
|
+
(current.ownerId !== primary.ownerId ||
|
|
1902
|
+
current.ownerLeaseGeneration !== primary.ownerLeaseGeneration)) {
|
|
1775
1903
|
committed += 1;
|
|
1904
|
+
// Location runtime §10: only the observed first target can receive Follow.
|
|
1905
|
+
if (current.ownerId === target.ownerId &&
|
|
1906
|
+
current.ownerLeaseGeneration === target.ownerLeaseGeneration)
|
|
1907
|
+
followAuthorities.set(participant.key, current);
|
|
1776
1908
|
}
|
|
1777
1909
|
if (participant.key === primaryAuthorityKey)
|
|
1778
1910
|
primaryCurrent = current;
|
|
1779
1911
|
}
|
|
1780
1912
|
if (committed === envelope.participants.length)
|
|
1781
|
-
return { kind: 'target' };
|
|
1913
|
+
return { kind: 'target', followAuthorities };
|
|
1782
1914
|
if (primaryCurrent !== undefined && isSourceOwner(primaryCurrent, primary)) {
|
|
1783
1915
|
return { kind: 'source', current: primaryCurrent };
|
|
1784
1916
|
}
|
|
@@ -1813,11 +1945,21 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1813
1945
|
}
|
|
1814
1946
|
/** Restores the hidden object and its saved-work prefix before Ready. */
|
|
1815
1947
|
async handlePrepareControl(meshName, stagingId, request, sourceNodeRid, signal) {
|
|
1948
|
+
// Host relocation flow 05: a sealed target refuses a new PREPARE before any
|
|
1949
|
+
// staging. The typed ShuttingDown reaches the source as relocationFailed.
|
|
1950
|
+
if (this.targetAdmissionSealed()) {
|
|
1951
|
+
throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.ShuttingDown, 'Relocation target admission is sealed for shutdown.');
|
|
1952
|
+
}
|
|
1953
|
+
signal =
|
|
1954
|
+
signal === undefined
|
|
1955
|
+
? this.targetPreparationStop.signal
|
|
1956
|
+
: AbortSignal.any([signal, this.targetPreparationStop.signal]);
|
|
1957
|
+
signal.throwIfAborted();
|
|
1816
1958
|
const targetStatus = this.requireMeshNode(meshName).status();
|
|
1817
1959
|
const targetOwner = this.options.currentOwner();
|
|
1818
1960
|
const targetDescriptor = this.options.localDescriptor?.(meshName);
|
|
1819
1961
|
if (targetOwner === undefined ||
|
|
1820
|
-
|
|
1962
|
+
!(0, routing_id_1.routingIdsEqual)(targetStatus.routingId, request.target.nodeRid) ||
|
|
1821
1963
|
targetStatus.lifecycleGeneration !== request.target.nodeGeneration ||
|
|
1822
1964
|
targetOwner.ownerId !== request.target.ownerId ||
|
|
1823
1965
|
targetOwner.leaseGeneration !== request.target.ownerLeaseGeneration ||
|
|
@@ -1847,11 +1989,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1847
1989
|
payload = await pendingAssembly.assembly.payload();
|
|
1848
1990
|
}
|
|
1849
1991
|
catch (error) {
|
|
1850
|
-
//
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
// the generic InternalFailure code (spec 15 §4).
|
|
1992
|
+
// Target shutdown preserves its lifecycle failure instead of reporting payload corruption.
|
|
1993
|
+
this.targetAssemblies.delete(stagingId);
|
|
1994
|
+
if (this.targetPreparationStop.signal.aborted)
|
|
1995
|
+
throw this.targetPreparationStop.signal.reason;
|
|
1855
1996
|
if (this.disposed)
|
|
1856
1997
|
throw error;
|
|
1857
1998
|
const message = error !== null && typeof error === 'object' && 'message' in error
|
|
@@ -1860,23 +2001,61 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1860
2001
|
throw new ServiceRelocationDataLostError(String(message));
|
|
1861
2002
|
}
|
|
1862
2003
|
const envelope = (0, service_relocation_runtime_1.decodeServiceRelocationEnvelope)(payload, request.targetAttemptGeneration);
|
|
1863
|
-
const
|
|
1864
|
-
|
|
1865
|
-
|
|
2004
|
+
const recovery = canonicalActorJoinRecovery(envelope.participants);
|
|
2005
|
+
const releaseJoinAdmission = recovery === undefined
|
|
2006
|
+
? () => undefined
|
|
2007
|
+
: this.options.spotManager().beginActorJoinRelocation(recovery.request.handoffId);
|
|
1866
2008
|
let materialized;
|
|
1867
2009
|
let reservation;
|
|
2010
|
+
let releaseActivation = () => undefined;
|
|
1868
2011
|
try {
|
|
2012
|
+
const inventoryEnvelope = attachCanonicalBoundSessions(await this.resolveTargetParticipantInventory(meshName, request, envelope, signal), request, meshName);
|
|
2013
|
+
validatePrepareEnvelope(request, inventoryEnvelope);
|
|
2014
|
+
releaseActivation =
|
|
2015
|
+
(await this.options.activationAdmission?.acquire(meshName, signal)) ?? (() => undefined);
|
|
1869
2016
|
// Deferred Join completion publishes into the source authority's
|
|
1870
2017
|
// canonical relocation slot. Do that while the row is still CAS-able;
|
|
1871
2018
|
// the aggregate marker installed by reserveTargetForPrepare deliberately
|
|
1872
2019
|
// rejects unrelated authority CAS until commit/abort.
|
|
1873
2020
|
materialized = await this.materializeTargetEnvelope(meshName, inventoryEnvelope, signal);
|
|
1874
2021
|
await materialized.owner.restoreSavedWork(materialized.staging, signal);
|
|
2022
|
+
for (const participant of inventoryEnvelope.participants) {
|
|
2023
|
+
const binding = participant.sessionBinding;
|
|
2024
|
+
if (binding === undefined)
|
|
2025
|
+
continue;
|
|
2026
|
+
const peerRid = (0, routing_id_1.decodeRoutingId)('', Buffer.from(binding.sessionOwnerNodeRid).toString('hex'));
|
|
2027
|
+
const node = this.requireMeshNode(meshName);
|
|
2028
|
+
if ((0, routing_id_1.routingIdsEqual)(node.status().routingId, peerRid))
|
|
2029
|
+
continue;
|
|
2030
|
+
if (node.isPeerRouteReady?.(peerRid, binding.sessionOwnerNodeGeneration) !== true) {
|
|
2031
|
+
// Subscribe before requesting reconciliation so a synchronous admission cannot be lost.
|
|
2032
|
+
const wait = node.waitForPeerRoute;
|
|
2033
|
+
if (wait === undefined)
|
|
2034
|
+
throw new Error('Mesh peer route observation is unavailable.');
|
|
2035
|
+
const ready = wait.call(node, peerRid, binding.sessionOwnerNodeGeneration, request.deadlineUnixMs, signal);
|
|
2036
|
+
void this.options
|
|
2037
|
+
.requestPeerReconcile?.(meshName, String(peerRid))
|
|
2038
|
+
.catch((error) => console.error('[zlink.runtime.relocation.session_route_reconcile_failed]', error));
|
|
2039
|
+
try {
|
|
2040
|
+
await ready;
|
|
2041
|
+
}
|
|
2042
|
+
catch (error) {
|
|
2043
|
+
if (error instanceof operation_registry_1.OperationTimeoutError) {
|
|
2044
|
+
throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.DeadlineExceeded, 'Relocation source operation deadline exceeded waiting for Session owner route.');
|
|
2045
|
+
}
|
|
2046
|
+
throw error;
|
|
2047
|
+
}
|
|
2048
|
+
}
|
|
2049
|
+
}
|
|
2050
|
+
if (request.deadlineUnixMs <= BigInt(Date.now())) {
|
|
2051
|
+
throw new contracts_1.ZLinkFrameworkException(contracts_1.ZLinkFrameworkErrorKind.DeadlineExceeded, 'Relocation source operation deadline exceeded before relay-ready.');
|
|
2052
|
+
}
|
|
2053
|
+
signal.throwIfAborted();
|
|
1875
2054
|
reservation = await this.reserveTargetForPrepare(meshName, request, inventoryEnvelope, signal);
|
|
1876
2055
|
const offer = {
|
|
1877
2056
|
prepare: request,
|
|
1878
2057
|
prepareFingerprint: fingerprint,
|
|
1879
|
-
authenticatedSourceNodeRid:
|
|
2058
|
+
authenticatedSourceNodeRid: sourceNodeRid,
|
|
1880
2059
|
envelope: inventoryEnvelope,
|
|
1881
2060
|
reservation
|
|
1882
2061
|
};
|
|
@@ -1887,11 +2066,14 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1887
2066
|
lane: Promise.resolve(),
|
|
1888
2067
|
cutoverReceived: false,
|
|
1889
2068
|
boundaryRelay: [],
|
|
1890
|
-
releaseActivation
|
|
2069
|
+
releaseActivation,
|
|
2070
|
+
releaseJoinAdmission
|
|
1891
2071
|
};
|
|
1892
2072
|
this.targetStages.set(stagingId, stage);
|
|
1893
2073
|
}
|
|
1894
2074
|
catch (error) {
|
|
2075
|
+
this.targetAssemblies.delete(stagingId);
|
|
2076
|
+
releaseJoinAdmission();
|
|
1895
2077
|
releaseActivation();
|
|
1896
2078
|
if (reservation !== undefined) {
|
|
1897
2079
|
await this.abortTargetReservation(reservation, signal).catch(() => undefined);
|
|
@@ -2031,24 +2213,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2031
2213
|
stage.releaseActivation();
|
|
2032
2214
|
resumeStartedAt = performance.now();
|
|
2033
2215
|
stage.phase = 'committed';
|
|
2034
|
-
await stage.owner.normalize(stage.staging, authority, signal);
|
|
2035
2216
|
await stage.owner.publish(stage.staging, authority, signal);
|
|
2036
|
-
// The target may have visited this Actor before and still retain that
|
|
2037
|
-
// tenure in its private Actor-client resolver. The target-only CAS is
|
|
2038
|
-
// now authoritative, so expire the old projection before OnJoinedActor
|
|
2039
|
-
// can issue an Actor request and deadlock behind source leave.
|
|
2040
|
-
for (const hidden of stage.staging.hidden.values()) {
|
|
2041
|
-
if (hidden.actor !== undefined) {
|
|
2042
|
-
this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
|
|
2043
|
-
}
|
|
2044
|
-
}
|
|
2045
|
-
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
2046
2217
|
// Clear the final relocation publication before admission becomes
|
|
2047
2218
|
// externally visible so a later Join cannot reserve against the
|
|
2048
2219
|
// previous root. A bounded CAS conflict must not roll back already
|
|
2049
2220
|
// committed target ownership or strand its staged object.
|
|
2050
2221
|
try {
|
|
2051
|
-
await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
2222
|
+
authority = await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
2052
2223
|
}
|
|
2053
2224
|
catch (error) {
|
|
2054
2225
|
console.warn('[zlink.runtime.relocation.publication_clear_failed]', stagingId, error);
|
|
@@ -2056,15 +2227,33 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2056
2227
|
mesh_name: meshName
|
|
2057
2228
|
});
|
|
2058
2229
|
}
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2230
|
+
for (const [index, participant] of stage.staging.envelope.participants.entries()) {
|
|
2231
|
+
const hidden = stage.staging.hidden.get(participant.key);
|
|
2232
|
+
if (hidden?.actor === undefined || hidden.replayPackets.length === 0)
|
|
2233
|
+
continue;
|
|
2234
|
+
const coordinator = this.targetReplyRelayCoordinator(meshName, stage, authority);
|
|
2235
|
+
hidden.replayStart = stage.finalize;
|
|
2236
|
+
hidden.onReplayResult = (packet, result) => this.relayTerminalReply(meshName, stage, coordinator, BigInt(index + 1), packet, result, signal);
|
|
2237
|
+
}
|
|
2238
|
+
await stage.owner.normalize(stage.staging, authority, signal);
|
|
2239
|
+
// The target may have visited this Actor before and still retain that
|
|
2240
|
+
// tenure in its private Actor-client resolver. The target-only CAS is
|
|
2241
|
+
// now authoritative, so expire the old projection before OnJoinedActor
|
|
2242
|
+
// can issue an Actor request and deadlock behind source leave.
|
|
2243
|
+
for (const hidden of stage.staging.hidden.values()) {
|
|
2244
|
+
if (hidden.actor !== undefined) {
|
|
2245
|
+
this.options.invalidateActorRoute?.(hidden.actor.context.actorId);
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
2062
2249
|
await this.publishSessionRoutes(stage.staging);
|
|
2063
|
-
await this.relayTerminalReplies(meshName, stage, this.targetReplyRelayCoordinator(meshName, stage, authority), signal);
|
|
2064
2250
|
stage.phase = 'open';
|
|
2065
2251
|
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
2066
2252
|
this.targetStages.delete(stagingId);
|
|
2067
2253
|
this.targetAssemblies.delete(stagingId);
|
|
2254
|
+
await stage.owner.openAdmission(stage.staging, signal);
|
|
2255
|
+
// S2→S3: owner confirmation to application dispatch opening.
|
|
2256
|
+
this.options.metrics?.duration(runtime_metrics_1.METRIC_NAMES.RelocationTargetResume, Math.max(0, performance.now() - resumeStartedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND, { mesh_name: meshName });
|
|
2068
2257
|
}
|
|
2069
2258
|
catch (error) {
|
|
2070
2259
|
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
@@ -2082,6 +2271,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2082
2271
|
}
|
|
2083
2272
|
throw error;
|
|
2084
2273
|
}
|
|
2274
|
+
finally {
|
|
2275
|
+
stage.releaseJoinAdmission();
|
|
2276
|
+
}
|
|
2085
2277
|
}
|
|
2086
2278
|
validateRelocationData(stage, request) {
|
|
2087
2279
|
const frozen = request.frozenRecord;
|
|
@@ -2096,7 +2288,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2096
2288
|
target.generation !== request.object.objectGeneration ||
|
|
2097
2289
|
target.targetNodeRid !== preparedTarget.nodeRid ||
|
|
2098
2290
|
target.targetNodeGeneration !== preparedTarget.nodeGeneration ||
|
|
2099
|
-
target.authorityOwnerGeneration !== request.object.expectedAuthorityOwnerGeneration
|
|
2291
|
+
target.authorityOwnerGeneration !== request.object.expectedAuthorityOwnerGeneration ||
|
|
2100
2292
|
target.ownerLeaseGeneration !== preparedTarget.ownerLeaseGeneration) {
|
|
2101
2293
|
throw new Error('Relocation data does not target the exact temporary Actor queue.');
|
|
2102
2294
|
}
|
|
@@ -2158,9 +2350,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2158
2350
|
if (profile.state.actorId !== input.actorId || !input.succeeded) {
|
|
2159
2351
|
throw new Error(`Actor Join relocation '${input.transferId}' source leave identity changed.`);
|
|
2160
2352
|
}
|
|
2161
|
-
this.
|
|
2162
|
-
this.terminalActorJoinSourceLeaves.remember(input.transferId, input.actorId);
|
|
2163
|
-
void this.finalizeActorJoinSource(profile);
|
|
2353
|
+
void this.retireActorJoinSource(input.transferId, profile);
|
|
2164
2354
|
return true;
|
|
2165
2355
|
}
|
|
2166
2356
|
async sendControlOneWay(meshName, targetNodeRid, request) {
|
|
@@ -2169,7 +2359,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2169
2359
|
async submitControlFrame(meshName, targetNodeRid, frame, kind) {
|
|
2170
2360
|
const submitted = await this.sendInfrastructureControl(meshName, targetNodeRid, frame);
|
|
2171
2361
|
if (submitted !== runtime_values_1.SubmitResult.Ok) {
|
|
2172
|
-
throw new
|
|
2362
|
+
throw new runtime_values_1.ZLinkBackendResultError('submit', submitted, undefined, {
|
|
2363
|
+
cause: new Error(`Relocation ${kind} was not accepted by RouteMesh.`)
|
|
2364
|
+
});
|
|
2173
2365
|
}
|
|
2174
2366
|
}
|
|
2175
2367
|
async clearTargetRelocationPublication(stage, primary, signal) {
|
|
@@ -2180,7 +2372,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2180
2372
|
// row; the caller records it and leaves the committed ownership intact.
|
|
2181
2373
|
const publication = this.codec.read(primary.payload);
|
|
2182
2374
|
if (publication === undefined)
|
|
2183
|
-
return;
|
|
2375
|
+
return primary;
|
|
2184
2376
|
const result = await this.requireLocationStore().compareExchangeAuthority(stage.staging.primaryAuthorityKey, primary.storeVersion, {
|
|
2185
2377
|
kind: 'put',
|
|
2186
2378
|
generationTransition: 'preserve',
|
|
@@ -2189,39 +2381,52 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2189
2381
|
if (result.kind !== 'stored') {
|
|
2190
2382
|
throw new Error('Relocation target authority normalization CAS failed.');
|
|
2191
2383
|
}
|
|
2384
|
+
return { ...result, kind: 'snapshot' };
|
|
2192
2385
|
}
|
|
2193
2386
|
async handleReplyRelay(meshName, relay, sourceNodeRid, signal) {
|
|
2194
2387
|
if (sourceNodeRid === null) {
|
|
2195
2388
|
throw new Error('Relocation reply relay has no authenticated target node.');
|
|
2196
2389
|
}
|
|
2197
|
-
await this.requireAdmittedPeer(meshName,
|
|
2198
|
-
const
|
|
2199
|
-
const
|
|
2200
|
-
if (authorityKey === undefined) {
|
|
2201
|
-
throw new Error('Relocation reply relay has no registered durable authority.');
|
|
2202
|
-
}
|
|
2203
|
-
const authority = await requireAuthority(this.requireLocationStore(), { value: authorityKey }, signal);
|
|
2204
|
-
if (authority.storeVersion.value !== relay.coordinator.expectedAuthorityStoreVersion ||
|
|
2205
|
-
authority.ownerId !== relay.coordinator.ownerId ||
|
|
2206
|
-
authority.ownerLeaseGeneration !== relay.coordinator.leaseGeneration ||
|
|
2207
|
-
String(authority.allocation.descriptor.rid) !== relay.coordinator.nodeRid ||
|
|
2208
|
-
authority.allocation.descriptorLifecycleGeneration !== relay.coordinator.nodeGeneration) {
|
|
2209
|
-
throw new Error('Relocation reply relay coordinator authority fence is stale.');
|
|
2210
|
-
}
|
|
2211
|
-
const accepted = this.options.actorTransfer.relayCanonicalMaintenanceTerminal(wireIdText(relay.operation), relay.replyRouteId.toString(), handoffResultFromRelay(relay), sourceNodeRid);
|
|
2212
|
-
if ((accepted.status !== 'terminalReceived' && accepted.status !== 'alreadyTerminal') ||
|
|
2213
|
-
accepted.source === undefined) {
|
|
2214
|
-
throw new Error('Canonical relocation reply relay collided with another source route.');
|
|
2215
|
-
}
|
|
2216
|
-
const requestSource = handoffSourceFence(accepted.source);
|
|
2217
|
-
const localStatus = this.requireMeshNode(meshName).status();
|
|
2390
|
+
await this.requireAdmittedPeer(meshName, sourceNodeRid, relay.coordinator, signal);
|
|
2391
|
+
const node = this.requireMeshNode(meshName);
|
|
2392
|
+
const localStatus = node.status();
|
|
2218
2393
|
const localOwner = this.options.currentOwner();
|
|
2219
|
-
if (localOwner === undefined
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2394
|
+
if (localOwner === undefined)
|
|
2395
|
+
throw new Error('Relocation requester owner lease is unavailable.');
|
|
2396
|
+
let status = node.acceptRelocatedActorTerminal?.(relay.operation, relay.replyRouteId, {
|
|
2397
|
+
terminalResult: relay.terminalResult,
|
|
2398
|
+
failureCode: relay.failureCode,
|
|
2399
|
+
...(relay.payload === undefined
|
|
2400
|
+
? {}
|
|
2401
|
+
: {
|
|
2402
|
+
payload: {
|
|
2403
|
+
packetName: relay.payload.packetName,
|
|
2404
|
+
contentType: relay.payload.contentType,
|
|
2405
|
+
payload: Buffer.from(relay.payload.bytes)
|
|
2406
|
+
}
|
|
2407
|
+
})
|
|
2408
|
+
});
|
|
2409
|
+
let requestSource = {
|
|
2410
|
+
ownerId: localOwner.ownerId,
|
|
2411
|
+
leaseGeneration: localOwner.leaseGeneration,
|
|
2412
|
+
nodeRid: localStatus.routingId,
|
|
2413
|
+
nodeGeneration: localStatus.lifecycleGeneration
|
|
2414
|
+
};
|
|
2415
|
+
if (status === undefined) {
|
|
2416
|
+
const accepted = this.options.actorTransfer.relayCanonicalMaintenanceTerminal(relay.operation.high.toString(16).padStart(16, '0') +
|
|
2417
|
+
relay.operation.low.toString(16).padStart(16, '0'), relay.replyRouteId.toString(), handoffResultFromRelay(relay), sourceNodeRid);
|
|
2418
|
+
if (accepted.status === 'notAcknowledged' || accepted.source === undefined) {
|
|
2419
|
+
throw new Error('Canonical relocation reply relay collided with another source route.');
|
|
2420
|
+
}
|
|
2421
|
+
requestSource = handoffSourceFence(accepted.source);
|
|
2422
|
+
if (!sameRequestSource(requestSource, {
|
|
2423
|
+
ownerId: localOwner.ownerId,
|
|
2424
|
+
leaseGeneration: localOwner.leaseGeneration,
|
|
2425
|
+
nodeRid: localStatus.routingId,
|
|
2426
|
+
nodeGeneration: localStatus.lifecycleGeneration
|
|
2427
|
+
}))
|
|
2428
|
+
throw new Error('Relocation reply relay request-source fence changed after capture.');
|
|
2429
|
+
status = accepted.status;
|
|
2225
2430
|
}
|
|
2226
2431
|
const ack = {
|
|
2227
2432
|
relocation: relay.relocation,
|
|
@@ -2229,16 +2434,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2229
2434
|
operation: relay.operation,
|
|
2230
2435
|
replyRouteId: relay.replyRouteId,
|
|
2231
2436
|
requestSource,
|
|
2232
|
-
status
|
|
2437
|
+
status
|
|
2233
2438
|
};
|
|
2234
2439
|
if ((await this.sendInfrastructureControl(meshName, sourceNodeRid, (0, service_stateful_wire_codec_1.encodeMaintenanceReplyRelayAck)(ack))) !== runtime_values_1.SubmitResult.Ok) {
|
|
2235
2440
|
throw new Error('Relocation reply relay ACK send was rejected.');
|
|
2236
2441
|
}
|
|
2237
2442
|
}
|
|
2238
2443
|
async sendReplyRelay(meshName, ackTargetNodeRid, request, expectedRequestSource, signal) {
|
|
2239
|
-
const targetRid = String(ackTargetNodeRid);
|
|
2240
2444
|
const identityKey = replyRelayIdentityKey(request);
|
|
2241
|
-
const key = replyRelayPendingKey(
|
|
2445
|
+
const key = replyRelayPendingKey(ackTargetNodeRid, request);
|
|
2242
2446
|
if (this.pendingReplyRelays.has(key)) {
|
|
2243
2447
|
throw new Error(`Relocation reply relay ACK '${key}' is already pending.`);
|
|
2244
2448
|
}
|
|
@@ -2252,7 +2456,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2252
2456
|
action();
|
|
2253
2457
|
};
|
|
2254
2458
|
const pending = {
|
|
2255
|
-
ackTargetNodeRid
|
|
2459
|
+
ackTargetNodeRid,
|
|
2256
2460
|
identityKey,
|
|
2257
2461
|
request,
|
|
2258
2462
|
expectedRequestSource,
|
|
@@ -2288,7 +2492,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2288
2492
|
}
|
|
2289
2493
|
async acceptReplyRelayAck(meshName, ack, sourceNodeRid, signal) {
|
|
2290
2494
|
const identityKey = replyRelayIdentityKey(ack);
|
|
2291
|
-
const exactKey = sourceNodeRid === null ? undefined : replyRelayPendingKey(
|
|
2495
|
+
const exactKey = sourceNodeRid === null ? undefined : replyRelayPendingKey(sourceNodeRid, ack);
|
|
2292
2496
|
const pending = exactKey === undefined ? undefined : this.pendingReplyRelays.get(exactKey);
|
|
2293
2497
|
if (pending === undefined) {
|
|
2294
2498
|
const collision = [...this.pendingReplyRelays.values()].find((value) => value.identityKey === identityKey);
|
|
@@ -2311,12 +2515,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2311
2515
|
}
|
|
2312
2516
|
}
|
|
2313
2517
|
async requireAdmittedPeer(meshName, authenticatedRid, expected, signal) {
|
|
2314
|
-
const descriptor = (await this.options.liveDescriptors(meshName, signal)).find((value) =>
|
|
2518
|
+
const descriptor = (await this.options.liveDescriptors(meshName, signal)).find((value) => (0, routing_id_1.routingIdsEqual)(value.rid, authenticatedRid) && value.state === 1);
|
|
2315
2519
|
if (descriptor === undefined ||
|
|
2316
2520
|
(expected !== undefined &&
|
|
2317
2521
|
(descriptor.ownerId !== expected.ownerId ||
|
|
2318
2522
|
descriptor.leaseGeneration !== expected.leaseGeneration ||
|
|
2319
|
-
|
|
2523
|
+
!(0, routing_id_1.routingIdsEqual)(descriptor.rid, expected.nodeRid) ||
|
|
2320
2524
|
descriptor.lifecycleGeneration !== expected.nodeGeneration))) {
|
|
2321
2525
|
throw new Error('Relocation reply relay peer is not the exact admitted source fence.');
|
|
2322
2526
|
}
|
|
@@ -2328,39 +2532,32 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2328
2532
|
if (owner === undefined ||
|
|
2329
2533
|
owner.ownerId !== target.ownerId ||
|
|
2330
2534
|
owner.leaseGeneration !== target.ownerLeaseGeneration ||
|
|
2331
|
-
|
|
2535
|
+
!(0, routing_id_1.routingIdsEqual)(status.routingId, target.nodeRid) ||
|
|
2332
2536
|
status.lifecycleGeneration !== target.nodeGeneration) {
|
|
2333
2537
|
throw new Error('Relocation reply relay target coordinator fence is stale.');
|
|
2334
2538
|
}
|
|
2335
2539
|
return {
|
|
2336
2540
|
ownerId: owner.ownerId,
|
|
2337
2541
|
leaseGeneration: owner.leaseGeneration,
|
|
2338
|
-
nodeRid:
|
|
2542
|
+
nodeRid: status.routingId,
|
|
2339
2543
|
nodeGeneration: status.lifecycleGeneration,
|
|
2340
2544
|
expectedAuthorityStoreVersion: authority.storeVersion.value
|
|
2341
2545
|
};
|
|
2342
2546
|
}
|
|
2343
|
-
async
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
operationId: packet.messageFollowContext.operationId,
|
|
2358
|
-
source: packet.source,
|
|
2359
|
-
result
|
|
2360
|
-
};
|
|
2361
|
-
await this.sendReplyRelay(meshName, packet.source.nodeRid, maintenanceReplyRelay(stage, coordinator, BigInt(participantIndex + 1), completion), handoffSourceFence(packet.source), signal);
|
|
2362
|
-
}
|
|
2363
|
-
}
|
|
2547
|
+
async relayTerminalReply(meshName, stage, coordinator, participantId, packet, result, signal) {
|
|
2548
|
+
if (!packet.returnResponse || packet.source === undefined)
|
|
2549
|
+
return;
|
|
2550
|
+
if (result.index !== packet.index) {
|
|
2551
|
+
throw new Error('Relocation request result changed after target dispatch.');
|
|
2552
|
+
}
|
|
2553
|
+
const completion = {
|
|
2554
|
+
index: packet.index,
|
|
2555
|
+
operationId: packet.messageFollowContext.operationId,
|
|
2556
|
+
source: packet.source,
|
|
2557
|
+
result
|
|
2558
|
+
};
|
|
2559
|
+
const source = handoffSourceFence(packet.source);
|
|
2560
|
+
await this.sendReplyRelay(meshName, source.nodeRid, maintenanceReplyRelay(stage, coordinator, participantId, completion, packet, this.options.registration.messageSerializers, this.options.flowCreationEnabled?.() ?? false), source, signal);
|
|
2364
2561
|
}
|
|
2365
2562
|
async exactSourceLeaseExpired(source, signal) {
|
|
2366
2563
|
const lease = await this.requireLocationStore().readOwnerLease(source.ownerId, signal);
|
|
@@ -2378,60 +2575,34 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2378
2575
|
await this.options.actorTransfer.publishRoutedActorOwnership(hidden.actor, carried === undefined ? undefined : decodeActorSession(carried.boundSessionState));
|
|
2379
2576
|
}
|
|
2380
2577
|
}
|
|
2381
|
-
async sendControl(meshName, targetNodeRid, request, signal, deadlineAtMs = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS,
|
|
2578
|
+
async sendControl(meshName, targetNodeRid, request, signal, deadlineAtMs = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS, onSubmitted) {
|
|
2382
2579
|
const key = controlAckKey(request);
|
|
2383
|
-
const frames =
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
signal?.throwIfAborted();
|
|
2389
|
-
const remainingMs = deadlineAtMs - performance.now();
|
|
2390
|
-
if (remainingMs <= 0) {
|
|
2391
|
-
throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.DeadlineExceeded, `Relocation control ACK '${key}' timed out.`, true, lastRequestError);
|
|
2392
|
-
}
|
|
2393
|
-
const attemptTimeoutMs = Math.max(1, Math.min(RELOCATION_ATTEMPT_TIMEOUT_MS, Math.ceil(remainingMs)));
|
|
2394
|
-
const startedAtMs = performance.now();
|
|
2395
|
-
try {
|
|
2396
|
-
const replyFrames = await this.requestInfrastructureControl(meshName, targetNodeRid, frames, attemptTimeoutMs);
|
|
2397
|
-
if (replyFrames.length !== 1) {
|
|
2398
|
-
throw new service_wire_m6a_codec_1.ServiceWireProtocolError(`Relocation control '${key}' reply must contain exactly one frame.`);
|
|
2399
|
-
}
|
|
2400
|
-
const response = (0, service_relocation_control_1.decodeServiceRelocationControlResponse)(Buffer.from(replyFrames[0]));
|
|
2401
|
-
if (response.kind === 'failed') {
|
|
2402
|
-
validateControlFailureResponse(request, response);
|
|
2403
|
-
throw relocationFailureException(response);
|
|
2404
|
-
}
|
|
2405
|
-
validateControlResponse(request, response);
|
|
2406
|
-
return response;
|
|
2407
|
-
}
|
|
2408
|
-
catch (error) {
|
|
2409
|
-
if (error instanceof service_wire_m6a_codec_1.ServiceWireProtocolError || error instanceof contracts_1.ZLinkFrameworkException) {
|
|
2410
|
-
throw error;
|
|
2411
|
-
}
|
|
2412
|
-
lastRequestError = error;
|
|
2413
|
-
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
2414
|
-
(0, diagnostics_1.relocationDebug)('control.request_retry', {
|
|
2415
|
-
key,
|
|
2416
|
-
kind: request.kind,
|
|
2417
|
-
targetRid: String(targetNodeRid)
|
|
2418
|
-
});
|
|
2419
|
-
}
|
|
2420
|
-
const retryDelayMs = Math.min(Math.max(0, attemptTimeoutMs - (performance.now() - startedAtMs)), Math.max(0, deadlineAtMs - performance.now()));
|
|
2421
|
-
if (retryDelayMs > 0) {
|
|
2422
|
-
await waitForRelocationRetry(retryDelayMs, signal);
|
|
2423
|
-
}
|
|
2424
|
-
}
|
|
2580
|
+
const frames = [(0, service_relocation_control_1.encodeServiceRelocationControlRequest)(request)];
|
|
2581
|
+
signal?.throwIfAborted();
|
|
2582
|
+
const remainingMs = Math.ceil(deadlineAtMs - performance.now());
|
|
2583
|
+
if (remainingMs <= 0) {
|
|
2584
|
+
throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.DeadlineExceeded, `Relocation control ACK '${key}' timed out.`, true);
|
|
2425
2585
|
}
|
|
2586
|
+
const replyFrames = await this.requestInfrastructureControl(meshName, targetNodeRid, frames, remainingMs, signal, onSubmitted);
|
|
2587
|
+
if (replyFrames.length !== 1) {
|
|
2588
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError(`Relocation control '${key}' reply must contain exactly one frame.`);
|
|
2589
|
+
}
|
|
2590
|
+
const response = (0, service_relocation_control_1.decodeServiceRelocationControlResponse)(Buffer.from(replyFrames[0]));
|
|
2591
|
+
if (response.kind === 'failed') {
|
|
2592
|
+
validateControlFailureResponse(request, response);
|
|
2593
|
+
throw relocationFailureException(response);
|
|
2594
|
+
}
|
|
2595
|
+
validateControlResponse(request, response);
|
|
2596
|
+
return response;
|
|
2426
2597
|
}
|
|
2427
2598
|
async selectTarget(meshName, localRid, targetApplicationVersion, requirements = [], signal) {
|
|
2428
2599
|
const descriptors = await this.options.liveDescriptors(meshName, signal);
|
|
2429
2600
|
const local = this.options.localDescriptor?.(meshName) ??
|
|
2430
|
-
descriptors.find((descriptor) =>
|
|
2601
|
+
descriptors.find((descriptor) => (0, routing_id_1.routingIdsEqual)(descriptor.rid, localRid));
|
|
2431
2602
|
const requiredVersion = targetApplicationVersion ?? local?.applicationVersion;
|
|
2432
2603
|
const peers = this.requireMeshNode(meshName).peers();
|
|
2433
2604
|
const candidates = descriptors
|
|
2434
|
-
.filter((descriptor) =>
|
|
2605
|
+
.filter((descriptor) => !(0, routing_id_1.routingIdsEqual)(descriptor.rid, localRid) &&
|
|
2435
2606
|
descriptor.state === contracts_1.ZLinkFrameworkRuntimeState.Serving &&
|
|
2436
2607
|
descriptor.objectRole === contracts_1.ZLinkObjectRole.Server &&
|
|
2437
2608
|
descriptor.placementWeight > 0 &&
|
|
@@ -2439,7 +2610,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2439
2610
|
(local?.maintenanceWave === undefined ||
|
|
2440
2611
|
descriptor.maintenanceWave !== local.maintenanceWave) &&
|
|
2441
2612
|
peers.some((peer) => peer.routingId !== null &&
|
|
2442
|
-
|
|
2613
|
+
(0, routing_id_1.routingIdsEqual)(peer.routingId, descriptor.rid) &&
|
|
2443
2614
|
peer.lifecycleGeneration === descriptor.lifecycleGeneration &&
|
|
2444
2615
|
peer.state === service_runtime_contracts_1.MeshPeerRuntimeState.Serving) &&
|
|
2445
2616
|
relocationTargetSupports(descriptor, requirements))
|
|
@@ -2473,7 +2644,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2473
2644
|
const publication = relocationPublication(prepare, envelope);
|
|
2474
2645
|
const participants = envelope.participants.map((participant) => {
|
|
2475
2646
|
const expected = authorities.get(participant.key);
|
|
2476
|
-
const membership = actorMembershipTarget(envelope
|
|
2647
|
+
const membership = actorMembershipTarget(envelope.memberships.find((value) => value.actorKey === participant.key), this.options.localDescriptor?.(meshName));
|
|
2477
2648
|
const entrySpotId = expected.allocation.objectKind === 'actor' && membership === undefined
|
|
2478
2649
|
? this.options.spotManager()?.entrySpotIdForMesh(meshName)
|
|
2479
2650
|
: undefined;
|
|
@@ -2497,18 +2668,14 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2497
2668
|
objectGeneration: expected.objectGeneration,
|
|
2498
2669
|
targetAttemptGeneration: prepare.targetAttemptGeneration,
|
|
2499
2670
|
coordinatorExpectedStoreVersion: expected.storeVersion.value,
|
|
2500
|
-
...(membership
|
|
2501
|
-
|
|
2671
|
+
...(membership ??
|
|
2672
|
+
(expected.allocation.objectKind === 'actor'
|
|
2502
2673
|
? {
|
|
2503
2674
|
actorSpotId: entrySpotId,
|
|
2504
2675
|
actorSpotGeneration: prepare.target.nodeGeneration,
|
|
2505
2676
|
actorSpotKind: contracts_1.ZLinkSpotKind.Entry
|
|
2506
2677
|
}
|
|
2507
|
-
: {}
|
|
2508
|
-
: {
|
|
2509
|
-
...membership,
|
|
2510
|
-
actorSpotKind: contracts_1.ZLinkSpotKind.User
|
|
2511
|
-
})
|
|
2678
|
+
: {}))
|
|
2512
2679
|
}),
|
|
2513
2680
|
membershipMutation: encodeMembershipMutation(envelope.memberships, participant.key)
|
|
2514
2681
|
};
|
|
@@ -2599,6 +2766,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2599
2766
|
failures.push(error);
|
|
2600
2767
|
}
|
|
2601
2768
|
stage.phase = 'failed';
|
|
2769
|
+
stage.releaseJoinAdmission();
|
|
2602
2770
|
if (failures.length === 1)
|
|
2603
2771
|
throw failures[0];
|
|
2604
2772
|
if (failures.length > 1) {
|
|
@@ -2729,7 +2897,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2729
2897
|
return { owner, staging, target };
|
|
2730
2898
|
}
|
|
2731
2899
|
/** Maps schema ordinals to the authoritative Location Store inventory. */
|
|
2732
|
-
async resolveTargetParticipantInventory(request, envelope, signal) {
|
|
2900
|
+
async resolveTargetParticipantInventory(meshName, request, envelope, signal) {
|
|
2733
2901
|
if (!envelope.participants.some((value) => value.rootSpotId !== undefined)) {
|
|
2734
2902
|
return envelope;
|
|
2735
2903
|
}
|
|
@@ -2752,7 +2920,8 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2752
2920
|
(0, authority_key_codec_1.decodeAuthorityKey)(candidate.key).kind !== 'actor')
|
|
2753
2921
|
continue;
|
|
2754
2922
|
const actor = (0, actor_authority_publication_1.decodeRelocatingActorAuthorityIdentity)(candidate.snapshot.payload, candidate.snapshot.objectGeneration);
|
|
2755
|
-
if (actor?.spotId === request.object.spotId
|
|
2923
|
+
if (actor?.spotId === request.object.spotId &&
|
|
2924
|
+
actor.spotGeneration === request.object.objectGeneration) {
|
|
2756
2925
|
entries.push(candidate);
|
|
2757
2926
|
}
|
|
2758
2927
|
}
|
|
@@ -2794,20 +2963,17 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2794
2963
|
authorityOwnerGeneration: snapshot.authorityOwnerGeneration
|
|
2795
2964
|
};
|
|
2796
2965
|
});
|
|
2797
|
-
//
|
|
2798
|
-
//
|
|
2799
|
-
// Location Store still names the source Spot until target cutover. Rebuild
|
|
2800
|
-
// the Actor lane from the already accepted ActorJoin admission, preserving
|
|
2801
|
-
// the target queue boundary for decoded saved-work.
|
|
2966
|
+
// Join recovery uses its accepted target membership. Maintenance Actor
|
|
2967
|
+
// roots restore into the target's startup Entry incarnation (§10.1).
|
|
2802
2968
|
const memberships = request.object.kind === 'actor'
|
|
2803
|
-
?
|
|
2969
|
+
? actorRootTargetMembership(manager, envelope.aggregateId, participants, this.options.localDescriptor?.(meshName))
|
|
2804
2970
|
: entries
|
|
2805
2971
|
.filter((entry) => entry.snapshot.allocation.objectKind === 'actor')
|
|
2806
2972
|
.map((entry) => ({
|
|
2807
2973
|
actorKey: entry.key.value,
|
|
2808
2974
|
spotKey: primaryKey.value,
|
|
2809
2975
|
spotObjectGeneration: primary.objectGeneration,
|
|
2810
|
-
membershipEpoch:
|
|
2976
|
+
membershipEpoch: INITIAL_MEMBERSHIP_EPOCH
|
|
2811
2977
|
}));
|
|
2812
2978
|
return {
|
|
2813
2979
|
...envelope,
|
|
@@ -2857,19 +3023,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2857
3023
|
async createRelocationAdapter(adapterType) {
|
|
2858
3024
|
return (await (0, spot_provider_1.createProviderInstance)(adapterType, this.options.providerResolver));
|
|
2859
3025
|
}
|
|
2860
|
-
recordRelocationInterruption(meshName, unitKind, executionMode, startedAt
|
|
3026
|
+
recordRelocationInterruption(meshName, unitKind, executionMode, startedAt) {
|
|
2861
3027
|
const seconds = Math.max(0, performance.now() - startedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND;
|
|
2862
3028
|
const attributes = {
|
|
2863
3029
|
unit_kind: unitKind,
|
|
2864
3030
|
...(executionMode === undefined ? {} : { execution_mode: executionMode })
|
|
2865
3031
|
};
|
|
2866
3032
|
this.options.metrics?.duration(runtime_metrics_1.METRIC_NAMES.RelocationInterruption, seconds, attributes);
|
|
2867
|
-
this.options.metrics?.count(runtime_metrics_1.METRIC_NAMES.RelocationCompleted, 1, {
|
|
2868
|
-
mesh_name: meshName,
|
|
2869
|
-
object_kind: unitKind,
|
|
2870
|
-
policy: unitKind === 'user_spot' && executionMode === 'per_actor' ? 'recreate' : 'configured',
|
|
2871
|
-
outcome
|
|
2872
|
-
});
|
|
2873
3033
|
if (seconds <= 1)
|
|
2874
3034
|
return;
|
|
2875
3035
|
console.warn(`[zlink.runtime.relocation.changed] ${unitKind} admission interruption ` +
|
|
@@ -2912,24 +3072,25 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2912
3072
|
}
|
|
2913
3073
|
return result;
|
|
2914
3074
|
}
|
|
2915
|
-
async commitSpotMessageFollow(meshName, seal,
|
|
3075
|
+
async commitSpotMessageFollow(meshName, seal, committed, activation) {
|
|
2916
3076
|
const node = this.requireMeshNode(meshName);
|
|
2917
3077
|
const commit = node.commitSpotMessageFollowIngress;
|
|
2918
3078
|
if (commit === undefined) {
|
|
2919
3079
|
throw new Error('Spot Message Follow backend support is incomplete.');
|
|
2920
3080
|
}
|
|
2921
|
-
const
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
3081
|
+
const accepted = await commit.call(node, seal, committed === undefined
|
|
3082
|
+
? null
|
|
3083
|
+
: {
|
|
3084
|
+
spot: {
|
|
3085
|
+
spotId: String(activation.spotId),
|
|
3086
|
+
generation: committed.objectGeneration
|
|
3087
|
+
},
|
|
3088
|
+
targetNodeRid: String(committed.allocation.descriptor.rid),
|
|
3089
|
+
targetNodeGeneration: committed.allocation.descriptorLifecycleGeneration,
|
|
3090
|
+
authorityOwnerGeneration: committed.authorityOwnerGeneration,
|
|
3091
|
+
ownerLeaseGeneration: committed.ownerLeaseGeneration,
|
|
3092
|
+
storeVersion: committed.storeVersion.value
|
|
3093
|
+
}, this.relocationLimits().messageFollowDurationMs);
|
|
2933
3094
|
if (!accepted) {
|
|
2934
3095
|
throw new Error(`Spot '${String(activation.spotId)}' Message Follow route became stale.`);
|
|
2935
3096
|
}
|
|
@@ -3002,16 +3163,30 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
3002
3163
|
}
|
|
3003
3164
|
return await send.call(node, targetNodeRid, record);
|
|
3004
3165
|
}
|
|
3005
|
-
async requestInfrastructureControl(meshName, targetNodeRid, frames, timeoutMs) {
|
|
3166
|
+
async requestInfrastructureControl(meshName, targetNodeRid, frames, timeoutMs, signal, onSubmitted) {
|
|
3006
3167
|
const node = this.requireMeshNode(meshName);
|
|
3007
3168
|
const request = node.requestInfrastructureControlFrames;
|
|
3008
3169
|
if (request === undefined) {
|
|
3009
3170
|
throw new Error('RouteMesh backend cannot submit bare infrastructure requests.');
|
|
3010
3171
|
}
|
|
3011
|
-
return await request.call(node, targetNodeRid, frames, { timeoutMs });
|
|
3172
|
+
return await request.call(node, targetNodeRid, frames, { timeoutMs, signal, onSubmitted });
|
|
3012
3173
|
}
|
|
3013
3174
|
}
|
|
3014
3175
|
exports.ZLinkHostServiceRelocationRuntime = ZLinkHostServiceRelocationRuntime;
|
|
3176
|
+
function committedSpotRoute(meshName, spotId, spotKind, targetNodeRid, authority) {
|
|
3177
|
+
return {
|
|
3178
|
+
routerChannelId: meshName,
|
|
3179
|
+
targetNodeRid,
|
|
3180
|
+
spotId,
|
|
3181
|
+
spotKind,
|
|
3182
|
+
targetSpotGeneration: authority?.objectGeneration,
|
|
3183
|
+
targetNodeGeneration: authority?.allocation.descriptorLifecycleGeneration,
|
|
3184
|
+
authorityOwnerGeneration: authority?.authorityOwnerGeneration,
|
|
3185
|
+
targetOwnerId: authority?.ownerId,
|
|
3186
|
+
ownerLeaseGeneration: authority?.ownerLeaseGeneration,
|
|
3187
|
+
authorityStoreVersion: authority?.storeVersion.value
|
|
3188
|
+
};
|
|
3189
|
+
}
|
|
3015
3190
|
function canonicalActorJoinRecovery(participants) {
|
|
3016
3191
|
const matches = participants.flatMap((participant) => participant.queuedMessages.flatMap((message) => {
|
|
3017
3192
|
const recovery = (0, actor_join_recovery_codec_1.decodeCanonicalActorJoinRecoverySavedWork)(message.payload);
|
|
@@ -3025,11 +3200,24 @@ function canonicalActorJoinRecovery(participants) {
|
|
|
3025
3200
|
function actorJoinAdmissionIdentity(envelope) {
|
|
3026
3201
|
return (canonicalActorJoinRecovery(envelope.participants)?.request.handoffId ?? envelope.aggregateId);
|
|
3027
3202
|
}
|
|
3028
|
-
function
|
|
3203
|
+
function actorRootTargetMembership(manager, relocationId, participants, localDescriptor) {
|
|
3029
3204
|
const recovery = canonicalActorJoinRecovery(participants);
|
|
3030
|
-
const admissionId = recovery?.request.handoffId ?? relocationId;
|
|
3031
|
-
const admission = manager?.formalRemoteActorAdmissions?.get(admissionId);
|
|
3032
3205
|
const actor = participants.find((value) => value.objectKind === 'actor');
|
|
3206
|
+
if (recovery === undefined) {
|
|
3207
|
+
if (actor === undefined || localDescriptor?.entrySpotId === undefined) {
|
|
3208
|
+
throw new Error(`Actor relocation '${relocationId}' has no target Entry membership.`);
|
|
3209
|
+
}
|
|
3210
|
+
return [
|
|
3211
|
+
{
|
|
3212
|
+
actorKey: actor.key,
|
|
3213
|
+
spotKey: (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', localDescriptor.entrySpotId).value,
|
|
3214
|
+
spotObjectGeneration: localDescriptor.lifecycleGeneration,
|
|
3215
|
+
membershipEpoch: INITIAL_MEMBERSHIP_EPOCH
|
|
3216
|
+
}
|
|
3217
|
+
];
|
|
3218
|
+
}
|
|
3219
|
+
const admissionId = recovery.request.handoffId;
|
|
3220
|
+
const admission = manager?.formalRemoteActorAdmissions?.get(admissionId);
|
|
3033
3221
|
if (admission === undefined || actor === undefined) {
|
|
3034
3222
|
throw new Error(`Actor relocation '${relocationId}' has no accepted target membership.`);
|
|
3035
3223
|
}
|
|
@@ -3054,34 +3242,43 @@ class LocalTargetPort {
|
|
|
3054
3242
|
async createHidden(participant, signal) {
|
|
3055
3243
|
const identity = (0, authority_key_codec_1.decodeAuthorityKey)({ value: participant.key }).globalId;
|
|
3056
3244
|
if (participant.objectKind === 'actor') {
|
|
3245
|
+
if (participant.timers.length !== 0) {
|
|
3246
|
+
throw new Error('Actor relocation cannot restore a Spot timer.');
|
|
3247
|
+
}
|
|
3057
3248
|
const membership = this.envelope.memberships.find((value) => value.actorKey === participant.key);
|
|
3058
3249
|
if (membership === undefined)
|
|
3059
3250
|
throw new Error(`Actor '${identity}' relocation membership is missing.`);
|
|
3060
|
-
const
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
const nativeEntrySpot = localDescriptor?.entrySpotId === spotIdentity;
|
|
3065
|
-
const actor = await this.requireActorManager().prepareRelocationActor(identity, participant.stableType, participant.objectGeneration, participant.authorityOwnerGeneration + 1n, nativeEntrySpot ? undefined : spotIdentity, membership.spotObjectGeneration, membership.membershipEpoch, signal);
|
|
3251
|
+
const targetMembership = actorMembershipTarget(membership, this.options.localDescriptor?.(this.meshName));
|
|
3252
|
+
const actor = await this.requireActorManager().prepareRelocationActor(identity, participant.stableType, participant.objectGeneration, participant.authorityOwnerGeneration, targetMembership.actorSpotKind === contracts_1.ZLinkSpotKind.Entry
|
|
3253
|
+
? undefined
|
|
3254
|
+
: targetMembership.actorSpotId, targetMembership.actorSpotGeneration, membership.membershipEpoch, signal);
|
|
3066
3255
|
return {
|
|
3067
3256
|
authorityKey: participant.key,
|
|
3068
3257
|
participant,
|
|
3069
3258
|
actor,
|
|
3070
3259
|
initialized: false,
|
|
3071
|
-
restoredTimers: [],
|
|
3072
|
-
replayResults: [],
|
|
3073
3260
|
replayPackets: []
|
|
3074
3261
|
};
|
|
3075
3262
|
}
|
|
3076
3263
|
const registration = this.registration(participant.objectKind, participant.stableType);
|
|
3077
|
-
const activation = await this.requireSpotManager().prepareRelocationSpot(this.meshName, participant.objectKind, participant.stableType, registration.implementation, identity, participant.objectGeneration, participant.authorityOwnerGeneration
|
|
3264
|
+
const activation = await this.requireSpotManager().prepareRelocationSpot(this.meshName, participant.objectKind, participant.stableType, registration.implementation, identity, participant.objectGeneration, participant.authorityOwnerGeneration, participant.timers.map((value) => ({
|
|
3265
|
+
name: value.timerId,
|
|
3266
|
+
handlerType: value.handlerType,
|
|
3267
|
+
periodMs: value.intervalMs,
|
|
3268
|
+
overrunPolicy: value.overrunPolicy,
|
|
3269
|
+
maxCatchUpTicks: value.maxCatchUpTicks,
|
|
3270
|
+
stopOnUnhandledException: value.stopOnUnhandledException,
|
|
3271
|
+
startedAtUnixMs: value.startedAtUnixMs,
|
|
3272
|
+
deliveryIndex: value.deliveryIndex,
|
|
3273
|
+
lastScheduledIndex: value.lastScheduledIndex,
|
|
3274
|
+
nextDueAtUnixMs: value.dueAtUnixMs,
|
|
3275
|
+
pendingTicks: value.pendingTicks
|
|
3276
|
+
})), signal);
|
|
3078
3277
|
return {
|
|
3079
3278
|
authorityKey: participant.key,
|
|
3080
3279
|
participant,
|
|
3081
3280
|
activation,
|
|
3082
3281
|
initialized: false,
|
|
3083
|
-
restoredTimers: [],
|
|
3084
|
-
replayResults: [],
|
|
3085
3282
|
replayPackets: []
|
|
3086
3283
|
};
|
|
3087
3284
|
}
|
|
@@ -3102,17 +3299,14 @@ class LocalTargetPort {
|
|
|
3102
3299
|
async restoreMemberships(hidden, memberships) {
|
|
3103
3300
|
for (const membership of memberships) {
|
|
3104
3301
|
const actor = hidden.get(membership.actorKey)?.actor;
|
|
3105
|
-
const
|
|
3106
|
-
|
|
3107
|
-
}).globalId;
|
|
3302
|
+
const targetMembership = actorMembershipTarget(membership, this.options.localDescriptor?.(this.meshName));
|
|
3303
|
+
const spotIdentity = targetMembership.actorSpotId;
|
|
3108
3304
|
const spot = hidden.get(membership.spotKey)?.activation ??
|
|
3109
3305
|
this.requireSpotManager().resolveRelocationActivation(this.meshName, spotIdentity);
|
|
3110
3306
|
if (actor === undefined)
|
|
3111
3307
|
throw new Error('Relocation membership Actor staging is missing.');
|
|
3112
3308
|
const state = this.requireActorManager().getState(actor.context.actorId);
|
|
3113
|
-
|
|
3114
|
-
const isEntryMembership = localDescriptor?.entrySpotId === spotIdentity;
|
|
3115
|
-
if (isEntryMembership) {
|
|
3309
|
+
if (targetMembership.actorSpotKind === contracts_1.ZLinkSpotKind.Entry) {
|
|
3116
3310
|
// Entry Spot identity is published as a Location membership key, but
|
|
3117
3311
|
// it is not a User Spot activation in the Actor runtime. The native
|
|
3118
3312
|
// restore already received the target MeshNode RID as its Entry
|
|
@@ -3121,7 +3315,7 @@ class LocalTargetPort {
|
|
|
3121
3315
|
state.clearJoinedSpot();
|
|
3122
3316
|
}
|
|
3123
3317
|
else {
|
|
3124
|
-
state.setJoinedSpot(spotIdentity, spot?.spot, membership.membershipEpoch);
|
|
3318
|
+
state.setJoinedSpot(spotIdentity, spot?.spot, membership.membershipEpoch, membership.spotObjectGeneration);
|
|
3125
3319
|
}
|
|
3126
3320
|
spot?.commitActorJoin(actor);
|
|
3127
3321
|
}
|
|
@@ -3213,47 +3407,20 @@ class LocalTargetPort {
|
|
|
3213
3407
|
hidden.replayPackets.push(packet);
|
|
3214
3408
|
}
|
|
3215
3409
|
async openAdmission(hidden) {
|
|
3216
|
-
if (hidden.actor
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
return;
|
|
3221
|
-
const state = this.requireActorManager().getState(hidden.actor.context.actorId);
|
|
3222
|
-
if (state === undefined)
|
|
3223
|
-
throw new Error('Relocated Actor state is not staged.');
|
|
3224
|
-
const results = await (0, actor_handoff_1.replayActorHandoffBacklog)(hidden.replayPackets, (parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef) => {
|
|
3225
|
-
if (state.spotId === undefined) {
|
|
3226
|
-
const runtime = this.options.spotNodeRuntime();
|
|
3227
|
-
if (runtime === undefined) {
|
|
3228
|
-
throw new Error('Relocated Entry Spot Actor dispatch is unavailable.');
|
|
3229
|
-
}
|
|
3230
|
-
return runtime.dispatchEntryActorPacket('remote', hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
|
|
3231
|
-
}
|
|
3232
|
-
return this.requireSpotManager().dispatchRoutedActorPacket('remote', state.spotId, hidden.actor.context.actorId, parts, returnResponse, remoteBoundSessionTarget, fallbackActorRef);
|
|
3233
|
-
});
|
|
3234
|
-
hidden.replayResults.push(...results);
|
|
3410
|
+
if (hidden.actor !== undefined) {
|
|
3411
|
+
this.requireActorManager().publishRelocationActor(hidden.actor.context.actorId);
|
|
3412
|
+
this.requireSpotManager().openActorJoinRelocation(this.meshName, actorJoinAdmissionIdentity(this.envelope), hidden.actor);
|
|
3413
|
+
}
|
|
3235
3414
|
}
|
|
3236
|
-
async
|
|
3237
|
-
if (hidden.
|
|
3238
|
-
throw new Error('Actor relocation cannot restore a Spot timer.');
|
|
3239
|
-
hidden.restoredTimers.push(timer);
|
|
3240
|
-
if (hidden.restoredTimers.length !== hidden.participant.timers.length)
|
|
3415
|
+
async normalize(hidden) {
|
|
3416
|
+
if (hidden.actor === undefined || hidden.replayPackets.length === 0)
|
|
3241
3417
|
return;
|
|
3242
|
-
hidden.
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
maxCatchUpTicks: value.maxCatchUpTicks,
|
|
3248
|
-
stopOnUnhandledException: value.stopOnUnhandledException,
|
|
3249
|
-
startedAtUnixMs: value.startedAtUnixMs,
|
|
3250
|
-
deliveryIndex: value.deliveryIndex,
|
|
3251
|
-
lastScheduledIndex: value.lastScheduledIndex,
|
|
3252
|
-
nextDueAtUnixMs: value.dueAtUnixMs,
|
|
3253
|
-
pendingTicks: value.pendingTicks
|
|
3254
|
-
})));
|
|
3418
|
+
if (hidden.replayStart === undefined || hidden.onReplayResult === undefined) {
|
|
3419
|
+
throw new Error('Relocated Actor backlog has no target activation owner.');
|
|
3420
|
+
}
|
|
3421
|
+
this.options.actorTransfer.admitRelocationPrefix(hidden.actor, hidden.participant.objectGeneration, hidden.replayPackets, hidden.replayStart, hidden.onReplayResult);
|
|
3422
|
+
hidden.replayPackets.length = 0;
|
|
3255
3423
|
}
|
|
3256
|
-
async normalize(_hidden) { }
|
|
3257
3424
|
async abort(hidden) {
|
|
3258
3425
|
if (hidden.actor !== undefined) {
|
|
3259
3426
|
await this.requireActorManager().abortRelocationActor(hidden.actor.context.actorId);
|
|
@@ -3311,13 +3478,14 @@ function rewriteAuthorityApplicationRoute(payload, target) {
|
|
|
3311
3478
|
}
|
|
3312
3479
|
return ((0, service_authority_payload_codec_1.rewriteServiceAuthorityRoute)(applicationPayload, target.owner, target) ?? applicationPayload);
|
|
3313
3480
|
}
|
|
3314
|
-
function actorMembershipTarget(
|
|
3315
|
-
const membership = envelope.memberships.find((value) => value.actorKey === actorKey);
|
|
3481
|
+
function actorMembershipTarget(membership, localDescriptor) {
|
|
3316
3482
|
if (membership === undefined)
|
|
3317
3483
|
return undefined;
|
|
3484
|
+
const spotId = (0, authority_key_codec_1.decodeAuthorityKey)({ value: membership.spotKey }).globalId;
|
|
3318
3485
|
return {
|
|
3319
|
-
actorSpotId:
|
|
3320
|
-
actorSpotGeneration: membership.spotObjectGeneration
|
|
3486
|
+
actorSpotId: spotId,
|
|
3487
|
+
actorSpotGeneration: membership.spotObjectGeneration,
|
|
3488
|
+
actorSpotKind: spotId === localDescriptor?.entrySpotId ? contracts_1.ZLinkSpotKind.Entry : contracts_1.ZLinkSpotKind.User
|
|
3321
3489
|
};
|
|
3322
3490
|
}
|
|
3323
3491
|
function primaryKey(envelope) {
|
|
@@ -3403,6 +3571,19 @@ function decodeQueuedHandoffPacket(message) {
|
|
|
3403
3571
|
...(messageFollowOrigin === undefined ? {} : { messageFollowOrigin })
|
|
3404
3572
|
}
|
|
3405
3573
|
])[0];
|
|
3574
|
+
const requestSource = decoded.source === undefined ? undefined : handoffSourceFence(decoded.source);
|
|
3575
|
+
if (messageFollowOrigin !== undefined &&
|
|
3576
|
+
requestSource !== undefined &&
|
|
3577
|
+
(0, routing_id_1.routingIdsEqual)(messageFollowOrigin.sourceNodeRid, requestSource.nodeRid)) {
|
|
3578
|
+
return {
|
|
3579
|
+
...decoded,
|
|
3580
|
+
index: index,
|
|
3581
|
+
messageFollowOrigin: {
|
|
3582
|
+
...messageFollowOrigin,
|
|
3583
|
+
requestSource
|
|
3584
|
+
}
|
|
3585
|
+
};
|
|
3586
|
+
}
|
|
3406
3587
|
return { ...decoded, index: index };
|
|
3407
3588
|
}
|
|
3408
3589
|
/**
|
|
@@ -3545,28 +3726,18 @@ function createServiceRelocationId(isInUse, entropy = node_crypto_1.randomBytes)
|
|
|
3545
3726
|
return candidate;
|
|
3546
3727
|
}
|
|
3547
3728
|
}
|
|
3548
|
-
function operationWireId(value) {
|
|
3549
|
-
const match = /^(\d+):(\d+)$/.exec(value);
|
|
3550
|
-
if (match === null)
|
|
3551
|
-
throw new Error(`Actor handoff operation '${value}' is not canonical.`);
|
|
3552
|
-
const result = { high: BigInt(match[1]), low: BigInt(match[2]) };
|
|
3553
|
-
if (result.high === 0n && result.low === 0n) {
|
|
3554
|
-
throw new Error('Actor handoff operation identity must not be zero.');
|
|
3555
|
-
}
|
|
3556
|
-
return result;
|
|
3557
|
-
}
|
|
3558
3729
|
function sameWireId(left, right) {
|
|
3559
3730
|
return left.high === right.high && left.low === right.low;
|
|
3560
3731
|
}
|
|
3561
3732
|
function sameCoordinator(left, right) {
|
|
3562
3733
|
return (left.ownerId === right.ownerId &&
|
|
3563
3734
|
BigInt(left.leaseGeneration) === right.leaseGeneration &&
|
|
3564
|
-
left.nodeRid
|
|
3735
|
+
(0, routing_id_1.routingIdsEqual)(left.nodeRid, right.nodeRid) &&
|
|
3565
3736
|
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
3566
3737
|
left.expectedAuthorityStoreVersion === right.expectedAuthorityStoreVersion);
|
|
3567
3738
|
}
|
|
3568
3739
|
function sameRelocationTarget(left, right) {
|
|
3569
|
-
return (left.nodeRid
|
|
3740
|
+
return ((0, routing_id_1.routingIdsEqual)(left.nodeRid, right.nodeRid) &&
|
|
3570
3741
|
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
3571
3742
|
left.ownerId === right.ownerId &&
|
|
3572
3743
|
BigInt(left.ownerLeaseGeneration) === right.ownerLeaseGeneration);
|
|
@@ -3600,7 +3771,7 @@ function isSourceOwner(current, expected) {
|
|
|
3600
3771
|
current.allocation.state === expected.allocation.state &&
|
|
3601
3772
|
current.allocation.objectKind === expected.allocation.objectKind &&
|
|
3602
3773
|
current.allocation.stableType === expected.allocation.stableType &&
|
|
3603
|
-
|
|
3774
|
+
(0, routing_id_1.routingIdsEqual)(current.allocation.descriptor.rid, expected.allocation.descriptor.rid) &&
|
|
3604
3775
|
current.allocation.descriptorLifecycleGeneration ===
|
|
3605
3776
|
expected.allocation.descriptorLifecycleGeneration);
|
|
3606
3777
|
}
|
|
@@ -3609,7 +3780,7 @@ function isExactCommittedTarget(current, expected, target, payload) {
|
|
|
3609
3780
|
current.authorityOwnerGeneration > expected.authorityOwnerGeneration &&
|
|
3610
3781
|
current.ownerId === target.ownerId &&
|
|
3611
3782
|
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
3612
|
-
|
|
3783
|
+
(0, routing_id_1.routingIdsEqual)(current.allocation.descriptor.rid, target.nodeRid) &&
|
|
3613
3784
|
current.allocation.descriptorLifecycleGeneration === target.nodeGeneration &&
|
|
3614
3785
|
Buffer.from(current.payload).equals(Buffer.from(payload)));
|
|
3615
3786
|
}
|
|
@@ -3632,40 +3803,52 @@ async function waitForRelocationRetry(delayMs, signal) {
|
|
|
3632
3803
|
function sameRequestSource(left, right) {
|
|
3633
3804
|
return (left.ownerId === right.ownerId &&
|
|
3634
3805
|
left.leaseGeneration === right.leaseGeneration &&
|
|
3635
|
-
left.nodeRid
|
|
3806
|
+
(0, routing_id_1.routingIdsEqual)(left.nodeRid, right.nodeRid) &&
|
|
3636
3807
|
left.nodeGeneration === right.nodeGeneration);
|
|
3637
3808
|
}
|
|
3638
3809
|
function handoffSourceFence(value) {
|
|
3639
3810
|
return {
|
|
3640
3811
|
ownerId: value.ownerId,
|
|
3641
3812
|
leaseGeneration: BigInt(value.ownerLeaseGeneration),
|
|
3642
|
-
nodeRid: value.nodeRid,
|
|
3813
|
+
nodeRid: (0, routing_id_1.decodeRoutingId)(value.nodeRid, value.nodeRidHex),
|
|
3643
3814
|
nodeGeneration: BigInt(value.nodeGeneration)
|
|
3644
3815
|
};
|
|
3645
3816
|
}
|
|
3646
|
-
function maintenanceReplyRelay(stage, coordinator, participantId, completion) {
|
|
3817
|
+
function maintenanceReplyRelay(stage, coordinator, participantId, completion, packet, serializers, flowEnabled) {
|
|
3647
3818
|
const value = completion.result.ok ? completion.result.response : undefined;
|
|
3819
|
+
const direct = packet.messageFollowOrigin?.requestSource !== undefined;
|
|
3820
|
+
const application = direct && completion.result.ok
|
|
3821
|
+
? (0, service_wire_m6a_codec_1.decodeApplicationPayloadView)((0, service_wire_m6a_codec_1.encodeMultipartApplicationPayload)((0, actor_reply_1.encodeActorReply)(Buffer.from(packet.header, 'base64'), protocol_1.ZLinkStreamMessageKind.Response, value, serializers, flowEnabled), service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_PACKET_NAME, service_wire_constants_generated_1.SERVICE_FRAMEWORK_MULTIPART_CONTENT_TYPE))
|
|
3822
|
+
: undefined;
|
|
3648
3823
|
return {
|
|
3649
3824
|
relocation: relocationWireId(stage.staging.envelope.aggregateId),
|
|
3650
3825
|
targetAttemptGeneration: stage.offer.prepare.targetAttemptGeneration,
|
|
3651
3826
|
coordinator,
|
|
3652
|
-
operation:
|
|
3827
|
+
operation: parseHandoffOperationId(completion.operationId),
|
|
3653
3828
|
replyRouteId: BigInt(completion.source.replyRouteId),
|
|
3654
3829
|
participantId,
|
|
3655
3830
|
sequence: BigInt(completion.index + 1),
|
|
3656
3831
|
terminalResult: completion.result.ok ? runtime_values_1.RequestResult.Ok : runtime_values_1.RequestResult.InternalError,
|
|
3657
3832
|
failureCode: completion.result.ok
|
|
3658
|
-
?
|
|
3659
|
-
:
|
|
3660
|
-
...(
|
|
3661
|
-
? {
|
|
3662
|
-
: {
|
|
3833
|
+
? service_wire_constants_generated_2.ServiceWireFrameworkErrorCode.none
|
|
3834
|
+
: service_wire_constants_generated_2.ServiceWireFrameworkErrorCode.requestFailed,
|
|
3835
|
+
...(application !== undefined
|
|
3836
|
+
? {
|
|
3663
3837
|
payload: {
|
|
3664
|
-
packetName:
|
|
3665
|
-
contentType:
|
|
3666
|
-
bytes:
|
|
3838
|
+
packetName: application.packetName,
|
|
3839
|
+
contentType: application.contentType,
|
|
3840
|
+
bytes: application.payload
|
|
3667
3841
|
}
|
|
3668
|
-
}
|
|
3842
|
+
}
|
|
3843
|
+
: value === undefined
|
|
3844
|
+
? {}
|
|
3845
|
+
: {
|
|
3846
|
+
payload: {
|
|
3847
|
+
packetName: SERVICE_RELOCATION_REPLY_PACKET_NAME,
|
|
3848
|
+
contentType: stream_wire_1.ZlinkStreamContentType.Json,
|
|
3849
|
+
bytes: Buffer.from(JSON.stringify(value), 'utf8')
|
|
3850
|
+
}
|
|
3851
|
+
})
|
|
3669
3852
|
};
|
|
3670
3853
|
}
|
|
3671
3854
|
function handoffResultFromRelay(relay) {
|
|
@@ -3697,6 +3880,13 @@ function wireIdText(value) {
|
|
|
3697
3880
|
class ServiceRelocationDataLostError extends Error {
|
|
3698
3881
|
}
|
|
3699
3882
|
exports.ServiceRelocationDataLostError = ServiceRelocationDataLostError;
|
|
3883
|
+
/** The local source lifecycle ended a pending relocation operation. */
|
|
3884
|
+
class ServiceRelocationRuntimeStoppedError extends Error {
|
|
3885
|
+
constructor() {
|
|
3886
|
+
super('Relocation runtime stopped.');
|
|
3887
|
+
this.name = 'AbortError';
|
|
3888
|
+
}
|
|
3889
|
+
}
|
|
3700
3890
|
/** Classifies relocation failure with the error model §2.1 mapping. */
|
|
3701
3891
|
function relocationFailedFailureCode(error) {
|
|
3702
3892
|
if (error instanceof ServiceRelocationDataLostError) {
|
|
@@ -3802,7 +3992,7 @@ function validatePrepareEnvelope(request, envelope) {
|
|
|
3802
3992
|
}
|
|
3803
3993
|
function validateTargetOneWayControl(stage, request, sourceNodeRid) {
|
|
3804
3994
|
const prepare = stage.offer.prepare;
|
|
3805
|
-
if (
|
|
3995
|
+
if (!(0, routing_id_1.routingIdsEqual)(sourceNodeRid, stage.offer.authenticatedSourceNodeRid) ||
|
|
3806
3996
|
!sameWireId(request.relocation, prepare.relocation) ||
|
|
3807
3997
|
request.targetAttemptGeneration !== prepare.targetAttemptGeneration ||
|
|
3808
3998
|
!sameCoordinator(request.coordinator, prepare.coordinator) ||
|
|
@@ -3859,12 +4049,7 @@ function canonicalQueuedFrozenRecord(participant, message, coordinator, target)
|
|
|
3859
4049
|
const identity = (0, authority_key_codec_1.decodeAuthorityKey)({ value: participant.key }).globalId;
|
|
3860
4050
|
const source = packet.source === undefined
|
|
3861
4051
|
? coordinatorSource(coordinator)
|
|
3862
|
-
:
|
|
3863
|
-
ownerId: packet.source.ownerId,
|
|
3864
|
-
leaseGeneration: BigInt(packet.source.ownerLeaseGeneration),
|
|
3865
|
-
nodeRid: packet.source.nodeRid,
|
|
3866
|
-
nodeGeneration: BigInt(packet.source.nodeGeneration)
|
|
3867
|
-
};
|
|
4052
|
+
: handoffSourceFence(packet.source);
|
|
3868
4053
|
const operationId = parseHandoffOperationId(packet.messageFollowContext.operationId);
|
|
3869
4054
|
if (packet.returnResponse && packet.source === undefined) {
|
|
3870
4055
|
throw new Error('Captured Actor request has no exact request-source fence.');
|
|
@@ -3884,7 +4069,7 @@ function canonicalQueuedFrozenRecord(participant, message, coordinator, target)
|
|
|
3884
4069
|
objectGeneration: participant.objectGeneration,
|
|
3885
4070
|
nodeRid: target.nodeRid,
|
|
3886
4071
|
nodeGeneration: target.nodeGeneration,
|
|
3887
|
-
authorityOwnerGeneration: participant.authorityOwnerGeneration
|
|
4072
|
+
authorityOwnerGeneration: participant.authorityOwnerGeneration,
|
|
3888
4073
|
ownerLeaseGeneration: target.ownerLeaseGeneration
|
|
3889
4074
|
},
|
|
3890
4075
|
operationId,
|
|
@@ -3967,43 +4152,40 @@ function sessionRelocationPendingKey(request) {
|
|
|
3967
4152
|
return sessionRelocationIdentityKey(request);
|
|
3968
4153
|
}
|
|
3969
4154
|
function sessionRelocationRouteSubmitKey(request, targetNodeRid) {
|
|
3970
|
-
return `${sessionRelocationIdentityKey(request)}:${
|
|
4155
|
+
return `${sessionRelocationIdentityKey(request)}:${(0, routing_id_1.encodeRoutingIdStorageHex)(targetNodeRid)}`;
|
|
3971
4156
|
}
|
|
3972
4157
|
function sessionRelocationResponseKey(response) {
|
|
3973
4158
|
return sessionRelocationIdentityKey(response);
|
|
3974
4159
|
}
|
|
3975
|
-
function sessionRelocationResponseFingerprint(response) {
|
|
3976
|
-
return (0, service_stateful_wire_codec_1.encodeSessionRelocationSealed)(response).toString('base64');
|
|
3977
|
-
}
|
|
3978
4160
|
function sessionRelocationIdentityKey(value) {
|
|
3979
4161
|
return `session-relocation:${(0, service_stateful_wire_codec_1.serviceSessionRelocationIdentityKey)(value)}:sealed`;
|
|
3980
4162
|
}
|
|
3981
4163
|
function validateSessionRelocationResponse(request, response) {
|
|
3982
4164
|
const sameCoordinator = request.coordinator.ownerId === response.coordinator.ownerId &&
|
|
3983
4165
|
request.coordinator.leaseGeneration === response.coordinator.leaseGeneration &&
|
|
3984
|
-
request.coordinator.nodeRid
|
|
4166
|
+
(0, routing_id_1.routingIdsEqual)(request.coordinator.nodeRid, response.coordinator.nodeRid) &&
|
|
3985
4167
|
request.coordinator.nodeGeneration === response.coordinator.nodeGeneration &&
|
|
3986
4168
|
request.coordinator.expectedAuthorityStoreVersion ===
|
|
3987
4169
|
response.coordinator.expectedAuthorityStoreVersion;
|
|
3988
|
-
const sameSession = request.session.sessionOwnerNodeRid
|
|
4170
|
+
const sameSession = (0, routing_id_1.routingIdsEqual)(request.session.sessionOwnerNodeRid, response.session.sessionOwnerNodeRid) &&
|
|
3989
4171
|
request.session.sessionOwnerNodeGeneration === response.session.sessionOwnerNodeGeneration &&
|
|
3990
4172
|
request.session.sessionOwnerId === response.session.sessionOwnerId &&
|
|
3991
4173
|
request.session.sessionOwnerLeaseGeneration === response.session.sessionOwnerLeaseGeneration &&
|
|
3992
|
-
request.session.sessionRid
|
|
4174
|
+
(0, routing_id_1.routingIdsEqual)(request.session.sessionRid, response.session.sessionRid) &&
|
|
3993
4175
|
request.session.bindingGeneration === response.session.bindingGeneration;
|
|
3994
4176
|
if (request.relocation.high !== response.relocation.high ||
|
|
3995
4177
|
request.relocation.low !== response.relocation.low ||
|
|
3996
4178
|
!sameCoordinator ||
|
|
3997
4179
|
!sameSession) {
|
|
3998
|
-
throw new
|
|
4180
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Session relocation ACK changed its exact fence.');
|
|
3999
4181
|
}
|
|
4000
4182
|
if (request.actor.actor.actorId !== response.actor.actor.actorId ||
|
|
4001
4183
|
request.actor.actor.generation !== response.actor.actor.generation ||
|
|
4002
|
-
request.actor.actor.nodeRid
|
|
4184
|
+
!(0, routing_id_1.routingIdsEqual)(request.actor.actor.nodeRid, response.actor.actor.nodeRid) ||
|
|
4003
4185
|
request.actor.targetNodeGeneration !== response.actor.targetNodeGeneration ||
|
|
4004
4186
|
request.actor.authorityOwnerGeneration !== response.actor.authorityOwnerGeneration ||
|
|
4005
4187
|
request.actor.ownerLeaseGeneration !== response.actor.ownerLeaseGeneration) {
|
|
4006
|
-
throw new
|
|
4188
|
+
throw new service_wire_m6a_codec_2.ServiceWireProtocolError('Command 43 ACK does not echo command 42.');
|
|
4007
4189
|
}
|
|
4008
4190
|
}
|
|
4009
4191
|
function replyRelayIdentityKey(value) {
|
|
@@ -4015,20 +4197,16 @@ function replyRelayIdentityKey(value) {
|
|
|
4015
4197
|
value.relocation.low.toString(),
|
|
4016
4198
|
value.coordinator.ownerId,
|
|
4017
4199
|
value.coordinator.leaseGeneration.toString(),
|
|
4018
|
-
value.coordinator.nodeRid,
|
|
4200
|
+
(0, routing_id_1.encodeRoutingIdStorageHex)(value.coordinator.nodeRid),
|
|
4019
4201
|
value.coordinator.nodeGeneration.toString(),
|
|
4020
4202
|
value.coordinator.expectedAuthorityStoreVersion
|
|
4021
4203
|
]);
|
|
4022
4204
|
}
|
|
4023
4205
|
function replyRelayPendingKey(ackTargetNodeRid, value) {
|
|
4024
|
-
return JSON.stringify([
|
|
4025
|
-
|
|
4026
|
-
|
|
4027
|
-
|
|
4028
|
-
payload[0] === 0x5a &&
|
|
4029
|
-
payload[1] === 0x4d &&
|
|
4030
|
-
payload[2] === 1 &&
|
|
4031
|
-
payload[3] === command);
|
|
4206
|
+
return JSON.stringify([
|
|
4207
|
+
(0, routing_id_1.encodeRoutingIdStorageHex)(ackTargetNodeRid),
|
|
4208
|
+
replyRelayIdentityKey(value)
|
|
4209
|
+
]);
|
|
4032
4210
|
}
|
|
4033
4211
|
function encodeHandoffQueuedMessages(backlog) {
|
|
4034
4212
|
return backlog.map((packet) => ({
|
|
@@ -4102,83 +4280,50 @@ function toServiceTimer(value) {
|
|
|
4102
4280
|
pendingTicks: value.pendingTicks
|
|
4103
4281
|
};
|
|
4104
4282
|
}
|
|
4105
|
-
const PREPARE_SIDEBAND_MAGIC = 'ZLNI';
|
|
4106
|
-
const PREPARE_SIDEBAND_VERSION = 1;
|
|
4107
|
-
const PREPARE_SIDEBAND_HEADER_BYTES = 11;
|
|
4108
|
-
const PREPARE_SIDEBAND_VERSION_OFFSET = 4;
|
|
4109
|
-
const PREPARE_SIDEBAND_KEY_LENGTH_OFFSET = 5;
|
|
4110
|
-
const PREPARE_SIDEBAND_PAYLOAD_LENGTH_OFFSET = 7;
|
|
4111
4283
|
const ACTOR_SESSION_JOURNAL_VERSION = 1;
|
|
4112
4284
|
const SERVICE_RELOCATION_REPLY_PACKET_NAME = 'zlink.relocation.reply';
|
|
4113
|
-
|
|
4114
|
-
|
|
4115
|
-
|
|
4116
|
-
|
|
4117
|
-
|
|
4118
|
-
|
|
4119
|
-
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
|
|
4132
|
-
|
|
4133
|
-
|
|
4134
|
-
|
|
4135
|
-
|
|
4136
|
-
|
|
4137
|
-
|
|
4138
|
-
|
|
4139
|
-
|
|
4140
|
-
|
|
4141
|
-
|
|
4142
|
-
|
|
4143
|
-
|
|
4144
|
-
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
|
|
4148
|
-
|
|
4149
|
-
|
|
4150
|
-
|
|
4151
|
-
|
|
4152
|
-
|
|
4153
|
-
|
|
4154
|
-
|
|
4155
|
-
if (keys.has(participantKey))
|
|
4156
|
-
return undefined;
|
|
4157
|
-
keys.add(participantKey);
|
|
4158
|
-
result.push({
|
|
4159
|
-
participantKey,
|
|
4160
|
-
payload: Buffer.from(bytes.subarray(PREPARE_SIDEBAND_HEADER_BYTES + keyLength))
|
|
4161
|
-
});
|
|
4162
|
-
}
|
|
4163
|
-
return result;
|
|
4164
|
-
}
|
|
4165
|
-
function attachPrepareBoundSessions(envelope, sideband) {
|
|
4166
|
-
if (sideband === undefined || sideband.length === 0)
|
|
4167
|
-
return envelope;
|
|
4168
|
-
const byKey = new Map(sideband.map((value) => [value.participantKey, Buffer.from(value.payload)]));
|
|
4169
|
-
const participants = envelope.participants.map((participant) => {
|
|
4170
|
-
const payload = byKey.get(participant.key);
|
|
4171
|
-
if (payload === undefined)
|
|
4172
|
-
return participant;
|
|
4173
|
-
if (participant.objectKind !== 'actor') {
|
|
4174
|
-
throw new TypeError('Bound-session relocation sideband names a non-Actor participant.');
|
|
4175
|
-
}
|
|
4176
|
-
byKey.delete(participant.key);
|
|
4177
|
-
return { ...participant, boundSessionState: payload };
|
|
4178
|
-
});
|
|
4179
|
-
if (byKey.size !== 0)
|
|
4180
|
-
throw new TypeError('Bound-session relocation sideband names an unknown participant.');
|
|
4181
|
-
return { ...envelope, participants };
|
|
4285
|
+
function attachCanonicalBoundSessions(envelope, request, meshName) {
|
|
4286
|
+
return {
|
|
4287
|
+
...envelope,
|
|
4288
|
+
participants: envelope.participants.map((participant) => {
|
|
4289
|
+
const binding = participant.sessionBinding;
|
|
4290
|
+
if (binding === undefined)
|
|
4291
|
+
return participant;
|
|
4292
|
+
if (participant.objectKind !== 'actor')
|
|
4293
|
+
throw new TypeError('Session binding names a non-Actor participant.');
|
|
4294
|
+
const sessionNodeRid = (0, routing_id_1.decodeRoutingId)('', Buffer.from(binding.sessionOwnerNodeRid).toString('hex'));
|
|
4295
|
+
const sessionRid = (0, routing_id_1.decodeRoutingId)('', Buffer.from(binding.sessionRid).toString('hex'));
|
|
4296
|
+
const session = {
|
|
4297
|
+
sessionOwnerNodeRid: sessionNodeRid,
|
|
4298
|
+
sessionOwnerNodeGeneration: binding.sessionOwnerNodeGeneration,
|
|
4299
|
+
sessionOwnerId: binding.sessionOwnerId,
|
|
4300
|
+
sessionOwnerLeaseGeneration: binding.sessionOwnerLeaseGeneration,
|
|
4301
|
+
sessionRid: sessionRid,
|
|
4302
|
+
bindingGeneration: binding.bindingGeneration
|
|
4303
|
+
};
|
|
4304
|
+
const target = {
|
|
4305
|
+
routerChannelId: meshName,
|
|
4306
|
+
targetNodeRid: sessionNodeRid,
|
|
4307
|
+
spotId: request.object.kind === 'actor'
|
|
4308
|
+
? (participant.rootSpotId ?? '')
|
|
4309
|
+
: request.object.spotId,
|
|
4310
|
+
sessionNodeRid,
|
|
4311
|
+
sessionRid,
|
|
4312
|
+
sessionOwnerNodeGeneration: binding.sessionOwnerNodeGeneration,
|
|
4313
|
+
sessionOwnerId: binding.sessionOwnerId,
|
|
4314
|
+
sessionOwnerLeaseGeneration: binding.sessionOwnerLeaseGeneration,
|
|
4315
|
+
bindingGeneration: binding.bindingGeneration,
|
|
4316
|
+
previousAuthorityOwnerGeneration: participant.authorityOwnerGeneration,
|
|
4317
|
+
relocationSealId: envelope.aggregateId,
|
|
4318
|
+
serviceWireRelocation: {
|
|
4319
|
+
relocation: request.relocation,
|
|
4320
|
+
coordinator: request.coordinator,
|
|
4321
|
+
session
|
|
4322
|
+
}
|
|
4323
|
+
};
|
|
4324
|
+
return { ...participant, boundSessionState: encodeActorSession(target) };
|
|
4325
|
+
})
|
|
4326
|
+
};
|
|
4182
4327
|
}
|
|
4183
4328
|
function encodeActorSession(target) {
|
|
4184
4329
|
if (target === undefined)
|
|
@@ -4187,9 +4332,14 @@ function encodeActorSession(target) {
|
|
|
4187
4332
|
version: ACTOR_SESSION_JOURNAL_VERSION,
|
|
4188
4333
|
routerChannelId: target.routerChannelId,
|
|
4189
4334
|
targetNodeRid: String(target.targetNodeRid),
|
|
4335
|
+
targetNodeRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.targetNodeRid),
|
|
4190
4336
|
spotId: String(target.spotId),
|
|
4191
4337
|
sessionNodeRid: target.sessionNodeRid === undefined ? undefined : String(target.sessionNodeRid),
|
|
4338
|
+
sessionNodeRidHex: target.sessionNodeRid === undefined
|
|
4339
|
+
? undefined
|
|
4340
|
+
: (0, routing_id_1.encodeRoutingIdStorageHex)(target.sessionNodeRid),
|
|
4192
4341
|
sessionRid: target.sessionRid === undefined ? undefined : String(target.sessionRid),
|
|
4342
|
+
sessionRidHex: target.sessionRid === undefined ? undefined : (0, routing_id_1.encodeRoutingIdStorageHex)(target.sessionRid),
|
|
4193
4343
|
sessionOwnerNodeGeneration: target.sessionOwnerNodeGeneration?.toString(),
|
|
4194
4344
|
sessionOwnerId: target.sessionOwnerId,
|
|
4195
4345
|
sessionOwnerLeaseGeneration: target.sessionOwnerLeaseGeneration?.toString(),
|
|
@@ -4204,14 +4354,17 @@ function encodeActorSession(target) {
|
|
|
4204
4354
|
relocationLow: target.serviceWireRelocation.relocation.low.toString(),
|
|
4205
4355
|
coordinatorOwnerId: target.serviceWireRelocation.coordinator.ownerId,
|
|
4206
4356
|
coordinatorLeaseGeneration: target.serviceWireRelocation.coordinator.leaseGeneration.toString(),
|
|
4207
|
-
coordinatorNodeRid: target.serviceWireRelocation.coordinator.nodeRid,
|
|
4357
|
+
coordinatorNodeRid: String(target.serviceWireRelocation.coordinator.nodeRid),
|
|
4358
|
+
coordinatorNodeRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.serviceWireRelocation.coordinator.nodeRid),
|
|
4208
4359
|
coordinatorNodeGeneration: target.serviceWireRelocation.coordinator.nodeGeneration.toString(),
|
|
4209
4360
|
expectedAuthorityStoreVersion: target.serviceWireRelocation.coordinator.expectedAuthorityStoreVersion,
|
|
4210
|
-
sessionOwnerNodeRid: target.serviceWireRelocation.session.sessionOwnerNodeRid,
|
|
4361
|
+
sessionOwnerNodeRid: String(target.serviceWireRelocation.session.sessionOwnerNodeRid),
|
|
4362
|
+
sessionOwnerNodeRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.serviceWireRelocation.session.sessionOwnerNodeRid),
|
|
4211
4363
|
sessionOwnerNodeGeneration: target.serviceWireRelocation.session.sessionOwnerNodeGeneration.toString(),
|
|
4212
4364
|
sessionOwnerId: target.serviceWireRelocation.session.sessionOwnerId,
|
|
4213
4365
|
sessionOwnerLeaseGeneration: target.serviceWireRelocation.session.sessionOwnerLeaseGeneration.toString(),
|
|
4214
|
-
sessionRid: target.serviceWireRelocation.session.sessionRid,
|
|
4366
|
+
sessionRid: String(target.serviceWireRelocation.session.sessionRid),
|
|
4367
|
+
sessionRidHex: (0, routing_id_1.encodeRoutingIdStorageHex)(target.serviceWireRelocation.session.sessionRid),
|
|
4215
4368
|
bindingGeneration: target.serviceWireRelocation.session.bindingGeneration.toString()
|
|
4216
4369
|
}
|
|
4217
4370
|
}), 'utf8');
|
|
@@ -4230,12 +4383,14 @@ function decodeActorSession(payload) {
|
|
|
4230
4383
|
const serviceWire = decodeActorSessionServiceWireFence(value.serviceWireRelocation);
|
|
4231
4384
|
return {
|
|
4232
4385
|
routerChannelId: value.routerChannelId,
|
|
4233
|
-
targetNodeRid: value.targetNodeRid,
|
|
4386
|
+
targetNodeRid: (0, routing_id_1.decodeRoutingId)(value.targetNodeRid, value.targetNodeRidHex),
|
|
4234
4387
|
spotId: value.spotId,
|
|
4235
4388
|
...(typeof value.sessionNodeRid === 'string'
|
|
4236
|
-
? { sessionNodeRid: value.sessionNodeRid }
|
|
4389
|
+
? { sessionNodeRid: (0, routing_id_1.decodeRoutingId)(value.sessionNodeRid, value.sessionNodeRidHex) }
|
|
4390
|
+
: {}),
|
|
4391
|
+
...(typeof value.sessionRid === 'string'
|
|
4392
|
+
? { sessionRid: (0, routing_id_1.decodeRoutingId)(value.sessionRid, value.sessionRidHex) }
|
|
4237
4393
|
: {}),
|
|
4238
|
-
...(typeof value.sessionRid === 'string' ? { sessionRid: value.sessionRid } : {}),
|
|
4239
4394
|
...(optionalBigInt('sessionOwnerNodeGeneration') === undefined
|
|
4240
4395
|
? {}
|
|
4241
4396
|
: { sessionOwnerNodeGeneration: optionalBigInt('sessionOwnerNodeGeneration') }),
|
|
@@ -4297,16 +4452,16 @@ function decodeActorSessionServiceWireFence(input) {
|
|
|
4297
4452
|
coordinator: {
|
|
4298
4453
|
ownerId: text('coordinatorOwnerId'),
|
|
4299
4454
|
leaseGeneration: positive('coordinatorLeaseGeneration'),
|
|
4300
|
-
nodeRid: text('coordinatorNodeRid'),
|
|
4455
|
+
nodeRid: (0, routing_id_1.decodeRoutingId)(text('coordinatorNodeRid'), value.coordinatorNodeRidHex),
|
|
4301
4456
|
nodeGeneration: positive('coordinatorNodeGeneration'),
|
|
4302
4457
|
expectedAuthorityStoreVersion: text('expectedAuthorityStoreVersion')
|
|
4303
4458
|
},
|
|
4304
4459
|
session: {
|
|
4305
|
-
sessionOwnerNodeRid:
|
|
4460
|
+
sessionOwnerNodeRid: (0, routing_id_1.decodeRoutingId)(typeof value.sessionOwnerNodeRid === 'string' ? value.sessionOwnerNodeRid : '', value.sessionOwnerNodeRidHex),
|
|
4306
4461
|
sessionOwnerNodeGeneration: positive('sessionOwnerNodeGeneration'),
|
|
4307
4462
|
sessionOwnerId: text('sessionOwnerId'),
|
|
4308
4463
|
sessionOwnerLeaseGeneration: positive('sessionOwnerLeaseGeneration'),
|
|
4309
|
-
sessionRid:
|
|
4464
|
+
sessionRid: (0, routing_id_1.decodeRoutingId)(typeof value.sessionRid === 'string' ? value.sessionRid : '', value.sessionRidHex),
|
|
4310
4465
|
bindingGeneration: positive('bindingGeneration')
|
|
4311
4466
|
}
|
|
4312
4467
|
};
|