@zlink-systems/framework 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/contracts/Channels/IZLinkFanoutClient.d.ts +0 -6
- package/dist/contracts/Common/CoreTypes.d.ts +6 -0
- package/dist/contracts/Common/CoreTypes.js +7 -0
- package/dist/contracts/Common/index.d.ts +1 -1
- package/dist/contracts/Common/index.js +0 -1
- package/dist/contracts/Configuration/InternalDefaults.d.ts +8 -0
- package/dist/contracts/Configuration/InternalDefaults.js +9 -1
- package/dist/contracts/Configuration/Registration.d.ts +13 -2
- package/dist/contracts/Configuration/Registration.js +58 -36
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.d.ts +13 -1
- package/dist/contracts/Configuration/RegistrationBuilderPolicy.js +38 -1
- package/dist/contracts/Configuration/RegistrationBuilders.d.ts +2 -3
- package/dist/contracts/Configuration/RegistrationBuilders.js +27 -32
- package/dist/contracts/Configuration/RegistrationTypes.d.ts +1 -0
- package/dist/contracts/Configuration/RegistrationValidators.js +15 -12
- package/dist/contracts/Configuration/TimerRegistrationValidator.js +3 -2
- package/dist/contracts/Dispatch/ZLinkDispatchOptions.d.ts +1 -1
- package/dist/contracts/Handlers/JsonContract.js +10 -4
- package/dist/contracts/Locations/Authority.d.ts +1 -1
- package/dist/contracts/Locations/Authority.js +3 -2
- package/dist/contracts/Locations/Options.js +1 -1
- package/dist/contracts/Locations/Stores.d.ts +12 -0
- package/dist/contracts/Locations/Stores.js +9 -0
- package/dist/contracts/RouteMesh/RuntimeTopology.d.ts +8 -0
- package/dist/contracts/Spots/Contracts.d.ts +1 -1
- package/dist/internal.d.ts +5 -0
- package/dist/internal.js +16 -1
- package/dist/nest-integration.d.ts +4 -2
- package/dist/nest-integration.js +14 -4
- package/dist/runtime/abort.d.ts +1 -0
- package/dist/runtime/abort.js +8 -5
- package/dist/runtime/activation-admission.d.ts +28 -0
- package/dist/runtime/activation-admission.js +57 -0
- package/dist/runtime/actors/actor-authority-payload-codec.d.ts +1 -2
- package/dist/runtime/actors/actor-authority-payload-codec.js +34 -18
- package/dist/runtime/actors/actor-client.d.ts +2 -2
- package/dist/runtime/actors/actor-client.js +42 -143
- package/dist/runtime/actors/actor-context.d.ts +3 -5
- package/dist/runtime/actors/actor-context.js +19 -17
- package/dist/runtime/actors/actor-handoff.d.ts +13 -7
- package/dist/runtime/actors/actor-handoff.js +36 -25
- package/dist/runtime/actors/actor-lifecycle-snapshot.js +2 -1
- package/dist/runtime/actors/actor-local-native-join.d.ts +11 -6
- package/dist/runtime/actors/actor-local-native-join.js +85 -123
- package/dist/runtime/actors/actor-native-lookup.js +2 -8
- package/dist/runtime/actors/actor-packet-relay-wire.d.ts +3 -3
- package/dist/runtime/actors/actor-packet-relay-wire.js +5 -3
- package/dist/runtime/actors/actor-remote-joiner.d.ts +3 -3
- package/dist/runtime/actors/actor-remote-joiner.js +11 -17
- package/dist/runtime/actors/actor-remote-wire.js +5 -3
- package/dist/runtime/actors/actor-request-deadline.d.ts +3 -0
- package/dist/runtime/actors/actor-request-deadline.js +31 -0
- package/dist/runtime/actors/actor-retry-delay.js +2 -1
- package/dist/runtime/actors/actor-runtime-contracts.d.ts +4 -2
- package/dist/runtime/actors/actor-runtime-state.d.ts +9 -21
- package/dist/runtime/actors/actor-runtime-state.js +70 -187
- package/dist/runtime/actors/actor-source-transfer.d.ts +5 -5
- package/dist/runtime/actors/bound-session-wire.js +2 -7
- package/dist/runtime/actors/deferred-join-completion.d.ts +19 -0
- package/dist/runtime/actors/deferred-join-completion.js +21 -0
- package/dist/runtime/actors/index.d.ts +10 -8
- package/dist/runtime/actors/index.js +101 -73
- package/dist/runtime/actors/spot-actor-dispatch.d.ts +3 -3
- package/dist/runtime/actors/spot-actor-dispatch.js +25 -10
- package/dist/runtime/actors/transferred-actor-rollback.d.ts +3 -5
- package/dist/runtime/actors/transferred-actor-rollback.js +9 -42
- package/dist/runtime/admission.d.ts +7 -1
- package/dist/runtime/admission.js +11 -1
- package/dist/runtime/application-jobs/application-job-queue-scope.d.ts +1 -1
- package/dist/runtime/application-jobs/application-job-queue-scope.js +1 -1
- package/dist/runtime/backend/contracts/index.d.ts +8 -33
- package/dist/runtime/backend/mesh-actor-session-node-adapter.js +3 -26
- package/dist/runtime/backend/mesh-dispatch-pump.d.ts +1 -2
- package/dist/runtime/backend/mesh-dispatch-pump.js +95 -107
- package/dist/runtime/backend/node/node-backend-adapter-factory.d.ts +1 -2
- package/dist/runtime/backend/node/node-backend-adapter-factory.js +4 -18
- package/dist/runtime/backend/node/node-backend-adapter-support.d.ts +7 -4
- package/dist/runtime/backend/node/node-backend-adapter-support.js +28 -34
- package/dist/runtime/backend/node/node-event-loop-poller.d.ts +2 -2
- package/dist/runtime/backend/node/node-event-loop-poller.js +11 -6
- package/dist/runtime/backend/node/node-monitor-backend-adapter.js +1 -0
- package/dist/runtime/backend/node/node-raw-binding-port.d.ts +1 -1
- package/dist/runtime/backend/node/node-raw-binding-port.js +51 -19
- package/dist/runtime/backend/node/node-raw-mesh-backend.d.ts +21 -19
- package/dist/runtime/backend/node/node-raw-mesh-backend.js +171 -111
- package/dist/runtime/backend/node/node-socket-backend-adapter.js +4 -3
- package/dist/runtime/backend/raw-binding-port.d.ts +13 -1
- package/dist/runtime/backend/runtime-values.d.ts +4 -1
- package/dist/runtime/backend/runtime-values.js +2 -0
- package/dist/runtime/channels/channel-clients.d.ts +20 -9
- package/dist/runtime/channels/channel-clients.js +68 -60
- package/dist/runtime/channels/channel-envelope-inspection.js +2 -1
- package/dist/runtime/channels/channel-envelope.js +47 -29
- package/dist/runtime/channels/channel-outbound-operations.d.ts +3 -0
- package/dist/runtime/channels/channel-outbound-operations.js +9 -18
- package/dist/runtime/channels/channel-receive-loops.d.ts +8 -3
- package/dist/runtime/channels/channel-receive-loops.js +27 -6
- package/dist/runtime/channels/channel-runtime-lifecycle.d.ts +12 -1
- package/dist/runtime/channels/channel-runtime-lifecycle.js +48 -28
- package/dist/runtime/channels/channel-runtime-manager.d.ts +2 -5
- package/dist/runtime/channels/channel-runtime-manager.js +4 -12
- package/dist/runtime/channels/channel-socket-options.js +3 -2
- package/dist/runtime/channels/channel-socket-registry.d.ts +30 -32
- package/dist/runtime/channels/channel-socket-registry.js +296 -235
- package/dist/runtime/channels/channel-transports.d.ts +11 -20
- package/dist/runtime/channels/channel-transports.js +68 -47
- package/dist/runtime/channels/client-server-location-runtime.d.ts +1 -1
- package/dist/runtime/channels/client-server-location-runtime.js +10 -11
- package/dist/runtime/channels/client-server-service-wire.d.ts +7 -2
- package/dist/runtime/channels/client-server-service-wire.js +32 -11
- package/dist/runtime/channels/dispatch-error-reporter.d.ts +1 -2
- package/dist/runtime/channels/dispatch-error-reporter.js +2 -1
- package/dist/runtime/channels/fanout-location-runtime.d.ts +16 -3
- package/dist/runtime/channels/fanout-location-runtime.js +133 -81
- package/dist/runtime/channels/spot-direct-envelope.d.ts +1 -0
- package/dist/runtime/channels/spot-direct-envelope.js +4 -3
- package/dist/runtime/channels/spot-node-route-transport.js +3 -2
- package/dist/runtime/diagnostics/diagnostic-text.d.ts +2 -0
- package/dist/runtime/diagnostics/diagnostic-text.js +7 -0
- package/dist/runtime/diagnostics/dispatch-error-details.d.ts +3 -0
- package/dist/runtime/diagnostics/dispatch-error-details.js +30 -2
- package/dist/runtime/diagnostics/flow-context.d.ts +6 -0
- package/dist/runtime/diagnostics/flow-context.js +16 -2
- package/dist/runtime/diagnostics/index.d.ts +4 -2
- package/dist/runtime/diagnostics/index.js +14 -4
- package/dist/runtime/diagnostics/message-flow.d.ts +1 -1
- package/dist/runtime/diagnostics/message-flow.js +10 -19
- package/dist/runtime/diagnostics/runtime-metrics.d.ts +94 -31
- package/dist/runtime/diagnostics/runtime-metrics.js +168 -105
- package/dist/runtime/diagnostics/runtime-observation-queue.js +3 -2
- package/dist/runtime/diagnostics/topology-runtime-projections.d.ts +2 -4
- package/dist/runtime/diagnostics/topology-runtime-projections.js +135 -154
- package/dist/runtime/disposal.d.ts +5 -0
- package/dist/runtime/disposal.js +22 -0
- package/dist/runtime/execution/index.d.ts +22 -9
- package/dist/runtime/execution/index.js +107 -37
- package/dist/runtime/execution/serial-execution-queue.d.ts +12 -7
- package/dist/runtime/execution/serial-execution-queue.js +99 -26
- package/dist/runtime/execution/state-lane.d.ts +8 -1
- package/dist/runtime/execution/state-lane.js +118 -3
- package/dist/runtime/foundation/actor-join-recovery-codec.d.ts +3 -0
- package/dist/runtime/foundation/actor-join-recovery-codec.js +15 -11
- package/dist/runtime/foundation/event-loop-resources.d.ts +2 -24
- package/dist/runtime/foundation/event-loop-resources.js +6 -119
- package/dist/runtime/foundation/listener-records.d.ts +11 -0
- package/dist/runtime/foundation/listener-records.js +20 -0
- package/dist/runtime/foundation/operation-identity.d.ts +2 -0
- package/dist/runtime/foundation/operation-identity.js +9 -5
- package/dist/runtime/foundation/operation-registry.d.ts +5 -7
- package/dist/runtime/foundation/operation-registry.js +34 -44
- package/dist/runtime/foundation/raw-service-mesh-runtime.d.ts +18 -26
- package/dist/runtime/foundation/raw-service-mesh-runtime.js +122 -469
- package/dist/runtime/foundation/runtime-state-projections.d.ts +3 -1
- package/dist/runtime/foundation/runtime-state-projections.js +8 -0
- package/dist/runtime/foundation/service-authority-payload-codec.d.ts +1 -0
- package/dist/runtime/foundation/service-authority-payload-codec.js +30 -12
- package/dist/runtime/foundation/service-discovery-registry.d.ts +4 -0
- package/dist/runtime/foundation/service-discovery-registry.js +22 -3
- package/dist/runtime/foundation/service-instance-activation-recovery-codec.js +8 -4
- package/dist/runtime/foundation/service-liveness-registry.js +2 -1
- package/dist/runtime/foundation/service-maintenance-runtime.d.ts +0 -1
- package/dist/runtime/foundation/service-maintenance-runtime.js +23 -8
- package/dist/runtime/foundation/service-metadata-codec.js +3 -3
- package/dist/runtime/foundation/service-relocation-aggregate-committer.d.ts +2 -5
- package/dist/runtime/foundation/service-relocation-aggregate-committer.js +1 -48
- package/dist/runtime/foundation/service-relocation-coordinator.d.ts +9 -0
- package/dist/runtime/foundation/service-relocation-coordinator.js +14 -1
- package/dist/runtime/foundation/service-relocation-object-owner.d.ts +8 -0
- package/dist/runtime/foundation/service-relocation-object-owner.js +19 -6
- package/dist/runtime/foundation/service-relocation-runtime.js +43 -32
- package/dist/runtime/foundation/service-runtime-contracts.d.ts +40 -8
- package/dist/runtime/foundation/service-runtime-contracts.js +25 -1
- package/dist/runtime/foundation/service-session-binding-ingress-port.d.ts +0 -19
- package/dist/runtime/foundation/service-stateful-registry.d.ts +2 -0
- package/dist/runtime/foundation/service-stateful-registry.js +10 -2
- package/dist/runtime/foundation/service-stateful-runtime.d.ts +28 -24
- package/dist/runtime/foundation/service-stateful-runtime.js +297 -244
- package/dist/runtime/foundation/service-stateful-wire-codec.d.ts +12 -7
- package/dist/runtime/foundation/service-stateful-wire-codec.js +240 -143
- package/dist/runtime/foundation/service-topology-registry.d.ts +1 -3
- package/dist/runtime/foundation/service-topology-registry.js +22 -61
- package/dist/runtime/foundation/service-wire-binary-primitives.d.ts +5 -0
- package/dist/runtime/foundation/service-wire-binary-primitives.js +6 -0
- package/dist/runtime/foundation/service-wire-codec.js +10 -8
- package/dist/runtime/foundation/service-wire-constants.generated.d.ts +1 -0
- package/dist/runtime/foundation/service-wire-constants.generated.js +2 -1
- package/dist/runtime/foundation/service-wire-m6a-codec.d.ts +2 -0
- package/dist/runtime/foundation/service-wire-m6a-codec.js +23 -13
- package/dist/runtime/framework-errors-internal.d.ts +2 -1
- package/dist/runtime/framework-errors-internal.js +343 -122
- package/dist/runtime/handlers/handler-instance-scope.d.ts +3 -1
- package/dist/runtime/handlers/handler-instance-scope.js +61 -74
- package/dist/runtime/host/actor-packet-relay.d.ts +9 -6
- package/dist/runtime/host/actor-packet-relay.js +48 -91
- package/dist/runtime/host/actor-placement-coordinator.js +7 -5
- package/dist/runtime/host/actor-runtime-options-factory.d.ts +2 -0
- package/dist/runtime/host/actor-runtime-options-factory.js +17 -4
- package/dist/runtime/host/actor-transfer-runtime.d.ts +28 -28
- package/dist/runtime/host/actor-transfer-runtime.js +210 -256
- package/dist/runtime/host/application-job-queue.js +16 -33
- package/dist/runtime/host/bound-session-relay.js +1 -1
- package/dist/runtime/host/channel-runtime-options-factory.js +1 -1
- package/dist/runtime/host/index.d.ts +19 -14
- package/dist/runtime/host/index.js +234 -175
- package/dist/runtime/host/instance-activation-authority.d.ts +2 -3
- package/dist/runtime/host/instance-activation-authority.js +17 -45
- package/dist/runtime/host/location-runtime-owner.d.ts +1 -1
- package/dist/runtime/host/location-runtime-owner.js +2 -2
- package/dist/runtime/host/remote-actor-join-receiver.d.ts +2 -2
- package/dist/runtime/host/remote-actor-join-receiver.js +1 -14
- package/dist/runtime/host/remote-actor-packet-target-store.d.ts +3 -2
- package/dist/runtime/host/remote-actor-packet-target-store.js +8 -47
- package/dist/runtime/host/remote-bound-session-relay.d.ts +2 -3
- package/dist/runtime/host/remote-bound-session-relay.js +11 -23
- package/dist/runtime/host/route-mesh-runtime.d.ts +14 -10
- package/dist/runtime/host/route-mesh-runtime.js +178 -257
- package/dist/runtime/host/runtime-shutdown.d.ts +2 -0
- package/dist/runtime/host/runtime-shutdown.js +18 -21
- package/dist/runtime/host/service-relocation-control.js +6 -1
- package/dist/runtime/host/service-relocation-host-runtime.d.ts +71 -38
- package/dist/runtime/host/service-relocation-host-runtime.js +816 -651
- package/dist/runtime/host/spot-address-transport.d.ts +2 -3
- package/dist/runtime/host/spot-address-transport.js +34 -136
- package/dist/runtime/host/spot-node-runtime-options-factory.d.ts +1 -1
- package/dist/runtime/host/spot-node-runtime-options-factory.js +3 -2
- package/dist/runtime/host/spot-runtime-options-factory.d.ts +4 -7
- package/dist/runtime/host/spot-runtime-options-factory.js +53 -21
- package/dist/runtime/host/stateful-authority-route-runtime.js +6 -2
- package/dist/runtime/host/user-spot-creation-coordinator.d.ts +14 -6
- package/dist/runtime/host/user-spot-creation-coordinator.js +96 -95
- package/dist/runtime/locations/aggregate-inventory-store.js +13 -10
- package/dist/runtime/locations/authority-key-codec.js +4 -3
- package/dist/runtime/locations/auto-connect-reconciler.d.ts +4 -0
- package/dist/runtime/locations/auto-connect-reconciler.js +37 -25
- package/dist/runtime/locations/creation-operation-id.js +6 -4
- package/dist/runtime/locations/domain-store-contract.d.ts +3 -3
- package/dist/runtime/locations/in-memory-authority-store.js +31 -15
- package/dist/runtime/locations/in-memory-location-store.d.ts +2 -2
- package/dist/runtime/locations/in-memory-location-store.js +17 -14
- package/dist/runtime/locations/in-memory-provider-location-store.d.ts +1 -0
- package/dist/runtime/locations/in-memory-provider-location-store.js +79 -27
- package/dist/runtime/locations/key-codec.js +3 -2
- package/dist/runtime/locations/lease-tracker.d.ts +3 -5
- package/dist/runtime/locations/lease-tracker.js +7 -11
- package/dist/runtime/locations/lifecycle.d.ts +5 -10
- package/dist/runtime/locations/lifecycle.js +4 -33
- package/dist/runtime/locations/location-store-failure.d.ts +2 -0
- package/dist/runtime/locations/location-store-failure.js +7 -0
- package/dist/runtime/locations/location-store-pages.d.ts +3 -0
- package/dist/runtime/locations/location-store-pages.js +13 -0
- package/dist/runtime/locations/location-store-repository.d.ts +5 -7
- package/dist/runtime/locations/location-store-repository.js +303 -332
- package/dist/runtime/locations/relocation-blob.js +18 -5
- package/dist/runtime/locations/resolvers.d.ts +5 -5
- package/dist/runtime/locations/resolvers.js +44 -30
- package/dist/runtime/locations/runtime.d.ts +5 -4
- package/dist/runtime/locations/runtime.js +123 -92
- package/dist/runtime/locations/spot-location-claims.d.ts +8 -2
- package/dist/runtime/locations/spot-location-claims.js +40 -23
- package/dist/runtime/messaging/payload-codec.js +26 -20
- package/dist/runtime/messaging/submission-result.d.ts +3 -0
- package/dist/runtime/messaging/submission-result.js +48 -0
- package/dist/runtime/protocol/service_wire_codec.generated.d.ts +2 -0
- package/dist/runtime/protocol/service_wire_codec.generated.js +17 -8
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.d.ts +1 -0
- package/dist/runtime/protocol/service_wire_pilot_codec.generated.js +3 -2
- package/dist/runtime/spots/formal-remote-actor-admission-registry.d.ts +3 -3
- package/dist/runtime/spots/formal-remote-actor-admission-registry.js +4 -4
- package/dist/runtime/spots/formal-remote-actor-transfer-registry.d.ts +6 -3
- package/dist/runtime/spots/index.d.ts +29 -29
- package/dist/runtime/spots/index.js +441 -313
- package/dist/runtime/spots/spot-activation-registry.d.ts +4 -4
- package/dist/runtime/spots/spot-activation-registry.js +27 -60
- package/dist/runtime/spots/spot-activation-state.d.ts +2 -11
- package/dist/runtime/spots/spot-activation-state.js +4 -17
- package/dist/runtime/spots/spot-activation.d.ts +10 -9
- package/dist/runtime/spots/spot-activation.js +103 -81
- package/dist/runtime/spots/spot-actor-admission-coordinator.d.ts +1 -2
- package/dist/runtime/spots/spot-actor-admission-coordinator.js +3 -40
- package/dist/runtime/spots/spot-actor-join-dispatch.d.ts +10 -18
- package/dist/runtime/spots/spot-actor-join-dispatch.js +5 -55
- package/dist/runtime/spots/spot-actor-lifecycle-drain.js +2 -2
- package/dist/runtime/spots/spot-actor-membership.d.ts +7 -5
- package/dist/runtime/spots/spot-actor-membership.js +44 -53
- package/dist/runtime/spots/spot-actor-packet-dispatch.d.ts +0 -1
- package/dist/runtime/spots/spot-actor-packet-dispatch.js +0 -9
- package/dist/runtime/spots/spot-actor-packet-relay-dispatch.d.ts +2 -2
- package/dist/runtime/spots/spot-actor-packet-relay-dispatch.js +1 -7
- package/dist/runtime/spots/spot-context.d.ts +2 -2
- package/dist/runtime/spots/spot-entry-activation.d.ts +3 -4
- package/dist/runtime/spots/spot-entry-activation.js +9 -21
- package/dist/runtime/spots/spot-lifecycle-metrics.js +3 -2
- package/dist/runtime/spots/spot-manager-public.d.ts +5 -6
- package/dist/runtime/spots/spot-manager-public.js +6 -5
- package/dist/runtime/spots/spot-node-autoconnect.js +22 -31
- package/dist/runtime/spots/spot-node-runtime-manager.d.ts +22 -21
- package/dist/runtime/spots/spot-node-runtime-manager.js +73 -82
- package/dist/runtime/spots/spot-outbound.d.ts +4 -3
- package/dist/runtime/spots/spot-outbound.js +10 -37
- package/dist/runtime/spots/spot-remote-codec.js +5 -3
- package/dist/runtime/spots/spot-route-packet-dispatch.d.ts +2 -2
- package/dist/runtime/spots/spot-route-packet-dispatch.js +8 -8
- package/dist/runtime/spots/spot-route-replies.d.ts +2 -2
- package/dist/runtime/spots/spot-route-replies.js +3 -1
- package/dist/runtime/spots/spot-routed-actor-admission.d.ts +3 -2
- package/dist/runtime/spots/spot-routed-actor-admission.js +3 -2
- package/dist/runtime/spots/spot-routed-frame-dispatch.d.ts +2 -7
- package/dist/runtime/spots/spot-routed-frame-dispatch.js +1 -70
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.d.ts +4 -0
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.js +65 -15
- package/dist/runtime/spots/spot-runtime-ports.d.ts +4 -20
- package/dist/runtime/spots/spot-serial-executor.d.ts +5 -0
- package/dist/runtime/spots/spot-serial-executor.js +9 -2
- package/dist/runtime/spots/spot-serial-turn-executor.d.ts +29 -6
- package/dist/runtime/spots/spot-serial-turn-executor.js +89 -13
- package/dist/runtime/spots/spot-timer.d.ts +3 -2
- package/dist/runtime/streams/actor-session-binding-registry.d.ts +17 -32
- package/dist/runtime/streams/actor-session-binding-registry.js +145 -367
- package/dist/runtime/streams/actor-session-binding-runtime-owner.d.ts +2 -6
- package/dist/runtime/streams/bound-actor-relay-sender.d.ts +6 -2
- package/dist/runtime/streams/bound-actor-relay-sender.js +57 -33
- package/dist/runtime/streams/bound-session-service.d.ts +7 -0
- package/dist/runtime/streams/bound-session-service.js +12 -19
- package/dist/runtime/streams/index.d.ts +15 -14
- package/dist/runtime/streams/index.js +38 -39
- package/dist/runtime/streams/managed-stream.d.ts +3 -0
- package/dist/runtime/streams/managed-stream.js +17 -8
- package/dist/runtime/streams/native-fallback-bound-session.js +1 -3
- package/dist/runtime/streams/protocol.d.ts +6 -33
- package/dist/runtime/streams/protocol.js +38 -110
- package/dist/runtime/streams/session-actor-coordinator.d.ts +2 -3
- package/dist/runtime/streams/session-actor-coordinator.js +12 -4
- package/dist/runtime/streams/session-calls.d.ts +1 -1
- package/dist/runtime/streams/session-calls.js +5 -4
- package/dist/runtime/streams/session-context.d.ts +6 -6
- package/dist/runtime/streams/session-context.js +22 -4
- package/dist/runtime/streams/session-requests.js +2 -1
- package/dist/runtime/streams/session-serial-executor.js +2 -2
- package/dist/runtime/streams/stream-binding-runtime-ports.d.ts +16 -1
- package/dist/runtime/streams/stream-frame-factory.js +2 -2
- package/dist/runtime/streams/stream-session-runtime.d.ts +10 -11
- package/dist/runtime/streams/stream-session-runtime.js +25 -33
- package/dist/runtime/workers/index.d.ts +2 -2
- package/dist/runtime/workers/index.js +5 -4
- package/package.json +3 -3
- package/src/contracts/Channels/IZLinkFanoutClient.ts +0 -7
- package/src/contracts/Common/CoreTypes.ts +7 -0
- package/src/contracts/Common/index.ts +1 -1
- package/src/contracts/Configuration/InternalDefaults.ts +12 -0
- package/src/contracts/Configuration/Registration.ts +92 -48
- package/src/contracts/Configuration/RegistrationBuilderPolicy.ts +44 -2
- package/src/contracts/Configuration/RegistrationBuilders.ts +77 -68
- package/src/contracts/Configuration/RegistrationTypes.ts +1 -0
- package/src/contracts/Configuration/RegistrationValidators.ts +19 -13
- package/src/contracts/Configuration/TimerRegistrationValidator.ts +4 -2
- package/src/contracts/Dispatch/ZLinkDispatchOptions.ts +1 -0
- package/src/contracts/Handlers/JsonContract.ts +14 -4
- package/src/contracts/Locations/Authority.ts +7 -3
- package/src/contracts/Locations/Options.ts +1 -1
- package/src/contracts/Locations/Stores.ts +14 -0
- package/src/contracts/RouteMesh/RuntimeTopology.ts +10 -0
- package/src/contracts/Spots/Contracts.ts +1 -1
- package/src/internal.ts +15 -0
- package/src/nest-integration.ts +28 -10
- package/src/runtime/abort.ts +6 -5
- package/src/runtime/activation-admission.ts +72 -0
- package/src/runtime/actors/actor-authority-payload-codec.ts +58 -20
- package/src/runtime/actors/actor-client.ts +77 -204
- package/src/runtime/actors/actor-context.ts +45 -39
- package/src/runtime/actors/actor-handoff.ts +55 -45
- package/src/runtime/actors/actor-lifecycle-snapshot.ts +2 -1
- package/src/runtime/actors/actor-local-native-join.ts +174 -215
- package/src/runtime/actors/actor-native-lookup.ts +2 -9
- package/src/runtime/actors/actor-packet-relay-wire.ts +18 -12
- package/src/runtime/actors/actor-remote-joiner.ts +15 -36
- package/src/runtime/actors/actor-remote-wire.ts +5 -3
- package/src/runtime/actors/actor-request-deadline.ts +49 -0
- package/src/runtime/actors/actor-retry-delay.ts +2 -1
- package/src/runtime/actors/actor-runtime-contracts.ts +4 -10
- package/src/runtime/actors/actor-runtime-state.ts +85 -247
- package/src/runtime/actors/actor-source-transfer.ts +5 -18
- package/src/runtime/actors/bound-session-wire.ts +2 -8
- package/src/runtime/actors/deferred-join-completion.ts +44 -0
- package/src/runtime/actors/index.ts +122 -88
- package/src/runtime/actors/spot-actor-dispatch.ts +60 -29
- package/src/runtime/actors/transferred-actor-rollback.ts +11 -62
- package/src/runtime/admission.ts +13 -2
- package/src/runtime/application-jobs/application-job-queue-scope.ts +4 -4
- package/src/runtime/backend/contracts/index.ts +15 -42
- package/src/runtime/backend/mesh-actor-session-node-adapter.ts +4 -28
- package/src/runtime/backend/mesh-dispatch-pump.ts +111 -117
- package/src/runtime/backend/node/node-backend-adapter-factory.ts +5 -25
- package/src/runtime/backend/node/node-backend-adapter-support.ts +32 -35
- package/src/runtime/backend/node/node-event-loop-poller.ts +16 -8
- package/src/runtime/backend/node/node-monitor-backend-adapter.ts +1 -0
- package/src/runtime/backend/node/node-raw-binding-port.ts +58 -23
- package/src/runtime/backend/node/node-raw-mesh-backend.ts +252 -162
- package/src/runtime/backend/node/node-socket-backend-adapter.ts +5 -4
- package/src/runtime/backend/raw-binding-port.ts +14 -1
- package/src/runtime/backend/runtime-values.ts +4 -1
- package/src/runtime/channels/channel-clients.ts +132 -96
- package/src/runtime/channels/channel-envelope-inspection.ts +2 -1
- package/src/runtime/channels/channel-envelope.ts +58 -29
- package/src/runtime/channels/channel-outbound-operations.ts +10 -31
- package/src/runtime/channels/channel-receive-loops.ts +39 -7
- package/src/runtime/channels/channel-runtime-lifecycle.ts +60 -27
- package/src/runtime/channels/channel-runtime-manager.ts +8 -15
- package/src/runtime/channels/channel-socket-options.ts +8 -2
- package/src/runtime/channels/channel-socket-registry.ts +369 -262
- package/src/runtime/channels/channel-transports.ts +161 -115
- package/src/runtime/channels/client-server-location-runtime.ts +17 -17
- package/src/runtime/channels/client-server-service-wire.ts +40 -12
- package/src/runtime/channels/dispatch-error-reporter.ts +7 -5
- package/src/runtime/channels/fanout-location-runtime.ts +175 -106
- package/src/runtime/channels/spot-direct-envelope.ts +1 -1
- package/src/runtime/channels/spot-node-route-transport.ts +5 -2
- package/src/runtime/diagnostics/diagnostic-text.ts +4 -0
- package/src/runtime/diagnostics/dispatch-error-details.ts +36 -1
- package/src/runtime/diagnostics/flow-context.ts +16 -2
- package/src/runtime/diagnostics/index.ts +27 -17
- package/src/runtime/diagnostics/message-flow.ts +15 -25
- package/src/runtime/diagnostics/runtime-metrics.ts +181 -113
- package/src/runtime/diagnostics/runtime-observation-queue.ts +3 -2
- package/src/runtime/diagnostics/topology-runtime-projections.ts +194 -162
- package/src/runtime/disposal.ts +19 -0
- package/src/runtime/execution/index.ts +116 -38
- package/src/runtime/execution/serial-execution-queue.ts +179 -34
- package/src/runtime/execution/state-lane.ts +142 -4
- package/src/runtime/foundation/actor-join-recovery-codec.ts +13 -8
- package/src/runtime/foundation/event-loop-resources.ts +5 -122
- package/src/runtime/foundation/listener-records.ts +21 -0
- package/src/runtime/foundation/operation-identity.ts +11 -5
- package/src/runtime/foundation/operation-registry.ts +33 -50
- package/src/runtime/foundation/raw-service-mesh-runtime.ts +170 -571
- package/src/runtime/foundation/runtime-state-projections.ts +9 -1
- package/src/runtime/foundation/service-authority-payload-codec.ts +41 -13
- package/src/runtime/foundation/service-discovery-registry.ts +32 -3
- package/src/runtime/foundation/service-instance-activation-recovery-codec.ts +12 -4
- package/src/runtime/foundation/service-liveness-registry.ts +2 -1
- package/src/runtime/foundation/service-maintenance-runtime.ts +30 -19
- package/src/runtime/foundation/service-metadata-codec.ts +2 -2
- package/src/runtime/foundation/service-relocation-aggregate-committer.ts +3 -66
- package/src/runtime/foundation/service-relocation-coordinator.ts +13 -0
- package/src/runtime/foundation/service-relocation-object-owner.ts +23 -7
- package/src/runtime/foundation/service-relocation-runtime.ts +57 -35
- package/src/runtime/foundation/service-runtime-contracts.ts +46 -10
- package/src/runtime/foundation/service-session-binding-ingress-port.ts +0 -25
- package/src/runtime/foundation/service-stateful-registry.ts +14 -2
- package/src/runtime/foundation/service-stateful-runtime.ts +435 -327
- package/src/runtime/foundation/service-stateful-wire-codec.ts +329 -199
- package/src/runtime/foundation/service-topology-registry.ts +30 -79
- package/src/runtime/foundation/service-wire-binary-primitives.ts +6 -0
- package/src/runtime/foundation/service-wire-codec.ts +15 -8
- package/src/runtime/foundation/service-wire-constants.generated.ts +1 -0
- package/src/runtime/foundation/service-wire-m6a-codec.ts +29 -15
- package/src/runtime/framework-errors-internal.ts +380 -134
- package/src/runtime/handlers/handler-instance-scope.ts +68 -95
- package/src/runtime/host/actor-packet-relay.ts +57 -147
- package/src/runtime/host/actor-placement-coordinator.ts +8 -5
- package/src/runtime/host/actor-runtime-options-factory.ts +27 -8
- package/src/runtime/host/actor-transfer-runtime.ts +266 -418
- package/src/runtime/host/application-job-queue.ts +28 -34
- package/src/runtime/host/bound-session-relay.ts +1 -2
- package/src/runtime/host/channel-runtime-options-factory.ts +1 -2
- package/src/runtime/host/index.ts +408 -312
- package/src/runtime/host/instance-activation-authority.ts +89 -109
- package/src/runtime/host/location-runtime-owner.ts +4 -2
- package/src/runtime/host/remote-actor-join-receiver.ts +3 -31
- package/src/runtime/host/remote-actor-packet-target-store.ts +17 -58
- package/src/runtime/host/remote-bound-session-relay.ts +12 -44
- package/src/runtime/host/route-mesh-runtime.ts +236 -275
- package/src/runtime/host/runtime-shutdown.ts +17 -21
- package/src/runtime/host/service-relocation-control.ts +14 -1
- package/src/runtime/host/service-relocation-host-runtime.ts +1161 -904
- package/src/runtime/host/spot-address-transport.ts +73 -195
- package/src/runtime/host/spot-node-runtime-options-factory.ts +6 -3
- package/src/runtime/host/spot-runtime-options-factory.ts +92 -30
- package/src/runtime/host/stateful-authority-route-runtime.ts +8 -4
- package/src/runtime/host/user-spot-creation-coordinator.ts +182 -163
- package/src/runtime/locations/aggregate-inventory-store.ts +16 -13
- package/src/runtime/locations/authority-key-codec.ts +7 -3
- package/src/runtime/locations/auto-connect-reconciler.ts +39 -26
- package/src/runtime/locations/creation-operation-id.ts +6 -3
- package/src/runtime/locations/domain-store-contract.ts +12 -3
- package/src/runtime/locations/in-memory-authority-store.ts +44 -15
- package/src/runtime/locations/in-memory-location-store.ts +50 -39
- package/src/runtime/locations/in-memory-provider-location-store.ts +110 -31
- package/src/runtime/locations/key-codec.ts +3 -2
- package/src/runtime/locations/lease-tracker.ts +8 -14
- package/src/runtime/locations/lifecycle.ts +16 -45
- package/src/runtime/locations/location-store-failure.ts +4 -0
- package/src/runtime/locations/location-store-pages.ts +18 -0
- package/src/runtime/locations/location-store-repository.ts +450 -478
- package/src/runtime/locations/relocation-blob.ts +17 -6
- package/src/runtime/locations/resolvers.ts +90 -58
- package/src/runtime/locations/runtime.ts +220 -110
- package/src/runtime/locations/spot-location-claims.ts +54 -26
- package/src/runtime/messaging/payload-codec.ts +35 -24
- package/src/runtime/messaging/submission-result.ts +57 -1
- package/src/runtime/protocol/service_wire_codec.generated.ts +27 -10
- package/src/runtime/protocol/service_wire_pilot_codec.generated.ts +4 -4
- package/src/runtime/spots/formal-remote-actor-admission-registry.ts +6 -6
- package/src/runtime/spots/formal-remote-actor-transfer-registry.ts +6 -3
- package/src/runtime/spots/index.ts +687 -437
- package/src/runtime/spots/spot-activation-registry.ts +42 -71
- package/src/runtime/spots/spot-activation-state.ts +4 -16
- package/src/runtime/spots/spot-activation.ts +167 -115
- package/src/runtime/spots/spot-actor-admission-coordinator.ts +5 -57
- package/src/runtime/spots/spot-actor-join-dispatch.ts +25 -79
- package/src/runtime/spots/spot-actor-lifecycle-drain.ts +20 -18
- package/src/runtime/spots/spot-actor-membership.ts +107 -75
- package/src/runtime/spots/spot-actor-packet-dispatch.ts +0 -16
- package/src/runtime/spots/spot-actor-packet-relay-dispatch.ts +3 -15
- package/src/runtime/spots/spot-context.ts +3 -3
- package/src/runtime/spots/spot-entry-activation.ts +13 -33
- package/src/runtime/spots/spot-lifecycle-metrics.ts +3 -2
- package/src/runtime/spots/spot-manager-public.ts +29 -21
- package/src/runtime/spots/spot-node-autoconnect.ts +20 -26
- package/src/runtime/spots/spot-node-runtime-manager.ts +145 -140
- package/src/runtime/spots/spot-outbound.ts +47 -68
- package/src/runtime/spots/spot-remote-codec.ts +7 -3
- package/src/runtime/spots/spot-route-packet-dispatch.ts +25 -24
- package/src/runtime/spots/spot-route-replies.ts +6 -4
- package/src/runtime/spots/spot-routed-actor-admission.ts +13 -6
- package/src/runtime/spots/spot-routed-frame-dispatch.ts +6 -77
- package/src/runtime/spots/spot-routed-spot-packet-dispatch.ts +95 -22
- package/src/runtime/spots/spot-runtime-ports.ts +10 -43
- package/src/runtime/spots/spot-serial-executor.ts +10 -2
- package/src/runtime/spots/spot-serial-turn-executor.ts +120 -13
- package/src/runtime/spots/spot-timer.ts +3 -2
- package/src/runtime/streams/actor-session-binding-registry.ts +182 -560
- package/src/runtime/streams/actor-session-binding-runtime-owner.ts +1 -19
- package/src/runtime/streams/bound-actor-relay-sender.ts +89 -43
- package/src/runtime/streams/bound-session-service.ts +23 -37
- package/src/runtime/streams/index.ts +102 -88
- package/src/runtime/streams/managed-stream.ts +29 -10
- package/src/runtime/streams/native-fallback-bound-session.ts +1 -3
- package/src/runtime/streams/protocol.ts +34 -88
- package/src/runtime/streams/session-actor-coordinator.ts +17 -5
- package/src/runtime/streams/session-calls.ts +6 -5
- package/src/runtime/streams/session-context.ts +54 -10
- package/src/runtime/streams/session-requests.ts +2 -1
- package/src/runtime/streams/session-serial-executor.ts +2 -2
- package/src/runtime/streams/stream-binding-runtime-ports.ts +22 -1
- package/src/runtime/streams/stream-frame-factory.ts +2 -3
- package/src/runtime/streams/stream-session-runtime.ts +57 -59
- package/src/runtime/workers/index.ts +13 -12
- package/dist/runtime/actors/deferred-join-accepted-journal.d.ts +0 -47
- package/dist/runtime/actors/deferred-join-accepted-journal.js +0 -992
- package/dist/runtime/actors/post-commit-actor-location.d.ts +0 -19
- package/dist/runtime/actors/post-commit-actor-location.js +0 -72
- package/dist/runtime/spots/spot-native-actor-join-admission.d.ts +0 -26
- package/dist/runtime/spots/spot-native-actor-join-admission.js +0 -106
- package/src/runtime/actors/deferred-join-accepted-journal.ts +0 -1346
- package/src/runtime/actors/post-commit-actor-location.ts +0 -121
- package/src/runtime/spots/spot-native-actor-join-admission.ts +0 -155
|
@@ -5,36 +5,50 @@ exports.decodeQueuedHandoffPacket = decodeQueuedHandoffPacket;
|
|
|
5
5
|
exports.createServiceRelocationId = createServiceRelocationId;
|
|
6
6
|
exports.relocationFailedFailureCode = relocationFailedFailureCode;
|
|
7
7
|
exports.encodeHandoffQueuedMessages = encodeHandoffQueuedMessages;
|
|
8
|
+
const stream_wire_1 = require("@zlink-systems/stream-wire");
|
|
8
9
|
const node_crypto_1 = require("node:crypto");
|
|
9
|
-
const runtime_values_1 = require("../backend/runtime-values");
|
|
10
10
|
const contracts_1 = require("../../contracts");
|
|
11
11
|
const Options_1 = require("../../contracts/Locations/Options");
|
|
12
|
-
const
|
|
13
|
-
const
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
const
|
|
19
|
-
const
|
|
12
|
+
const Stores_1 = require("../../contracts/Locations/Stores");
|
|
13
|
+
const actor_authority_publication_1 = require("../actors/actor-authority-publication");
|
|
14
|
+
const actor_handoff_1 = require("../actors/actor-handoff");
|
|
15
|
+
const actor_remote_wire_1 = require("../actors/actor-remote-wire");
|
|
16
|
+
const actor_runtime_state_1 = require("../actors/actor-runtime-state");
|
|
17
|
+
const runtime_values_1 = require("../backend/runtime-values");
|
|
18
|
+
const diagnostics_1 = require("../diagnostics");
|
|
19
|
+
const runtime_metrics_1 = require("../diagnostics/runtime-metrics");
|
|
20
|
+
const state_lane_1 = require("../execution/state-lane");
|
|
21
|
+
const actor_join_recovery_codec_1 = require("../foundation/actor-join-recovery-codec");
|
|
22
|
+
const operation_identity_1 = require("../foundation/operation-identity");
|
|
23
|
+
const service_authority_payload_codec_1 = require("../foundation/service-authority-payload-codec");
|
|
20
24
|
const service_maintenance_runtime_1 = require("../foundation/service-maintenance-runtime");
|
|
21
|
-
const service_relocation_object_owner_1 = require("../foundation/service-relocation-object-owner");
|
|
22
25
|
const service_relocation_aggregate_committer_1 = require("../foundation/service-relocation-aggregate-committer");
|
|
26
|
+
const service_relocation_coordinator_1 = require("../foundation/service-relocation-coordinator");
|
|
27
|
+
const service_relocation_object_owner_1 = require("../foundation/service-relocation-object-owner");
|
|
28
|
+
const service_relocation_runtime_1 = require("../foundation/service-relocation-runtime");
|
|
29
|
+
const service_runtime_contracts_1 = require("../foundation/service-runtime-contracts");
|
|
30
|
+
const service_stateful_wire_codec_1 = require("../foundation/service-stateful-wire-codec");
|
|
31
|
+
const service_wire_constants_generated_1 = require("../foundation/service-wire-constants.generated");
|
|
32
|
+
const service_wire_m6a_codec_1 = require("../foundation/service-wire-m6a-codec");
|
|
33
|
+
const framework_errors_internal_1 = require("../framework-errors-internal");
|
|
34
|
+
const authority_key_codec_1 = require("../locations/authority-key-codec");
|
|
35
|
+
const location_store_failure_1 = require("../locations/location-store-failure");
|
|
36
|
+
const routing_id_1 = require("../routing-id");
|
|
23
37
|
const spot_provider_1 = require("../spots/spot-provider");
|
|
24
|
-
const actor_runtime_state_1 = require("../actors/actor-runtime-state");
|
|
25
|
-
const actor_handoff_1 = require("../actors/actor-handoff");
|
|
26
38
|
const spot_remote_codec_1 = require("../spots/spot-remote-codec");
|
|
27
|
-
const actor_authority_publication_1 = require("../actors/actor-authority-publication");
|
|
28
|
-
const service_authority_payload_codec_1 = require("../foundation/service-authority-payload-codec");
|
|
29
39
|
const actor_transfer_runtime_1 = require("./actor-transfer-runtime");
|
|
30
|
-
const authority_key_codec_1 = require("../locations/authority-key-codec");
|
|
31
|
-
const service_relocation_control_1 = require("./service-relocation-control");
|
|
32
|
-
const service_stateful_wire_codec_1 = require("../foundation/service-stateful-wire-codec");
|
|
33
|
-
const service_wire_constants_generated_1 = require("../foundation/service-wire-constants.generated");
|
|
34
40
|
const bounded_replay_map_1 = require("./bounded-replay-map");
|
|
41
|
+
const relocation_direct_transfer_1 = require("./relocation-direct-transfer");
|
|
35
42
|
const relocation_integrity_fault_gate_1 = require("./relocation-integrity-fault-gate");
|
|
36
|
-
const
|
|
37
|
-
const
|
|
43
|
+
const relocation_state_adapter_1 = require("./relocation-state-adapter");
|
|
44
|
+
const service_relocation_control_1 = require("./service-relocation-control");
|
|
45
|
+
const RELOCATION_ATTEMPT_TIMEOUT_MS = 250;
|
|
46
|
+
const RELOCATION_AUTHORITY_RETRY_DELAY_MS = 25;
|
|
47
|
+
const RELOCATION_MAX_SEND_ATTEMPTS = 120;
|
|
48
|
+
const RELOCATION_AUTHORITY_POLL_INTERVAL_MS = 10;
|
|
49
|
+
const RELOCATION_RESEND_DELAY_MS = 250;
|
|
50
|
+
const RELOCATION_ABORTABLE_OPERATION_TIMEOUT_MS = 24 * 60 * 60 * 1000;
|
|
51
|
+
const RELOCATION_OPERATION_TIMEOUT_MS = 30_000;
|
|
38
52
|
class ZLinkRelocationStateIncompatibleError extends Error {
|
|
39
53
|
constructor(message) {
|
|
40
54
|
super(message);
|
|
@@ -68,8 +82,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
68
82
|
* Terminal restore failures (assembly/checksum/factory/restore error): the
|
|
69
83
|
* target already sent relocationFailed (command 53) once. An exact-identity
|
|
70
84
|
* Prepare resend replays the stored response rather than retrying the
|
|
71
|
-
* restore
|
|
72
|
-
* key supersedes the stale entry (spec 28 §3, §9).
|
|
85
|
+
* restore. A different exact identity never changes this entry (spec 28 §4.3).
|
|
73
86
|
*/
|
|
74
87
|
targetReadyFailures = new Map();
|
|
75
88
|
/**
|
|
@@ -109,8 +122,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
109
122
|
}
|
|
110
123
|
this.reserveExactRelocationId(input.relocationId);
|
|
111
124
|
let resolveReady;
|
|
112
|
-
const ready = new Promise((resolve) => {
|
|
113
|
-
resolveReady =
|
|
125
|
+
const ready = new Promise((resolve, reject) => {
|
|
126
|
+
resolveReady = (failure) => {
|
|
127
|
+
if (failure === undefined)
|
|
128
|
+
resolve();
|
|
129
|
+
else
|
|
130
|
+
reject(failure.cause);
|
|
131
|
+
};
|
|
114
132
|
});
|
|
115
133
|
this.sourceActorJoinProfiles.set(input.relocationId, {
|
|
116
134
|
actor: input.actor,
|
|
@@ -121,8 +139,8 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
121
139
|
targetNodeRid: input.target.targetNodeRid,
|
|
122
140
|
relocationContentType: this.actorRegistration(input.state.meshName ?? input.meshName, input.state.actorType ?? '')
|
|
123
141
|
.relocation.kind === 'snapshot'
|
|
124
|
-
?
|
|
125
|
-
:
|
|
142
|
+
? actor_join_recovery_codec_1.SNAPSHOT_RELOCATION_CONTENT_TYPE
|
|
143
|
+
: actor_join_recovery_codec_1.RECREATE_RELOCATION_CONTENT_TYPE,
|
|
126
144
|
completionOperationId: input.completionOperationId,
|
|
127
145
|
canonicalRecovery: input.canonicalRecovery,
|
|
128
146
|
ready,
|
|
@@ -141,8 +159,14 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
141
159
|
resolveReady();
|
|
142
160
|
}
|
|
143
161
|
catch (error) {
|
|
144
|
-
|
|
145
|
-
|
|
162
|
+
if (error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) {
|
|
163
|
+
resolveReady(error);
|
|
164
|
+
this.completeActorJoinSourceCleanup(input.state.actorId);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
this.sourceActorJoinProfiles.delete(input.relocationId);
|
|
168
|
+
throw error;
|
|
169
|
+
}
|
|
146
170
|
}
|
|
147
171
|
return {
|
|
148
172
|
actorRef: {
|
|
@@ -157,20 +181,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
157
181
|
async dispose() {
|
|
158
182
|
this.disposed = true;
|
|
159
183
|
const errors = [];
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
await stage.finalize;
|
|
166
|
-
}
|
|
167
|
-
else if (stage.phase === 'ready') {
|
|
168
|
-
await this.abortTargetStage(stage);
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
catch (error) {
|
|
172
|
-
errors.push(error);
|
|
173
|
-
}
|
|
184
|
+
try {
|
|
185
|
+
await this.drainTargetAttempts();
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
errors.push(error);
|
|
174
189
|
}
|
|
175
190
|
this.targetStages.clear();
|
|
176
191
|
this.terminalTargets.clear();
|
|
@@ -181,10 +196,6 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
181
196
|
this.targetReadyFailures.clear();
|
|
182
197
|
this.targetReadyResponses.clear();
|
|
183
198
|
for (const window of this.sourceCutoverWindows.values()) {
|
|
184
|
-
if (window.windowTimer !== undefined)
|
|
185
|
-
clearTimeout(window.windowTimer);
|
|
186
|
-
if (window.retryTimer !== undefined)
|
|
187
|
-
clearTimeout(window.retryTimer);
|
|
188
199
|
if (window.followTimer !== undefined)
|
|
189
200
|
clearTimeout(window.followTimer);
|
|
190
201
|
}
|
|
@@ -217,22 +228,67 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
217
228
|
if (errors.length > 1)
|
|
218
229
|
throw new AggregateError(errors, 'Relocation runtime stop failed.');
|
|
219
230
|
}
|
|
231
|
+
/** Host shutdown waits for the existing attempt terminal before releasing its lease. */
|
|
232
|
+
async drainTargetAttempts() {
|
|
233
|
+
const results = await Promise.allSettled([...this.targetStages.entries()].map(([stagingId, stage]) => {
|
|
234
|
+
if (stage.fallback !== undefined)
|
|
235
|
+
clearTimeout(stage.fallback);
|
|
236
|
+
return stage.lane.finally(() => {
|
|
237
|
+
if (stage.phase === 'failed')
|
|
238
|
+
return stage.finalize;
|
|
239
|
+
if (this.targetStageSealed(stage)) {
|
|
240
|
+
const terminal = this.discardTargetStage(stagingId, stage);
|
|
241
|
+
stage.finalize = terminal;
|
|
242
|
+
return terminal;
|
|
243
|
+
}
|
|
244
|
+
return this.beginTargetFinalize(stage.offer.reservation.prepared.plan.targetDescriptor.meshName, stagingId, stage);
|
|
245
|
+
});
|
|
246
|
+
}));
|
|
247
|
+
const errors = results.flatMap((result) => result.status === 'rejected' ? [result.reason] : []);
|
|
248
|
+
if (errors.length === 1)
|
|
249
|
+
throw errors[0];
|
|
250
|
+
if (errors.length > 1)
|
|
251
|
+
throw new AggregateError(errors, 'Relocation target drain failed.');
|
|
252
|
+
}
|
|
253
|
+
/** Spec 28 §4.4: the target owner alone orders shutdown seal and verified cutover. */
|
|
254
|
+
targetStageSealed(stage) {
|
|
255
|
+
return (!stage.cutoverReceived && (this.disposed || this.options.targetAdmissionSealed?.() === true));
|
|
256
|
+
}
|
|
220
257
|
completeActorJoinSourceCleanup(actorId) {
|
|
221
258
|
for (const [relocationId, profile] of this.sourceActorJoinProfiles) {
|
|
222
259
|
if (profile.state.actorId !== actorId)
|
|
223
260
|
continue;
|
|
224
261
|
this.sourceActorJoinProfiles.delete(relocationId);
|
|
225
262
|
this.terminalActorJoinSourceLeaves.remember(relocationId, actorId);
|
|
226
|
-
void profile
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
263
|
+
void this.finalizeActorJoinSource(profile);
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
async finalizeActorJoinSource(profile) {
|
|
267
|
+
const failures = [];
|
|
268
|
+
try {
|
|
269
|
+
await profile.ready;
|
|
270
|
+
}
|
|
271
|
+
catch (error) {
|
|
272
|
+
failures.push(error);
|
|
273
|
+
}
|
|
274
|
+
try {
|
|
275
|
+
await this.options.actorTransfer.completeRelocationSourceLeave(profile.actor, profile.sourceSpotId);
|
|
276
|
+
}
|
|
277
|
+
catch (error) {
|
|
278
|
+
failures.push(error);
|
|
279
|
+
}
|
|
280
|
+
try {
|
|
281
|
+
await this.options
|
|
282
|
+
.actorManager()
|
|
283
|
+
?.completeRelocationSource(profile.state.actorId, profile.sourceActorRef);
|
|
284
|
+
}
|
|
285
|
+
catch (error) {
|
|
286
|
+
failures.push(error);
|
|
287
|
+
}
|
|
288
|
+
if (failures.length > 0) {
|
|
289
|
+
console.error('[zlink.runtime.relocation.source_cleanup_failed]', failures.length === 1
|
|
290
|
+
? failures[0]
|
|
291
|
+
: new AggregateError(failures, 'Actor source retirement failed.'));
|
|
236
292
|
}
|
|
237
293
|
}
|
|
238
294
|
async relocateMesh(meshName, targetApplicationVersion, signal, stopStartingSignal) {
|
|
@@ -302,19 +358,49 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
302
358
|
publishState: () => undefined,
|
|
303
359
|
forceStop: () => undefined
|
|
304
360
|
});
|
|
361
|
+
const started = [];
|
|
305
362
|
for (const unit of work) {
|
|
306
363
|
maintenance.enqueue({
|
|
307
364
|
id: unit.id,
|
|
308
365
|
ready: () => true,
|
|
309
|
-
relocate:
|
|
366
|
+
relocate: (relocationSignal) => {
|
|
367
|
+
const run = unit.relocate(relocationSignal);
|
|
368
|
+
if ((0, state_lane_1.isStructuralGuardEnabled)()) {
|
|
369
|
+
(0, state_lane_1.trackDiagnosticCompletion)(run, `relocation ${unit.id}`);
|
|
370
|
+
}
|
|
371
|
+
started.push(run);
|
|
372
|
+
return run;
|
|
373
|
+
}
|
|
310
374
|
});
|
|
311
375
|
}
|
|
312
376
|
const operationSignal = signal ?? new AbortController().signal;
|
|
313
|
-
const deadlineMs = signal === undefined
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
377
|
+
const deadlineMs = signal === undefined
|
|
378
|
+
? RELOCATION_OPERATION_TIMEOUT_MS
|
|
379
|
+
: RELOCATION_ABORTABLE_OPERATION_TIMEOUT_MS;
|
|
380
|
+
let failure;
|
|
381
|
+
try {
|
|
382
|
+
const result = await raceAbort(maintenance.start('retire', deadlineMs, stopStartingSignal, operationSignal), operationSignal);
|
|
383
|
+
if (result.state === 'completed')
|
|
384
|
+
return;
|
|
385
|
+
failure = result.terminalError ?? new Error(`Host relocation ended in '${result.state}'.`);
|
|
317
386
|
}
|
|
387
|
+
catch (error) {
|
|
388
|
+
failure = error;
|
|
389
|
+
}
|
|
390
|
+
// The host result follows the settled authority of every started unit
|
|
391
|
+
// (spec 30 §13): a unit completes only after its target commit is
|
|
392
|
+
// confirmed, so a failure beside a completed unit leaves units on both
|
|
393
|
+
// sides.
|
|
394
|
+
const settled = await Promise.allSettled(started);
|
|
395
|
+
const targetSide = settled.filter((value) => value.status === 'fulfilled' || value.reason instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError).length;
|
|
396
|
+
const authorityError = settled.find((value) => value.status === 'rejected' && value.reason instanceof service_relocation_coordinator_1.ServiceRelocationAuthorityError);
|
|
397
|
+
if (authorityError !== undefined)
|
|
398
|
+
throw authorityError.reason;
|
|
399
|
+
// A unit that never started also stays on the source.
|
|
400
|
+
if (targetSide > 0 && targetSide < work.length) {
|
|
401
|
+
throw new service_relocation_coordinator_1.ServiceRelocationAuthorityError('Relocation units settled on both the source and the target.', { cause: failure });
|
|
402
|
+
}
|
|
403
|
+
throw failure;
|
|
318
404
|
}
|
|
319
405
|
async tryHandleControl(meshName, record, signal) {
|
|
320
406
|
if (record.parts.length === 0)
|
|
@@ -379,41 +465,50 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
379
465
|
}
|
|
380
466
|
const operationKey = relocationStagingId(request);
|
|
381
467
|
const fingerprint = stringifyWire(request);
|
|
468
|
+
const readyFailure = this.targetReadyFailures.get(operationKey);
|
|
469
|
+
const readyPending = this.targetReadyResponses.get(operationKey);
|
|
470
|
+
let operation = this.targetPrepareOperations.get(operationKey);
|
|
471
|
+
const acceptedFingerprint = readyFailure?.fingerprint ??
|
|
472
|
+
readyPending?.fingerprint ??
|
|
473
|
+
operation?.fingerprint ??
|
|
474
|
+
this.targetAssemblies.get(operationKey)?.fingerprint ??
|
|
475
|
+
this.targetStages.get(operationKey)?.offer.prepareFingerprint ??
|
|
476
|
+
this.terminalTargets.get(operationKey);
|
|
477
|
+
if (request.sourceNodeRid !== String(record.sourceNodeRid))
|
|
478
|
+
return true;
|
|
479
|
+
if (acceptedFingerprint !== undefined && acceptedFingerprint !== fingerprint) {
|
|
480
|
+
if (!samePrepareIdentity(acceptedFingerprint, request))
|
|
481
|
+
return true;
|
|
482
|
+
this.replyPrepare(record, relocationFailed(request, service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.relocationDataLost));
|
|
483
|
+
return true;
|
|
484
|
+
}
|
|
382
485
|
// A restore that failed outright is terminal: the source observes the
|
|
383
486
|
// same explicit relocationFailed on every identical Prepare resend
|
|
384
|
-
// rather than a silent stall.
|
|
385
|
-
// same RelocationId+attempt key supersedes the stale failure — the
|
|
386
|
-
// newer attempt wins (spec 28 §3).
|
|
387
|
-
const readyFailure = this.targetReadyFailures.get(operationKey);
|
|
487
|
+
// rather than a silent stall.
|
|
388
488
|
if (readyFailure !== undefined) {
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
return true;
|
|
392
|
-
}
|
|
393
|
-
this.targetReadyFailures.delete(operationKey);
|
|
489
|
+
this.replyPrepare(record, readyFailure.response);
|
|
490
|
+
return true;
|
|
394
491
|
}
|
|
395
492
|
// Spec 28: an exact-identity Restore resend against staging that
|
|
396
493
|
// already restored successfully reuses that staging — it re-submits
|
|
397
494
|
// READY rather than restarting the restore.
|
|
398
|
-
const readyPending = this.targetReadyResponses.get(operationKey);
|
|
399
495
|
if (readyPending !== undefined) {
|
|
400
|
-
if (readyPending.fingerprint !== fingerprint) {
|
|
401
|
-
throw new Error(`Relocation '${operationKey}' repeated Prepare with different bytes.`);
|
|
402
|
-
}
|
|
403
496
|
this.submitTargetReady(meshName, operationKey, record, readyPending.response);
|
|
404
497
|
return true;
|
|
405
498
|
}
|
|
406
|
-
let operation = this.targetPrepareOperations.get(operationKey);
|
|
407
|
-
if (operation !== undefined && operation.fingerprint !== fingerprint) {
|
|
408
|
-
throw new Error(`Relocation '${operationKey}' repeated Prepare with different bytes.`);
|
|
409
|
-
}
|
|
410
499
|
if (operation === undefined) {
|
|
411
500
|
// The prepare turn must not block the ordered dispatch lane: the
|
|
412
501
|
// payload chunks that complete this operation arrive as later records
|
|
413
502
|
// on the same connection. Register the assembly synchronously, then
|
|
414
503
|
// finish the restore and the Ready reply asynchronously.
|
|
415
|
-
|
|
416
|
-
|
|
504
|
+
let promise;
|
|
505
|
+
try {
|
|
506
|
+
this.registerTargetAssembly(operationKey, request, String(record.sourceNodeRid));
|
|
507
|
+
promise = this.handlePrepareControl(meshName, operationKey, request, record.sourceNodeRid, signal);
|
|
508
|
+
}
|
|
509
|
+
catch (error) {
|
|
510
|
+
promise = Promise.reject(error);
|
|
511
|
+
}
|
|
417
512
|
operation = { fingerprint, promise };
|
|
418
513
|
this.targetPrepareOperations.set(operationKey, operation);
|
|
419
514
|
void promise
|
|
@@ -441,7 +536,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
441
536
|
// restore or staging): this is terminal. Send relocationFailed
|
|
442
537
|
// (command 53) once and remember it so an identical Prepare resend
|
|
443
538
|
// replays the same response instead of retrying the restore.
|
|
444
|
-
const failed = relocationFailed(request, relocationFailedFailureCode(error
|
|
539
|
+
const failed = relocationFailed(request, relocationFailedFailureCode(error));
|
|
445
540
|
this.targetReadyFailures.set(operationKey, {
|
|
446
541
|
fingerprint: stringifyWire(request),
|
|
447
542
|
response: failed
|
|
@@ -522,7 +617,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
522
617
|
String(sourceNodeRid) !== pending.authenticatedSourceNodeRid ||
|
|
523
618
|
request.senderRole !== 'source' ||
|
|
524
619
|
!sameCoordinator(request.coordinator, pending.prepare.coordinator) ||
|
|
525
|
-
|
|
620
|
+
!sameRelocationObject(request.object, pending.prepare.object)) {
|
|
526
621
|
// Spec 28 §4.3 — a chunk with a different exact identity is never
|
|
527
622
|
// attached to an in-progress assembly; it is discarded.
|
|
528
623
|
console.warn('[zlink.runtime.relocation.stale_state_chunk]', operationKey);
|
|
@@ -541,12 +636,6 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
541
636
|
registerTargetAssembly(operationKey, request, authenticatedSourceNodeRid) {
|
|
542
637
|
const existing = this.targetAssemblies.get(operationKey);
|
|
543
638
|
if (existing !== undefined) {
|
|
544
|
-
if (existing.fingerprint !== stringifyWire(request)) {
|
|
545
|
-
// Same exact identity with a different declared length or checksum is
|
|
546
|
-
// an explicit conflict failure; the existing assembly is neither
|
|
547
|
-
// reused nor overwritten (spec 28 §4.3).
|
|
548
|
-
throw existing.assembly.fail(`Relocation '${operationKey}' repeated Prepare with a conflicting manifest.`);
|
|
549
|
-
}
|
|
550
639
|
return existing;
|
|
551
640
|
}
|
|
552
641
|
const pending = {
|
|
@@ -557,12 +646,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
557
646
|
};
|
|
558
647
|
this.targetAssemblies.set(operationKey, pending);
|
|
559
648
|
if (this.relocationIntegrityFaultGate()?.consumeIdentityConflict() === true) {
|
|
560
|
-
|
|
561
|
-
// conflict branch so existing staging is neither reused nor overwritten.
|
|
562
|
-
return this.registerTargetAssembly(operationKey, {
|
|
563
|
-
...request,
|
|
564
|
-
payloadChecksumCrc32c: request.payloadChecksumCrc32c ^ 1
|
|
565
|
-
}, authenticatedSourceNodeRid);
|
|
649
|
+
throw new ServiceRelocationDataLostError(pending.assembly.fail(`Relocation '${operationKey}' repeated Prepare with a conflicting manifest.`).message);
|
|
566
650
|
}
|
|
567
651
|
return pending;
|
|
568
652
|
}
|
|
@@ -689,7 +773,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
689
773
|
return;
|
|
690
774
|
}
|
|
691
775
|
if (this.pendingSessionRelocations.get(key) === pending) {
|
|
692
|
-
pending.retryTimer = setTimeout(() => void send(),
|
|
776
|
+
pending.retryTimer = setTimeout(() => void send(), RELOCATION_RESEND_DELAY_MS);
|
|
693
777
|
}
|
|
694
778
|
}
|
|
695
779
|
catch (error) {
|
|
@@ -827,6 +911,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
827
911
|
}
|
|
828
912
|
}
|
|
829
913
|
async relocateSpotAggregate(meshName, activation, kind, actorStates, target, targetApplicationVersion, signal) {
|
|
914
|
+
await activation.serial.executeControlLifecycleOperation(() => this.relocateSpotAggregateCore(meshName, activation, kind, actorStates, target, targetApplicationVersion, signal));
|
|
915
|
+
}
|
|
916
|
+
async relocateSpotAggregateCore(meshName, activation, kind, actorStates, target, targetApplicationVersion, signal) {
|
|
830
917
|
const store = this.requireLocationStore();
|
|
831
918
|
const spotKey = (0, authority_key_codec_1.encodeAuthorityKey)(kind, String(activation.spotId));
|
|
832
919
|
const spotAuthority = await requireAuthority(store, spotKey, signal);
|
|
@@ -866,12 +953,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
866
953
|
objectGeneration: spotAuthority.objectGeneration,
|
|
867
954
|
authorityOwnerGeneration: spotAuthority.authorityOwnerGeneration,
|
|
868
955
|
seal: async (captureSignal) => {
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
956
|
+
const { spotCaptureOperation } = await this.afterSpotCloseDecision(meshName, activation.spotId, () => {
|
|
957
|
+
interruptionStartedAt = performance.now();
|
|
958
|
+
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
959
|
+
// Both seals run in the same event-loop turn after the Close decision.
|
|
960
|
+
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
961
|
+
});
|
|
875
962
|
const preparedSessions = Promise.all(actorStates.map(async (state) => ({
|
|
876
963
|
state,
|
|
877
964
|
actor: state.actor,
|
|
@@ -891,34 +978,64 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
891
978
|
},
|
|
892
979
|
captureApplicationState: (captureSignal) => this.captureApplication(spotRegistration.relocation, activation.spot, captureSignal),
|
|
893
980
|
commitSeal: async () => {
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
981
|
+
const failures = [];
|
|
982
|
+
try {
|
|
983
|
+
if (!spotSealCommitted) {
|
|
984
|
+
if (spotCapture === undefined || !(await activation.commitRelocation(spotCapture))) {
|
|
985
|
+
throw new Error(`Spot '${String(activation.spotId)}' relocation seal became stale.`);
|
|
986
|
+
}
|
|
987
|
+
spotSealCommitted = true;
|
|
988
|
+
}
|
|
989
|
+
if (spotMessageFollowSeal !== undefined) {
|
|
990
|
+
await this.commitSpotMessageFollow(meshName, spotMessageFollowSeal, spotKey, activation, target);
|
|
897
991
|
}
|
|
898
|
-
spotSealCommitted = true;
|
|
899
992
|
}
|
|
900
|
-
|
|
901
|
-
|
|
993
|
+
catch (error) {
|
|
994
|
+
failures.push(error);
|
|
902
995
|
}
|
|
903
996
|
for (const session of sessions) {
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
997
|
+
try {
|
|
998
|
+
const committedAuthority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('actor', session.state.actorId));
|
|
999
|
+
const targetActorRef = {
|
|
1000
|
+
actorId: session.state.actorId,
|
|
1001
|
+
objectGeneration: actorAuthorities.get(session.state.actorId).objectGeneration,
|
|
1002
|
+
meshName,
|
|
1003
|
+
nodeRid: target.rid
|
|
1004
|
+
};
|
|
1005
|
+
await session.prepared.commit({
|
|
1006
|
+
routerChannelId: meshName,
|
|
1007
|
+
targetNodeRid: target.rid,
|
|
1008
|
+
spotId: activation.spotId,
|
|
1009
|
+
spotKind: kind === 'user_spot' ? contracts_1.ZLinkSpotKind.User : contracts_1.ZLinkSpotKind.Instance,
|
|
1010
|
+
authorityOwnerGeneration: committedAuthority.authorityOwnerGeneration
|
|
1011
|
+
}, targetActorRef, (0, actor_transfer_runtime_1.committedActorOwnerFence)(session.state.actorId, targetActorRef, committedAuthority));
|
|
1012
|
+
}
|
|
1013
|
+
catch (error) {
|
|
1014
|
+
failures.push(error);
|
|
1015
|
+
}
|
|
1016
|
+
try {
|
|
1017
|
+
activation.commitActorDeparture(session.state.actorId);
|
|
1018
|
+
}
|
|
1019
|
+
catch (error) {
|
|
1020
|
+
failures.push(error);
|
|
1021
|
+
}
|
|
1022
|
+
try {
|
|
1023
|
+
await this.requireActorManager().completeRelocationSource(session.state.actorId);
|
|
1024
|
+
}
|
|
1025
|
+
catch (error) {
|
|
1026
|
+
failures.push(error);
|
|
1027
|
+
}
|
|
920
1028
|
}
|
|
921
|
-
|
|
1029
|
+
try {
|
|
1030
|
+
await this.requireSpotManager().completeRelocationSource(activation);
|
|
1031
|
+
}
|
|
1032
|
+
catch (error) {
|
|
1033
|
+
failures.push(error);
|
|
1034
|
+
}
|
|
1035
|
+
if (failures.length > 0)
|
|
1036
|
+
throw failures.length === 1
|
|
1037
|
+
? failures[0]
|
|
1038
|
+
: new AggregateError(failures, 'Spot source retirement failed.');
|
|
922
1039
|
},
|
|
923
1040
|
abortSeal: async () => {
|
|
924
1041
|
const failures = [];
|
|
@@ -986,7 +1103,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
986
1103
|
}
|
|
987
1104
|
catch (error) {
|
|
988
1105
|
outcome = 'failed';
|
|
989
|
-
if (readinessBoundaryConsumed &&
|
|
1106
|
+
if (readinessBoundaryConsumed &&
|
|
1107
|
+
!(error instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) &&
|
|
1108
|
+
!(error instanceof service_relocation_coordinator_1.ServiceRelocationAuthorityError)) {
|
|
990
1109
|
await activation
|
|
991
1110
|
.completeConsumedRelocationBoundary(contracts_1.ZLinkSpotRelocationReadyOutcome.Continued)
|
|
992
1111
|
.catch(() => undefined);
|
|
@@ -1003,7 +1122,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1003
1122
|
async relocateStandaloneActor(meshName, state, target, targetApplicationVersion, signal, targetMembership, aggregateIdOverride, retainSourceForActorJoin = false,
|
|
1004
1123
|
/** Target's advertised relocation state chunk cap (Actor Join only; spec 15 §4.2). */
|
|
1005
1124
|
advertisedReceiveChunkLimitBytes) {
|
|
1006
|
-
|
|
1125
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1126
|
+
(0, diagnostics_1.relocationDebug)('standalone_actor.begin', { actorId: state.actorId });
|
|
1127
|
+
}
|
|
1007
1128
|
const authority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId), signal);
|
|
1008
1129
|
target ??= await this.selectTarget(meshName, String(this.requireMeshNode(meshName).status().routingId), targetApplicationVersion, [
|
|
1009
1130
|
{
|
|
@@ -1036,14 +1157,20 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1036
1157
|
};
|
|
1037
1158
|
let outcome = 'completed';
|
|
1038
1159
|
try {
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1160
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1161
|
+
(0, diagnostics_1.relocationDebug)('standalone_actor.wait_budget', {
|
|
1162
|
+
actorId: state.actorId,
|
|
1163
|
+
targetRid: String(target.rid)
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1043
1166
|
await this.waitForRelocationBudgetHeadroom(String(target.rid), signal);
|
|
1044
|
-
|
|
1167
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1168
|
+
(0, diagnostics_1.relocationDebug)('standalone_actor.capture_begin', { actorId: state.actorId });
|
|
1169
|
+
}
|
|
1045
1170
|
const captured = await new service_relocation_object_owner_1.ServiceRelocationObjectCaptureOwner().captureStandaloneActor(aggregateId, 1n, measuredUnit, membership, signal);
|
|
1046
|
-
|
|
1171
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1172
|
+
(0, diagnostics_1.relocationDebug)('standalone_actor.capture_complete', { actorId: state.actorId });
|
|
1173
|
+
}
|
|
1047
1174
|
await this.runCoordinator(meshName, target, authority, captured, new Map([[measuredUnit.authorityKey, authority]]), signal, sessions, advertisedReceiveChunkLimitBytes);
|
|
1048
1175
|
}
|
|
1049
1176
|
catch (error) {
|
|
@@ -1058,6 +1185,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1058
1185
|
}
|
|
1059
1186
|
}
|
|
1060
1187
|
async relocatePerActorSpotShell(meshName, activation, target, targetApplicationVersion, signal) {
|
|
1188
|
+
await activation.serial.executeControlLifecycleOperation(() => this.relocatePerActorSpotShellCore(meshName, activation, target, targetApplicationVersion, signal));
|
|
1189
|
+
}
|
|
1190
|
+
async relocatePerActorSpotShellCore(meshName, activation, target, targetApplicationVersion, signal) {
|
|
1061
1191
|
const spotKey = (0, authority_key_codec_1.encodeAuthorityKey)('user_spot', String(activation.spotId));
|
|
1062
1192
|
const spotAuthority = await requireAuthority(this.requireLocationStore(), spotKey, signal);
|
|
1063
1193
|
const spotRegistration = this.spotRegistration(meshName, 'user_spot', spotAuthority.allocation.stableType);
|
|
@@ -1080,9 +1210,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1080
1210
|
objectGeneration: spotAuthority.objectGeneration,
|
|
1081
1211
|
authorityOwnerGeneration: spotAuthority.authorityOwnerGeneration,
|
|
1082
1212
|
seal: async (captureSignal) => {
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1213
|
+
const { spotCaptureOperation } = await this.afterSpotCloseDecision(meshName, activation.spotId, () => {
|
|
1214
|
+
interruptionStartedAt = performance.now();
|
|
1215
|
+
spotMessageFollowSeal = this.sealSpotMessageFollow(meshName, spotAuthority, activation);
|
|
1216
|
+
return { spotCaptureOperation: activation.captureRelocation(captureSignal) };
|
|
1217
|
+
});
|
|
1218
|
+
spotCapture = await spotCaptureOperation;
|
|
1086
1219
|
return {
|
|
1087
1220
|
boundSessionState: Buffer.alloc(0),
|
|
1088
1221
|
queuedMessages: [],
|
|
@@ -1169,29 +1302,47 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1169
1302
|
target === undefined ||
|
|
1170
1303
|
meshName === undefined)
|
|
1171
1304
|
return;
|
|
1172
|
-
const
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1305
|
+
const failures = [];
|
|
1306
|
+
try {
|
|
1307
|
+
const membershipSpotId = targetMembership?.spotId ?? (target.entrySpotId ?? String(target.rid));
|
|
1308
|
+
const committedAuthority = await requireAuthority(this.requireLocationStore(), (0, authority_key_codec_1.encodeAuthorityKey)('actor', state.actorId));
|
|
1309
|
+
const targetActorRef = {
|
|
1310
|
+
actorId: state.actorId,
|
|
1311
|
+
objectGeneration: authority.objectGeneration,
|
|
1312
|
+
meshName,
|
|
1313
|
+
nodeRid: target.rid
|
|
1314
|
+
};
|
|
1315
|
+
await ownSession.prepared.commit({
|
|
1316
|
+
routerChannelId: meshName,
|
|
1317
|
+
targetNodeRid: target.rid,
|
|
1318
|
+
spotId: membershipSpotId,
|
|
1319
|
+
spotKind: targetMembership?.spotKind ?? contracts_1.ZLinkSpotKind.Entry,
|
|
1320
|
+
authorityOwnerGeneration: committedAuthority.authorityOwnerGeneration
|
|
1321
|
+
}, targetActorRef, (0, actor_transfer_runtime_1.committedActorOwnerFence)(state.actorId, targetActorRef, committedAuthority));
|
|
1322
|
+
}
|
|
1323
|
+
catch (error) {
|
|
1324
|
+
failures.push(error);
|
|
1325
|
+
}
|
|
1187
1326
|
if (!retainSourceForActorJoin) {
|
|
1188
|
-
|
|
1327
|
+
try {
|
|
1328
|
+
await this.requireActorManager().completeRelocationSource(state.actorId);
|
|
1329
|
+
}
|
|
1330
|
+
catch (error) {
|
|
1331
|
+
failures.push(error);
|
|
1332
|
+
}
|
|
1189
1333
|
}
|
|
1334
|
+
if (failures.length > 0)
|
|
1335
|
+
throw failures.length === 1
|
|
1336
|
+
? failures[0]
|
|
1337
|
+
: new AggregateError(failures, 'Actor source retirement failed.');
|
|
1190
1338
|
},
|
|
1191
1339
|
abortSeal: async () => {
|
|
1192
1340
|
if (standalone && ownSession !== undefined)
|
|
1193
1341
|
await ownSession.prepared.rollback();
|
|
1194
|
-
}
|
|
1342
|
+
},
|
|
1343
|
+
discardSeal: (reason) => (standalone
|
|
1344
|
+
? ownSession
|
|
1345
|
+
: sessions.find((value) => value.state === state))?.prepared.discard(reason)
|
|
1195
1346
|
};
|
|
1196
1347
|
}
|
|
1197
1348
|
reserveRelocationId() {
|
|
@@ -1217,12 +1368,16 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1217
1368
|
async runCoordinator(meshName, target, primary, captured, _authorities, signal, sessions = [],
|
|
1218
1369
|
/** Target's advertised relocation state chunk cap (Actor Join only; spec 15 §4.2). */
|
|
1219
1370
|
advertisedReceiveChunkLimitBytes) {
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1371
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1372
|
+
(0, diagnostics_1.relocationDebug)('coordinator.begin', {
|
|
1373
|
+
aggregateId: captured.envelope.aggregateId,
|
|
1374
|
+
targetRid: String(target.rid)
|
|
1375
|
+
});
|
|
1376
|
+
}
|
|
1224
1377
|
const localStatus = this.requireMeshNode(meshName).status();
|
|
1225
|
-
const controlDeadlineAtMs = signal === undefined
|
|
1378
|
+
const controlDeadlineAtMs = signal === undefined
|
|
1379
|
+
? performance.now() + RELOCATION_OPERATION_TIMEOUT_MS
|
|
1380
|
+
: Number.MAX_SAFE_INTEGER;
|
|
1226
1381
|
const coordinator = {
|
|
1227
1382
|
ownerId: primary.ownerId,
|
|
1228
1383
|
leaseGeneration: primary.ownerLeaseGeneration,
|
|
@@ -1264,11 +1419,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1264
1419
|
// configured limit unchanged (spec 15 §4.2).
|
|
1265
1420
|
const chunkLimitBytes = (0, relocation_direct_transfer_1.effectiveActorJoinChunkLimitBytes)(limits.relocationPayloadChunkLimitBytes, advertisedReceiveChunkLimitBytes);
|
|
1266
1421
|
const plan = (0, relocation_direct_transfer_1.planRelocationChunks)(encoded, chunkLimitBytes);
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1422
|
+
// The Restore absolute deadline is this unit's relocation deadline: the
|
|
1423
|
+
// caller's signal, or the control deadline without one. From it the
|
|
1424
|
+
// source settles the authority with its Preserve fence (Location runtime
|
|
1425
|
+
// §10). The target keeps no deadline of its own.
|
|
1426
|
+
const restoreDeadlineReached = () => signal?.aborted === true || performance.now() >= controlDeadlineAtMs;
|
|
1270
1427
|
let readyReceived = false;
|
|
1271
|
-
let
|
|
1428
|
+
let settlement;
|
|
1272
1429
|
try {
|
|
1273
1430
|
const prepare = {
|
|
1274
1431
|
kind: 'prepare',
|
|
@@ -1291,18 +1448,26 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1291
1448
|
// the target assembled, verified, and restored the payload.
|
|
1292
1449
|
const readyOperation = this.sendControl(meshName, target.rid, prepare, signal, controlDeadlineAtMs, encodePrepareSideband(captured.envelope));
|
|
1293
1450
|
try {
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1451
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1452
|
+
(0, diagnostics_1.relocationDebug)('coordinator.state_transfer_begin', {
|
|
1453
|
+
aggregateId: captured.envelope.aggregateId
|
|
1454
|
+
});
|
|
1455
|
+
}
|
|
1297
1456
|
await this.sendRelocationStateChunks(meshName, target, prepare, encoded, plan, signal);
|
|
1298
1457
|
}
|
|
1299
1458
|
catch (chunkError) {
|
|
1300
1459
|
readyOperation.catch(() => undefined);
|
|
1301
1460
|
throw chunkError;
|
|
1302
1461
|
}
|
|
1303
|
-
|
|
1462
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1463
|
+
(0, diagnostics_1.relocationDebug)('coordinator.await_ready', { aggregateId: captured.envelope.aggregateId });
|
|
1464
|
+
}
|
|
1304
1465
|
const ready = await readyOperation;
|
|
1305
|
-
|
|
1466
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1467
|
+
(0, diagnostics_1.relocationDebug)('coordinator.ready_received', {
|
|
1468
|
+
aggregateId: captured.envelope.aggregateId
|
|
1469
|
+
});
|
|
1470
|
+
}
|
|
1306
1471
|
validateControlResponse(prepare, ready);
|
|
1307
1472
|
// Ready arms the target's cutover-wait fallback. From this point
|
|
1308
1473
|
// forward the source can no longer reopen dispatch without risking two
|
|
@@ -1343,9 +1508,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1343
1508
|
boundaryRecordCount: BigInt(boundaryBatch.length),
|
|
1344
1509
|
boundaryChecksumCrc32c: (0, service_relocation_runtime_1.crc32c)(Buffer.concat(boundaryBatch.map((value) => value.canonicalBytes)))
|
|
1345
1510
|
});
|
|
1346
|
-
// The
|
|
1347
|
-
//
|
|
1348
|
-
//
|
|
1511
|
+
// The cutover submit result is not the authority result. The memory
|
|
1512
|
+
// copy of the boundary batch and cutover stays until the authority
|
|
1513
|
+
// settlement below (spec 28 §4.4).
|
|
1349
1514
|
let submitFailure;
|
|
1350
1515
|
try {
|
|
1351
1516
|
for (const record of boundaryBatch) {
|
|
@@ -1357,38 +1522,44 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1357
1522
|
submitFailure = error;
|
|
1358
1523
|
console.warn('[zlink.runtime.relocation.cutover_submit_failed]', relayAuthorityId, error);
|
|
1359
1524
|
}
|
|
1360
|
-
this.beginSourceCutoverWindow(meshName, target.rid, relayAuthorityId, [...boundaryBatch.map((value) => value.frame), cutoverFrame], submitFailure !== undefined, limits);
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1525
|
+
const window = this.beginSourceCutoverWindow(meshName, target.rid, relayAuthorityId, [...boundaryBatch.map((value) => value.frame), cutoverFrame], submitFailure !== undefined, limits);
|
|
1526
|
+
try {
|
|
1527
|
+
settlement = await this.settleSourceAuthority(captured.envelope, primary, targetFence, restoreDeadlineReached, window);
|
|
1528
|
+
}
|
|
1529
|
+
finally {
|
|
1530
|
+
this.closeSourceCutoverWindow(relayAuthorityId, window);
|
|
1531
|
+
}
|
|
1532
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
1533
|
+
(0, diagnostics_1.relocationDebug)('coordinator.authority_settled', {
|
|
1534
|
+
aggregateId: captured.envelope.aggregateId,
|
|
1535
|
+
settlement
|
|
1536
|
+
});
|
|
1537
|
+
}
|
|
1538
|
+
if (settlement === 'source') {
|
|
1539
|
+
// Preserve won: the captured queue, timers and held records return to
|
|
1540
|
+
// the source queue in their accepted order, and dispatch reopens.
|
|
1541
|
+
await captured.abortSource();
|
|
1542
|
+
throw new Error(`Relocation '${relayAuthorityId}' ended by the source Preserve fence; the source kept the authority.`);
|
|
1543
|
+
}
|
|
1544
|
+
if (settlement === 'lost') {
|
|
1545
|
+
// Expired owner: dispatch and Store changes stop for good and every
|
|
1546
|
+
// pending request without a terminal gets one Unavailable.
|
|
1547
|
+
const unavailable = (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected, `Relocation '${relayAuthorityId}' source owner lease ended before its authority settled.`, true);
|
|
1548
|
+
await captured.discardSource(unavailable);
|
|
1549
|
+
throw new service_relocation_coordinator_1.ServiceRelocationAuthorityError(unavailable.message, { cause: unavailable });
|
|
1550
|
+
}
|
|
1365
1551
|
for (const session of sessions)
|
|
1366
1552
|
session.prepared.setReplayResults([]);
|
|
1367
1553
|
await captured.commitSource();
|
|
1368
|
-
sourceCommitted = true;
|
|
1369
1554
|
await this.options.reconcileStatefulAuthorityRoutes?.(signal);
|
|
1370
1555
|
}
|
|
1371
1556
|
catch (error) {
|
|
1372
|
-
if (
|
|
1557
|
+
if (settlement !== 'target')
|
|
1373
1558
|
throw error;
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
session.prepared.setReplayResults([]);
|
|
1379
|
-
await captured.commitSource();
|
|
1380
|
-
sourceCommitted = true;
|
|
1381
|
-
}
|
|
1382
|
-
await this.options.reconcileStatefulAuthorityRoutes?.();
|
|
1383
|
-
const authority = await requireAuthority(this.requireLocationStore(), { value: primaryKey(captured.envelope) }, undefined);
|
|
1384
|
-
throw new service_relocation_coordinator_1.ServiceRelocationPostCommitError(authority, { id: `${captured.envelope.aggregateId}:${captured.envelope.aggregateGeneration}` }, error);
|
|
1385
|
-
}
|
|
1386
|
-
catch (convergenceError) {
|
|
1387
|
-
if (convergenceError instanceof service_relocation_coordinator_1.ServiceRelocationPostCommitError) {
|
|
1388
|
-
throw convergenceError;
|
|
1389
|
-
}
|
|
1390
|
-
throw new AggregateError([error, convergenceError], 'Relocation became irreversible after Ready and target convergence failed.');
|
|
1391
|
-
}
|
|
1559
|
+
// Target commit is confirmed; later source cleanup failures cannot move
|
|
1560
|
+
// the authority back (spec 28 §10).
|
|
1561
|
+
const authority = await requireAuthority(this.requireLocationStore(), { value: primaryKey(captured.envelope) }, undefined);
|
|
1562
|
+
throw new service_relocation_coordinator_1.ServiceRelocationPostCommitError(authority, { id: `${captured.envelope.aggregateId}:${captured.envelope.aggregateGeneration}` }, error);
|
|
1392
1563
|
}
|
|
1393
1564
|
finally {
|
|
1394
1565
|
if (!readyReceived) {
|
|
@@ -1461,12 +1632,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1461
1632
|
return budget;
|
|
1462
1633
|
}
|
|
1463
1634
|
/**
|
|
1464
|
-
* Starts the source
|
|
1465
|
-
*
|
|
1466
|
-
*
|
|
1467
|
-
*
|
|
1468
|
-
*
|
|
1469
|
-
* route removal point — closes the unit for SafeToShutdown.
|
|
1635
|
+
* Starts the source cutover window after the cutover submit terminal. The
|
|
1636
|
+
* boundary batch and cutover copies live in framework memory (not charged
|
|
1637
|
+
* to the in-flight budget) until the authority settlement closes the
|
|
1638
|
+
* window. S4 — the Message Follow route removal point — closes the unit for
|
|
1639
|
+
* SafeToShutdown.
|
|
1470
1640
|
*/
|
|
1471
1641
|
beginSourceCutoverWindow(meshName, targetNodeRid, relocationKey, frames, initialSubmitFailed, limits) {
|
|
1472
1642
|
const submittedAt = performance.now();
|
|
@@ -1474,54 +1644,145 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1474
1644
|
meshName,
|
|
1475
1645
|
targetNodeRid,
|
|
1476
1646
|
frames,
|
|
1647
|
+
resendPending: initialSubmitFailed,
|
|
1477
1648
|
cutoverSubmittedAtMs: submittedAt,
|
|
1478
1649
|
windowClosed: false,
|
|
1479
1650
|
followExpired: false
|
|
1480
1651
|
};
|
|
1481
1652
|
this.sourceCutoverWindows.set(relocationKey, window);
|
|
1482
|
-
const settle = () => {
|
|
1483
|
-
if (window.windowClosed && window.followExpired) {
|
|
1484
|
-
this.sourceCutoverWindows.delete(relocationKey);
|
|
1485
|
-
}
|
|
1486
|
-
};
|
|
1487
|
-
window.windowTimer = setTimeout(() => {
|
|
1488
|
-
window.windowClosed = true;
|
|
1489
|
-
window.frames = undefined;
|
|
1490
|
-
if (window.retryTimer !== undefined)
|
|
1491
|
-
clearTimeout(window.retryTimer);
|
|
1492
|
-
settle();
|
|
1493
|
-
}, limits.relocationCutoverWaitTimeoutMs);
|
|
1494
|
-
window.windowTimer.unref();
|
|
1495
1653
|
window.followTimer = setTimeout(() => {
|
|
1496
1654
|
window.followExpired = true;
|
|
1497
1655
|
// S1→S4 route convergence: the span the source must keep its Message
|
|
1498
1656
|
// Follow route (spec 30 §7.1).
|
|
1499
|
-
this.options.metrics?.duration(
|
|
1500
|
-
|
|
1657
|
+
this.options.metrics?.duration(runtime_metrics_1.METRIC_NAMES.RelocationRouteConvergence, Math.max(0, performance.now() - submittedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND, { mesh_name: meshName });
|
|
1658
|
+
this.releaseSourceCutoverWindow(relocationKey, window);
|
|
1501
1659
|
}, limits.messageFollowDurationMs);
|
|
1502
1660
|
window.followTimer.unref();
|
|
1503
|
-
|
|
1504
|
-
|
|
1661
|
+
return window;
|
|
1662
|
+
}
|
|
1663
|
+
/** The authority settlement ended: the retained batch copy is released. */
|
|
1664
|
+
closeSourceCutoverWindow(relocationKey, window) {
|
|
1665
|
+
window.windowClosed = true;
|
|
1666
|
+
window.frames = undefined;
|
|
1667
|
+
this.releaseSourceCutoverWindow(relocationKey, window);
|
|
1505
1668
|
}
|
|
1506
|
-
|
|
1507
|
-
if (window.windowClosed
|
|
1669
|
+
releaseSourceCutoverWindow(relocationKey, window) {
|
|
1670
|
+
if (window.windowClosed &&
|
|
1671
|
+
window.followExpired &&
|
|
1672
|
+
this.sourceCutoverWindows.get(relocationKey) === window) {
|
|
1673
|
+
this.sourceCutoverWindows.delete(relocationKey);
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
/**
|
|
1677
|
+
* Sends the whole boundary batch and cutover again after a failed submit
|
|
1678
|
+
* or an observed loss of the target connection (spec 28 §4.4). The target
|
|
1679
|
+
* replaces its partial span with the whole batch.
|
|
1680
|
+
*/
|
|
1681
|
+
async resendSourceCutover(window) {
|
|
1682
|
+
const frames = window.frames;
|
|
1683
|
+
if (frames === undefined)
|
|
1508
1684
|
return;
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1685
|
+
const peers = this.requireMeshNode(window.meshName).peers();
|
|
1686
|
+
const peer = peers.find((value) => value.routingId !== null && String(value.routingId) === String(window.targetNodeRid));
|
|
1687
|
+
if (peer !== undefined && peer.state !== service_runtime_contracts_1.MeshPeerRuntimeState.Serving) {
|
|
1688
|
+
window.resendPending = true;
|
|
1689
|
+
return;
|
|
1690
|
+
}
|
|
1691
|
+
if (!window.resendPending)
|
|
1692
|
+
return;
|
|
1693
|
+
try {
|
|
1694
|
+
for (const frame of frames) {
|
|
1695
|
+
await this.submitControlFrame(window.meshName, window.targetNodeRid, frame, 'cutover retransmission');
|
|
1696
|
+
}
|
|
1697
|
+
window.resendPending = false;
|
|
1698
|
+
}
|
|
1699
|
+
catch {
|
|
1700
|
+
// The next settlement round retries the whole batch.
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
/**
|
|
1704
|
+
* The one source authority settlement after Ready (Location runtime
|
|
1705
|
+
* §6.1·§10, spec 28 §4.4). It reads the Store until the target commit or
|
|
1706
|
+
* the Restore deadline. From the Restore deadline it runs the source
|
|
1707
|
+
* Preserve fence: a CAS on the StoreVersion expected by target NewOwner
|
|
1708
|
+
* excludes a late target commit. `target`: the
|
|
1709
|
+
* target committed. `source`: Preserve fenced the target out while the
|
|
1710
|
+
* source owner lease was valid. `lost`: the source owner lease ended first
|
|
1711
|
+
* or another authority owns the object. An uncertain Store or lease result
|
|
1712
|
+
* keeps the work retained and repeats the same decision.
|
|
1713
|
+
*/
|
|
1714
|
+
async settleSourceAuthority(envelope, primary, target, restoreDeadlineReached, window) {
|
|
1715
|
+
const owner = { ownerId: primary.ownerId, leaseGeneration: primary.ownerLeaseGeneration };
|
|
1716
|
+
const store = this.requireLocationStore();
|
|
1717
|
+
for (;;) {
|
|
1718
|
+
if (this.disposed)
|
|
1719
|
+
throw new Error('Relocation runtime stopped.');
|
|
1720
|
+
await this.resendSourceCutover(window);
|
|
1721
|
+
const observed = await this.readSourceSettlement(envelope, primary, target).catch((error) => {
|
|
1722
|
+
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error))
|
|
1723
|
+
throw error;
|
|
1724
|
+
return undefined;
|
|
1725
|
+
});
|
|
1726
|
+
if (observed?.kind === 'target')
|
|
1727
|
+
return 'target';
|
|
1728
|
+
if (observed?.kind === 'other')
|
|
1729
|
+
return 'lost';
|
|
1730
|
+
if (restoreDeadlineReached() && observed?.kind === 'source') {
|
|
1731
|
+
// An unreadable lease is an uncertain result, not an expiry.
|
|
1732
|
+
if (await this.exactSourceLeaseExpired(owner).catch((error) => {
|
|
1733
|
+
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error))
|
|
1734
|
+
throw error;
|
|
1735
|
+
return false;
|
|
1736
|
+
}))
|
|
1737
|
+
return 'lost';
|
|
1738
|
+
const publication = this.codec.read(observed.current.payload);
|
|
1739
|
+
const preserved = await store
|
|
1740
|
+
.compareExchangeAuthority({ value: primaryKey(envelope) }, observed.current.storeVersion, {
|
|
1741
|
+
kind: 'put',
|
|
1742
|
+
generationTransition: 'preserve',
|
|
1743
|
+
payload: publication === undefined
|
|
1744
|
+
? observed.current.payload
|
|
1745
|
+
: this.codec.clear(observed.current.payload, publication.reference)
|
|
1746
|
+
})
|
|
1747
|
+
.catch((error) => {
|
|
1748
|
+
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error))
|
|
1749
|
+
throw error;
|
|
1750
|
+
return undefined;
|
|
1751
|
+
});
|
|
1752
|
+
if (preserved?.kind === 'stored')
|
|
1753
|
+
return 'source';
|
|
1754
|
+
}
|
|
1755
|
+
// The caller awaits this settlement, so the retry timer keeps the loop alive.
|
|
1756
|
+
await new Promise((resolve) => setTimeout(resolve, RELOCATION_AUTHORITY_POLL_INTERVAL_MS));
|
|
1757
|
+
}
|
|
1758
|
+
}
|
|
1759
|
+
async readSourceSettlement(envelope, primary, target) {
|
|
1760
|
+
const primaryAuthorityKey = primaryKey(envelope);
|
|
1761
|
+
let committed = 0;
|
|
1762
|
+
let primaryCurrent;
|
|
1763
|
+
for (const participant of envelope.participants) {
|
|
1764
|
+
const current = await this.requireLocationStore().readAuthority({
|
|
1765
|
+
value: participant.key
|
|
1766
|
+
});
|
|
1767
|
+
if (current.kind !== 'snapshot')
|
|
1768
|
+
return { kind: 'other' };
|
|
1769
|
+
if (current.objectGeneration === participant.objectGeneration &&
|
|
1770
|
+
current.ownerId === target.ownerId &&
|
|
1771
|
+
current.ownerLeaseGeneration === target.ownerLeaseGeneration &&
|
|
1772
|
+
current.authorityOwnerGeneration > participant.authorityOwnerGeneration &&
|
|
1773
|
+
String(current.allocation.descriptor.rid) === target.nodeRid &&
|
|
1774
|
+
current.allocation.descriptorLifecycleGeneration === target.nodeGeneration) {
|
|
1775
|
+
committed += 1;
|
|
1776
|
+
}
|
|
1777
|
+
if (participant.key === primaryAuthorityKey)
|
|
1778
|
+
primaryCurrent = current;
|
|
1779
|
+
}
|
|
1780
|
+
if (committed === envelope.participants.length)
|
|
1781
|
+
return { kind: 'target' };
|
|
1782
|
+
if (primaryCurrent !== undefined && isSourceOwner(primaryCurrent, primary)) {
|
|
1783
|
+
return { kind: 'source', current: primaryCurrent };
|
|
1784
|
+
}
|
|
1785
|
+
return { kind: 'other' };
|
|
1525
1786
|
}
|
|
1526
1787
|
/**
|
|
1527
1788
|
* True when every relocation this source started reached its Message
|
|
@@ -1552,9 +1813,6 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1552
1813
|
}
|
|
1553
1814
|
/** Restores the hidden object and its saved-work prefix before Ready. */
|
|
1554
1815
|
async handlePrepareControl(meshName, stagingId, request, sourceNodeRid, signal) {
|
|
1555
|
-
if (sourceNodeRid === null || String(sourceNodeRid) !== request.sourceNodeRid) {
|
|
1556
|
-
throw new Error('Relocation prepare source node fence does not match the authenticated peer.');
|
|
1557
|
-
}
|
|
1558
1816
|
const targetStatus = this.requireMeshNode(meshName).status();
|
|
1559
1817
|
const targetOwner = this.options.currentOwner();
|
|
1560
1818
|
const targetDescriptor = this.options.localDescriptor?.(meshName);
|
|
@@ -1570,22 +1828,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1570
1828
|
const fingerprint = stringifyWire(request);
|
|
1571
1829
|
const existing = this.targetStages.get(stagingId);
|
|
1572
1830
|
if (existing !== undefined) {
|
|
1573
|
-
if (existing.offer.authenticatedSourceNodeRid !== String(sourceNodeRid) ||
|
|
1574
|
-
existing.offer.prepareFingerprint !== fingerprint) {
|
|
1575
|
-
throw new Error('Relocation prepare retry source node changed.');
|
|
1576
|
-
}
|
|
1577
1831
|
return relocationReady(request);
|
|
1578
1832
|
}
|
|
1579
1833
|
const terminal = this.terminalTargets.get(stagingId);
|
|
1580
1834
|
if (terminal !== undefined) {
|
|
1581
|
-
if (terminal !== fingerprint) {
|
|
1582
|
-
throw new Error(`Relocation '${stagingId}' repeated Prepare with different bytes.`);
|
|
1583
|
-
}
|
|
1584
1835
|
this.terminalTargets.touch(stagingId);
|
|
1585
1836
|
return relocationReady(request);
|
|
1586
1837
|
}
|
|
1587
1838
|
const pendingAssembly = this.targetAssemblies.get(stagingId);
|
|
1588
|
-
if (pendingAssembly === undefined
|
|
1839
|
+
if (pendingAssembly === undefined) {
|
|
1589
1840
|
throw new Error(`Relocation '${stagingId}' has no matching payload assembly.`);
|
|
1590
1841
|
}
|
|
1591
1842
|
// The directly transferred payload replaces the shared durable root: the
|
|
@@ -1611,6 +1862,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1611
1862
|
const envelope = (0, service_relocation_runtime_1.decodeServiceRelocationEnvelope)(payload, request.targetAttemptGeneration);
|
|
1612
1863
|
const inventoryEnvelope = attachPrepareBoundSessions(await this.resolveTargetParticipantInventory(request, envelope, signal), request.nodeInternalBoundSessions);
|
|
1613
1864
|
validatePrepareEnvelope(request, inventoryEnvelope);
|
|
1865
|
+
const releaseActivation = (await this.options.activationAdmission?.acquire(meshName, signal)) ?? (() => undefined);
|
|
1614
1866
|
let materialized;
|
|
1615
1867
|
let reservation;
|
|
1616
1868
|
try {
|
|
@@ -1620,13 +1872,12 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1620
1872
|
// rejects unrelated authority CAS until commit/abort.
|
|
1621
1873
|
materialized = await this.materializeTargetEnvelope(meshName, inventoryEnvelope, signal);
|
|
1622
1874
|
await materialized.owner.restoreSavedWork(materialized.staging, signal);
|
|
1623
|
-
reservation = await this.reserveTargetForPrepare(meshName, request, inventoryEnvelope,
|
|
1875
|
+
reservation = await this.reserveTargetForPrepare(meshName, request, inventoryEnvelope, signal);
|
|
1624
1876
|
const offer = {
|
|
1625
1877
|
prepare: request,
|
|
1626
1878
|
prepareFingerprint: fingerprint,
|
|
1627
1879
|
authenticatedSourceNodeRid: String(sourceNodeRid),
|
|
1628
1880
|
envelope: inventoryEnvelope,
|
|
1629
|
-
restoreDeadlineAtMs: performance.now() + RELOCATION_OPERATION_RETENTION_MS,
|
|
1630
1881
|
reservation
|
|
1631
1882
|
};
|
|
1632
1883
|
const stage = {
|
|
@@ -1635,11 +1886,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1635
1886
|
phase: 'ready',
|
|
1636
1887
|
lane: Promise.resolve(),
|
|
1637
1888
|
cutoverReceived: false,
|
|
1638
|
-
boundaryRelay: []
|
|
1889
|
+
boundaryRelay: [],
|
|
1890
|
+
releaseActivation
|
|
1639
1891
|
};
|
|
1640
1892
|
this.targetStages.set(stagingId, stage);
|
|
1641
1893
|
}
|
|
1642
1894
|
catch (error) {
|
|
1895
|
+
releaseActivation();
|
|
1643
1896
|
if (reservation !== undefined) {
|
|
1644
1897
|
await this.abortTargetReservation(reservation, signal).catch(() => undefined);
|
|
1645
1898
|
}
|
|
@@ -1650,12 +1903,25 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1650
1903
|
}
|
|
1651
1904
|
return relocationReady(request);
|
|
1652
1905
|
}
|
|
1906
|
+
/**
|
|
1907
|
+
* The cutover wait of a Ready stage (spec 28 §4.4) is a Warning only: it
|
|
1908
|
+
* records `cutover_timeout` and never submits the target CAS or opens
|
|
1909
|
+
* dispatch. From then on the target settlement reads the Location Store so
|
|
1910
|
+
* a source Preserve fence or a lost target lease ends the staging; only a
|
|
1911
|
+
* verified cutover lets that settlement submit the CAS.
|
|
1912
|
+
*/
|
|
1653
1913
|
armTargetCutoverFallback(meshName, stagingId) {
|
|
1654
1914
|
const stage = this.targetStages.get(stagingId);
|
|
1655
1915
|
if (stage === undefined || stage.fallback !== undefined || stage.finalize !== undefined)
|
|
1656
1916
|
return;
|
|
1657
1917
|
stage.fallback = setTimeout(() => {
|
|
1658
|
-
|
|
1918
|
+
if (stage.finalize !== undefined)
|
|
1919
|
+
return;
|
|
1920
|
+
console.warn('[zlink.runtime.relocation.cutover_timeout]', stagingId);
|
|
1921
|
+
this.options.metrics?.count(runtime_metrics_1.METRIC_NAMES.RelocationCutoverTimeout, 1, {
|
|
1922
|
+
mesh_name: meshName
|
|
1923
|
+
});
|
|
1924
|
+
void this.beginTargetFinalize(meshName, stagingId, stage).catch((error) => console.error('[zlink.runtime.relocation.location_update_failed]', error));
|
|
1659
1925
|
}, this.relocationLimits().relocationCutoverWaitTimeoutMs);
|
|
1660
1926
|
stage.fallback.unref();
|
|
1661
1927
|
}
|
|
@@ -1666,7 +1932,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1666
1932
|
const stagingId = relocationStagingId(request);
|
|
1667
1933
|
const stage = this.targetStages.get(stagingId);
|
|
1668
1934
|
if (stage === undefined) {
|
|
1669
|
-
if (this.terminalTargets.get(stagingId) !== undefined) {
|
|
1935
|
+
if (this.disposed || this.terminalTargets.get(stagingId) !== undefined) {
|
|
1670
1936
|
if (request.kind === 'cutover') {
|
|
1671
1937
|
console.warn('[zlink.runtime.relocation.late_cutover]', stagingId);
|
|
1672
1938
|
}
|
|
@@ -1675,20 +1941,33 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1675
1941
|
throw new Error(`Relocation staging '${stagingId}' is missing.`);
|
|
1676
1942
|
}
|
|
1677
1943
|
validateTargetOneWayControl(stage, request, sourceNodeRid);
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1944
|
+
stage.lane = stage.lane.then(() => {
|
|
1945
|
+
// A verified cutover closes the boundary batch. A later record or cutover
|
|
1946
|
+
// is a duplicate retransmission and never changes the staged batch.
|
|
1947
|
+
if (stage.cutoverReceived || (stage.phase !== 'ready' && stage.phase !== 'finalizing')) {
|
|
1948
|
+
if (request.kind === 'cutover') {
|
|
1949
|
+
console.warn('[zlink.runtime.relocation.late_cutover]', stagingId);
|
|
1950
|
+
}
|
|
1951
|
+
return;
|
|
1952
|
+
}
|
|
1953
|
+
if (request.kind === 'data') {
|
|
1954
|
+
return this.stageBoundaryRelay(stage, request);
|
|
1955
|
+
}
|
|
1956
|
+
if (this.targetStageSealed(stage))
|
|
1957
|
+
return;
|
|
1958
|
+
this.reconcileBoundaryRelay(stage, request, stagingId);
|
|
1959
|
+
stage.cutoverReceived = true;
|
|
1960
|
+
});
|
|
1961
|
+
await stage.lane;
|
|
1962
|
+
if (request.kind === 'data')
|
|
1681
1963
|
return;
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1964
|
+
// A settlement started by the cutover wait Warning is still reading the
|
|
1965
|
+
// Store; it submits the CAS on its next round now that cutover verified.
|
|
1966
|
+
if (stage.finalize !== undefined)
|
|
1685
1967
|
return;
|
|
1686
|
-
}
|
|
1687
|
-
this.reconcileBoundaryRelay(stage, request, stagingId);
|
|
1688
|
-
stage.cutoverReceived = true;
|
|
1689
1968
|
if (stage.fallback !== undefined)
|
|
1690
1969
|
clearTimeout(stage.fallback);
|
|
1691
|
-
await this.beginTargetFinalize(meshName, stagingId, stage,
|
|
1970
|
+
await this.beginTargetFinalize(meshName, stagingId, stage, signal);
|
|
1692
1971
|
}
|
|
1693
1972
|
/** Validates one boundary relay record and buffers it until cutover applies it. */
|
|
1694
1973
|
async stageBoundaryRelay(stage, request) {
|
|
@@ -1707,14 +1986,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1707
1986
|
}
|
|
1708
1987
|
const expectedCount = Number(request.boundaryRecordCount);
|
|
1709
1988
|
const staged = stage.boundaryRelay;
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
console.warn('[zlink.runtime.relocation.boundary_batch_replaced]', stagingId);
|
|
1989
|
+
// The batch is the last `boundaryRecordCount` records: the source sends
|
|
1990
|
+
// the whole batch right before this cutover on the ordered connection,
|
|
1991
|
+
// so a retransmission replaces an earlier partial span as a whole.
|
|
1992
|
+
if (staged.length >= expectedCount) {
|
|
1993
|
+
const batch = staged.slice(staged.length - expectedCount);
|
|
1994
|
+
if ((0, service_relocation_runtime_1.crc32c)(Buffer.concat(batch.map((value) => value.frozenRecord.canonicalBytes))) ===
|
|
1995
|
+
request.boundaryChecksumCrc32c) {
|
|
1718
1996
|
staged.splice(0, staged.length - expectedCount);
|
|
1719
1997
|
return;
|
|
1720
1998
|
}
|
|
@@ -1724,18 +2002,14 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1724
2002
|
throw new Error(`Relocation cutover boundary confirmation mismatch for '${stagingId}' ` +
|
|
1725
2003
|
`(staged=${staged.length} declared=${expectedCount}).`);
|
|
1726
2004
|
}
|
|
1727
|
-
beginTargetFinalize(meshName, stagingId, stage,
|
|
2005
|
+
beginTargetFinalize(meshName, stagingId, stage, signal) {
|
|
1728
2006
|
if (stage.finalize !== undefined)
|
|
1729
2007
|
return stage.finalize;
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
mesh_name: meshName
|
|
1734
|
-
});
|
|
1735
|
-
}
|
|
2008
|
+
// Records staged before this point finish first. The settlement is not
|
|
2009
|
+
// put on the lane, so a cutover that arrives while it waits for the
|
|
2010
|
+
// authority result is still reconciled.
|
|
1736
2011
|
const finalize = stage.lane.then(() => this.finalizeTargetStage(meshName, stagingId, stage, signal));
|
|
1737
2012
|
stage.finalize = finalize;
|
|
1738
|
-
stage.lane = finalize;
|
|
1739
2013
|
return finalize;
|
|
1740
2014
|
}
|
|
1741
2015
|
async finalizeTargetStage(meshName, stagingId, stage, signal) {
|
|
@@ -1746,14 +2020,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1746
2020
|
let authority;
|
|
1747
2021
|
let resumeStartedAt;
|
|
1748
2022
|
try {
|
|
1749
|
-
// The boundary relay span is applied to the temporary queue before the
|
|
1750
|
-
// owner CAS so the merged order is saved work, pre-boundary relay, then
|
|
1751
|
-
// post-boundary temporary records (spec 28 §4.6).
|
|
1752
|
-
for (const relay of stage.boundaryRelay.splice(0)) {
|
|
1753
|
-
await this.applyRelocationData(stage, relay, signal);
|
|
1754
|
-
}
|
|
1755
2023
|
authority = await this.commitTargetReservation(stage, stage.offer.reservation, signal);
|
|
2024
|
+
if (authority === undefined) {
|
|
2025
|
+
// The seal owner may have ended staging while a Store read was pending.
|
|
2026
|
+
if (stage.phase === 'failed')
|
|
2027
|
+
return;
|
|
2028
|
+
return this.discardTargetStage(stagingId, stage, signal);
|
|
2029
|
+
}
|
|
1756
2030
|
authorityCommitted = true;
|
|
2031
|
+
stage.releaseActivation();
|
|
1757
2032
|
resumeStartedAt = performance.now();
|
|
1758
2033
|
stage.phase = 'committed';
|
|
1759
2034
|
await stage.owner.normalize(stage.staging, authority, signal);
|
|
@@ -1768,13 +2043,10 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1768
2043
|
}
|
|
1769
2044
|
}
|
|
1770
2045
|
await this.finalizeActorJoinProfiles(meshName, stage, signal);
|
|
1771
|
-
//
|
|
1772
|
-
//
|
|
1773
|
-
//
|
|
1774
|
-
//
|
|
1775
|
-
// roll back already committed target ownership or strand its staged
|
|
1776
|
-
// object: the next reservation recognizes the retained journal root and
|
|
1777
|
-
// continues its existing cleanup/retry path.
|
|
2046
|
+
// Clear the final relocation publication before admission becomes
|
|
2047
|
+
// externally visible so a later Join cannot reserve against the
|
|
2048
|
+
// previous root. A bounded CAS conflict must not roll back already
|
|
2049
|
+
// committed target ownership or strand its staged object.
|
|
1778
2050
|
try {
|
|
1779
2051
|
await this.clearTargetRelocationPublication(stage, authority, signal);
|
|
1780
2052
|
}
|
|
@@ -1786,7 +2058,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1786
2058
|
}
|
|
1787
2059
|
await stage.owner.openAdmission(stage.staging, signal);
|
|
1788
2060
|
// S2→S3: owner confirmation to application dispatch opening.
|
|
1789
|
-
this.options.metrics?.duration(
|
|
2061
|
+
this.options.metrics?.duration(runtime_metrics_1.METRIC_NAMES.RelocationTargetResume, Math.max(0, performance.now() - resumeStartedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND, { mesh_name: meshName });
|
|
1790
2062
|
await this.publishSessionRoutes(stage.staging);
|
|
1791
2063
|
await this.relayTerminalReplies(meshName, stage, this.targetReplyRelayCoordinator(meshName, stage, authority), signal);
|
|
1792
2064
|
stage.phase = 'open';
|
|
@@ -1795,14 +2067,16 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1795
2067
|
this.targetAssemblies.delete(stagingId);
|
|
1796
2068
|
}
|
|
1797
2069
|
catch (error) {
|
|
1798
|
-
stage.phase = 'failed';
|
|
1799
2070
|
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
1800
2071
|
this.targetStages.delete(stagingId);
|
|
1801
2072
|
this.targetAssemblies.delete(stagingId);
|
|
1802
2073
|
if (!authorityCommitted) {
|
|
1803
|
-
|
|
2074
|
+
// abortTargetStage may already have assigned failed before throwing.
|
|
2075
|
+
if (stage.phase !== 'failed')
|
|
2076
|
+
await this.abortTargetStage(stage);
|
|
1804
2077
|
throw error;
|
|
1805
2078
|
}
|
|
2079
|
+
stage.phase = 'failed';
|
|
1806
2080
|
if (authority !== undefined) {
|
|
1807
2081
|
throw new service_relocation_coordinator_1.ServiceRelocationPostCommitError(authority, stage.staging, error);
|
|
1808
2082
|
}
|
|
@@ -1886,16 +2160,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1886
2160
|
}
|
|
1887
2161
|
this.sourceActorJoinProfiles.delete(input.transferId);
|
|
1888
2162
|
this.terminalActorJoinSourceLeaves.remember(input.transferId, input.actorId);
|
|
1889
|
-
void profile
|
|
1890
|
-
.then(async () => {
|
|
1891
|
-
await this.options.actorTransfer.completeRelocationSourceLeave(profile.actor, profile.sourceSpotId);
|
|
1892
|
-
await this.options
|
|
1893
|
-
.actorManager()
|
|
1894
|
-
?.completeRelocationSource(input.actorId, profile.sourceActorRef);
|
|
1895
|
-
})
|
|
1896
|
-
.catch((error) => {
|
|
1897
|
-
console.error('[zlink.runtime.relocation.source_leave_failed]', error);
|
|
1898
|
-
});
|
|
2163
|
+
void this.finalizeActorJoinSource(profile);
|
|
1899
2164
|
return true;
|
|
1900
2165
|
}
|
|
1901
2166
|
async sendControlOneWay(meshName, targetNodeRid, request) {
|
|
@@ -1907,58 +2172,23 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
1907
2172
|
throw new Error(`Relocation ${kind} was not accepted by RouteMesh.`);
|
|
1908
2173
|
}
|
|
1909
2174
|
}
|
|
1910
|
-
async waitForTargetAuthorities(envelope, target, deadlineAtMs, signal) {
|
|
1911
|
-
for (;;) {
|
|
1912
|
-
signal?.throwIfAborted();
|
|
1913
|
-
let exact = true;
|
|
1914
|
-
for (const participant of envelope.participants) {
|
|
1915
|
-
const current = await this.requireLocationStore().readAuthority({ value: participant.key }, signal);
|
|
1916
|
-
if (current.kind !== 'snapshot' ||
|
|
1917
|
-
current.objectGeneration !== participant.objectGeneration ||
|
|
1918
|
-
current.ownerId !== target.ownerId ||
|
|
1919
|
-
current.ownerLeaseGeneration !== target.ownerLeaseGeneration ||
|
|
1920
|
-
current.authorityOwnerGeneration <= participant.authorityOwnerGeneration ||
|
|
1921
|
-
String(current.allocation.descriptor.rid) !== target.nodeRid ||
|
|
1922
|
-
current.allocation.descriptorLifecycleGeneration !== target.nodeGeneration) {
|
|
1923
|
-
exact = false;
|
|
1924
|
-
break;
|
|
1925
|
-
}
|
|
1926
|
-
}
|
|
1927
|
-
if (exact)
|
|
1928
|
-
return;
|
|
1929
|
-
if (performance.now() >= deadlineAtMs) {
|
|
1930
|
-
throw new Error('Relocation target owner was not confirmed before its original deadline.');
|
|
1931
|
-
}
|
|
1932
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
1933
|
-
}
|
|
1934
|
-
}
|
|
1935
2175
|
async clearTargetRelocationPublication(stage, primary, signal) {
|
|
1936
2176
|
await this.releaseParticipantAuthorities(stage.staging.envelope, stage.staging.primaryAuthorityKey.value, signal);
|
|
1937
|
-
// The
|
|
1938
|
-
//
|
|
1939
|
-
//
|
|
1940
|
-
//
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
generationTransition: 'preserve',
|
|
1952
|
-
payload: this.codec.clear(current.payload, publication.reference)
|
|
1953
|
-
}, signal);
|
|
1954
|
-
if (result.kind === 'stored')
|
|
1955
|
-
return;
|
|
1956
|
-
const read = await this.requireLocationStore().readAuthority(stage.staging.primaryAuthorityKey, signal);
|
|
1957
|
-
if (read.kind !== 'snapshot')
|
|
1958
|
-
break;
|
|
1959
|
-
current = read;
|
|
2177
|
+
// The target owner is the only writer of the committed primary
|
|
2178
|
+
// authority between its commit and this clear, so one CAS against the
|
|
2179
|
+
// committed snapshot decides. A conflict means another owner changed the
|
|
2180
|
+
// row; the caller records it and leaves the committed ownership intact.
|
|
2181
|
+
const publication = this.codec.read(primary.payload);
|
|
2182
|
+
if (publication === undefined)
|
|
2183
|
+
return;
|
|
2184
|
+
const result = await this.requireLocationStore().compareExchangeAuthority(stage.staging.primaryAuthorityKey, primary.storeVersion, {
|
|
2185
|
+
kind: 'put',
|
|
2186
|
+
generationTransition: 'preserve',
|
|
2187
|
+
payload: this.codec.clear(primary.payload, publication.reference)
|
|
2188
|
+
}, signal);
|
|
2189
|
+
if (result.kind !== 'stored') {
|
|
2190
|
+
throw new Error('Relocation target authority normalization CAS failed.');
|
|
1960
2191
|
}
|
|
1961
|
-
throw new Error('Relocation target authority normalization CAS failed.');
|
|
1962
2192
|
}
|
|
1963
2193
|
async handleReplyRelay(meshName, relay, sourceNodeRid, signal) {
|
|
1964
2194
|
if (sourceNodeRid === null) {
|
|
@@ -2039,13 +2269,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2039
2269
|
pending.resolve('sourceLeaseExpired');
|
|
2040
2270
|
return;
|
|
2041
2271
|
}
|
|
2042
|
-
if (++attempts >
|
|
2272
|
+
if (++attempts > RELOCATION_MAX_SEND_ATTEMPTS) {
|
|
2043
2273
|
pending.reject(new Error(`Relocation reply relay ACK '${key}' timed out.`));
|
|
2044
2274
|
return;
|
|
2045
2275
|
}
|
|
2046
2276
|
await this.sendInfrastructureControl(meshName, ackTargetNodeRid, (0, service_stateful_wire_codec_1.encodeMaintenanceReplyRelay)(request));
|
|
2047
2277
|
if (this.pendingReplyRelays.get(key) === pending) {
|
|
2048
|
-
pending.timer = setTimeout(() => void send(),
|
|
2278
|
+
pending.timer = setTimeout(() => void send(), RELOCATION_RESEND_DELAY_MS);
|
|
2049
2279
|
}
|
|
2050
2280
|
}
|
|
2051
2281
|
catch (error) {
|
|
@@ -2140,13 +2370,15 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2140
2370
|
lease.leaseExpiresAt.getTime() <= lease.storeNow.getTime());
|
|
2141
2371
|
}
|
|
2142
2372
|
async publishSessionRoutes(staging) {
|
|
2143
|
-
for (const hidden of staging.hidden
|
|
2144
|
-
if (hidden.actor
|
|
2145
|
-
|
|
2146
|
-
|
|
2373
|
+
for (const [key, hidden] of staging.hidden) {
|
|
2374
|
+
if (hidden.actor === undefined)
|
|
2375
|
+
continue;
|
|
2376
|
+
// The Session seal travels with the relocation envelope, not the Actor.
|
|
2377
|
+
const carried = staging.envelope.participants.find((value) => value.key === key);
|
|
2378
|
+
await this.options.actorTransfer.publishRoutedActorOwnership(hidden.actor, carried === undefined ? undefined : decodeActorSession(carried.boundSessionState));
|
|
2147
2379
|
}
|
|
2148
2380
|
}
|
|
2149
|
-
async sendControl(meshName, targetNodeRid, request, signal, deadlineAtMs = performance.now() +
|
|
2381
|
+
async sendControl(meshName, targetNodeRid, request, signal, deadlineAtMs = performance.now() + RELOCATION_OPERATION_TIMEOUT_MS, sideband = []) {
|
|
2150
2382
|
const key = controlAckKey(request);
|
|
2151
2383
|
const frames = sideband.length === 0
|
|
2152
2384
|
? [(0, service_relocation_control_1.encodeServiceRelocationControlRequest)(request)]
|
|
@@ -2158,7 +2390,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2158
2390
|
if (remainingMs <= 0) {
|
|
2159
2391
|
throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.DeadlineExceeded, `Relocation control ACK '${key}' timed out.`, true, lastRequestError);
|
|
2160
2392
|
}
|
|
2161
|
-
const attemptTimeoutMs = Math.max(1, Math.min(
|
|
2393
|
+
const attemptTimeoutMs = Math.max(1, Math.min(RELOCATION_ATTEMPT_TIMEOUT_MS, Math.ceil(remainingMs)));
|
|
2162
2394
|
const startedAtMs = performance.now();
|
|
2163
2395
|
try {
|
|
2164
2396
|
const replyFrames = await this.requestInfrastructureControl(meshName, targetNodeRid, frames, attemptTimeoutMs);
|
|
@@ -2178,11 +2410,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2178
2410
|
throw error;
|
|
2179
2411
|
}
|
|
2180
2412
|
lastRequestError = error;
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2413
|
+
if ((0, diagnostics_1.isRelocationDebugEnabled)()) {
|
|
2414
|
+
(0, diagnostics_1.relocationDebug)('control.request_retry', {
|
|
2415
|
+
key,
|
|
2416
|
+
kind: request.kind,
|
|
2417
|
+
targetRid: String(targetNodeRid)
|
|
2418
|
+
});
|
|
2419
|
+
}
|
|
2186
2420
|
const retryDelayMs = Math.min(Math.max(0, attemptTimeoutMs - (performance.now() - startedAtMs)), Math.max(0, deadlineAtMs - performance.now()));
|
|
2187
2421
|
if (retryDelayMs > 0) {
|
|
2188
2422
|
await waitForRelocationRetry(retryDelayMs, signal);
|
|
@@ -2207,7 +2441,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2207
2441
|
peers.some((peer) => peer.routingId !== null &&
|
|
2208
2442
|
String(peer.routingId) === String(descriptor.rid) &&
|
|
2209
2443
|
peer.lifecycleGeneration === descriptor.lifecycleGeneration &&
|
|
2210
|
-
peer.state ===
|
|
2444
|
+
peer.state === service_runtime_contracts_1.MeshPeerRuntimeState.Serving) &&
|
|
2211
2445
|
relocationTargetSupports(descriptor, requirements))
|
|
2212
2446
|
.sort((left, right) => String(left.rid).localeCompare(String(right.rid)));
|
|
2213
2447
|
const target = selectWeightedRelocationTarget(candidates);
|
|
@@ -2234,39 +2468,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2234
2468
|
}
|
|
2235
2469
|
return authorities;
|
|
2236
2470
|
}
|
|
2237
|
-
async reserveTargetForPrepare(meshName, prepare, envelope,
|
|
2471
|
+
async reserveTargetForPrepare(meshName, prepare, envelope, signal) {
|
|
2238
2472
|
const authorities = await this.readTargetParticipantAuthorities(envelope, signal);
|
|
2239
2473
|
const publication = relocationPublication(prepare, envelope);
|
|
2240
|
-
// Carries the exact reference + CRC of a deferred-Join publication the
|
|
2241
|
-
// target reservation must keep connected through the CAS (spec 28 —
|
|
2242
|
-
// clearing/replacing it here would strand the journal's own
|
|
2243
|
-
// prepared->committed transition with nothing left for it to find).
|
|
2244
|
-
const retainedDeferredJoinRoots = new Map();
|
|
2245
|
-
for (const participant of envelope.participants) {
|
|
2246
|
-
const staged = targetPort.deferredJoinRoot(participant.key);
|
|
2247
|
-
if (staged !== undefined) {
|
|
2248
|
-
retainedDeferredJoinRoots.set(participant.key, {
|
|
2249
|
-
reference: staged.reference.value,
|
|
2250
|
-
checksumCrc32c: staged.checksumCrc32c
|
|
2251
|
-
});
|
|
2252
|
-
continue;
|
|
2253
|
-
}
|
|
2254
|
-
const current = this.codec.read(authorities.get(participant.key).payload);
|
|
2255
|
-
if (participant.objectKind === 'actor' &&
|
|
2256
|
-
current?.canonical === true &&
|
|
2257
|
-
(await this.options.actorTransfer.isDeferredJoinAcceptedRootPublication(current.reference, current.checksumCrc32c, {
|
|
2258
|
-
authorityKey: participant.key,
|
|
2259
|
-
objectKind: 'actor',
|
|
2260
|
-
objectGeneration: participant.objectGeneration,
|
|
2261
|
-
aggregateId: current.aggregateId,
|
|
2262
|
-
aggregateGeneration: current.aggregateGeneration
|
|
2263
|
-
}, signal))) {
|
|
2264
|
-
retainedDeferredJoinRoots.set(participant.key, {
|
|
2265
|
-
reference: current.reference,
|
|
2266
|
-
checksumCrc32c: current.checksumCrc32c
|
|
2267
|
-
});
|
|
2268
|
-
}
|
|
2269
|
-
}
|
|
2270
2474
|
const participants = envelope.participants.map((participant) => {
|
|
2271
2475
|
const expected = authorities.get(participant.key);
|
|
2272
2476
|
const membership = actorMembershipTarget(envelope, participant.key);
|
|
@@ -2278,19 +2482,11 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2278
2482
|
entrySpotId === undefined) {
|
|
2279
2483
|
throw new Error(`Relocation target RouteMesh '${meshName}' has no Entry Spot.`);
|
|
2280
2484
|
}
|
|
2281
|
-
const retainedDeferredJoinRoot = retainedDeferredJoinRoots.get(participant.key);
|
|
2282
|
-
const participantPublication = retainedDeferredJoinRoot === undefined
|
|
2283
|
-
? publication
|
|
2284
|
-
: {
|
|
2285
|
-
...publication,
|
|
2286
|
-
reference: retainedDeferredJoinRoot.reference,
|
|
2287
|
-
checksumCrc32c: retainedDeferredJoinRoot.checksumCrc32c
|
|
2288
|
-
};
|
|
2289
2485
|
return {
|
|
2290
2486
|
key: { value: participant.key },
|
|
2291
2487
|
expected,
|
|
2292
2488
|
ownerTransition: 'newOwner',
|
|
2293
|
-
authorityPayload: this.authorityPayloadForPublication(expected.payload,
|
|
2489
|
+
authorityPayload: this.authorityPayloadForPublication(expected.payload, publication, {
|
|
2294
2490
|
owner: {
|
|
2295
2491
|
ownerId: prepare.target.ownerId,
|
|
2296
2492
|
leaseGeneration: prepare.target.ownerLeaseGeneration
|
|
@@ -2313,7 +2509,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2313
2509
|
...membership,
|
|
2314
2510
|
actorSpotKind: contracts_1.ZLinkSpotKind.User
|
|
2315
2511
|
})
|
|
2316
|
-
}
|
|
2512
|
+
}),
|
|
2317
2513
|
membershipMutation: encodeMembershipMutation(envelope.memberships, participant.key)
|
|
2318
2514
|
};
|
|
2319
2515
|
});
|
|
@@ -2332,19 +2528,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2332
2528
|
prepared: await new service_relocation_aggregate_committer_1.ServiceRelocationAggregateCommitter(this.requireLocationStore()).prepare(plan, signal)
|
|
2333
2529
|
};
|
|
2334
2530
|
}
|
|
2335
|
-
authorityPayloadForPublication(payload, publication, target
|
|
2531
|
+
authorityPayloadForPublication(payload, publication, target) {
|
|
2336
2532
|
const existing = this.codec.read(payload);
|
|
2337
2533
|
const canonicalIdentity = (0, service_relocation_runtime_1.serviceRelocationAuthoritySlotIdentity)(payload);
|
|
2338
|
-
const replacesCanonicalJournal = replacesDeferredJoinRoot && existing?.canonical === true;
|
|
2339
2534
|
if (canonicalIdentity !== undefined &&
|
|
2340
2535
|
(existing === undefined ||
|
|
2341
|
-
(existing.canonical === true &&
|
|
2342
|
-
|
|
2343
|
-
const published = existing === undefined
|
|
2344
|
-
? this.codec.publish(payload, publication)
|
|
2345
|
-
: replacesCanonicalJournal && existing.aggregateId !== publication.aggregateId
|
|
2346
|
-
? this.codec.publish(this.codec.clear(payload, existing.reference), publication)
|
|
2347
|
-
: payload;
|
|
2536
|
+
(existing.canonical === true && existing.aggregateId === publication.aggregateId))) {
|
|
2537
|
+
const published = existing === undefined ? this.codec.publish(payload, publication) : payload;
|
|
2348
2538
|
if (target === undefined)
|
|
2349
2539
|
return published;
|
|
2350
2540
|
return (0, service_relocation_runtime_1.projectServiceRelocationAuthorityTargetReady)((0, service_relocation_runtime_1.replaceServiceRelocationAuthorityApplicationPayload)(published, rewriteAuthorityApplicationRoute(published, target)), {
|
|
@@ -2394,6 +2584,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2394
2584
|
async abortTargetStage(stage, signal) {
|
|
2395
2585
|
if (stage.fallback !== undefined)
|
|
2396
2586
|
clearTimeout(stage.fallback);
|
|
2587
|
+
stage.releaseActivation();
|
|
2397
2588
|
const failures = [];
|
|
2398
2589
|
try {
|
|
2399
2590
|
await this.abortTargetReservation(stage.offer.reservation, signal);
|
|
@@ -2414,24 +2605,64 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2414
2605
|
throw new AggregateError(failures, 'Relocation target rollback was incomplete.');
|
|
2415
2606
|
}
|
|
2416
2607
|
}
|
|
2608
|
+
async discardTargetStage(stagingId, stage, signal) {
|
|
2609
|
+
stage.phase = 'failed';
|
|
2610
|
+
console.warn('[zlink.runtime.relocation.target_staging_discarded]', stagingId);
|
|
2611
|
+
try {
|
|
2612
|
+
await this.abortTargetStage(stage, signal);
|
|
2613
|
+
}
|
|
2614
|
+
finally {
|
|
2615
|
+
this.terminalTargets.remember(stagingId, stage.offer.prepareFingerprint);
|
|
2616
|
+
this.targetStages.delete(stagingId);
|
|
2617
|
+
this.targetAssemblies.delete(stagingId);
|
|
2618
|
+
}
|
|
2619
|
+
}
|
|
2417
2620
|
async commitTargetReservation(stage, reservation, signal) {
|
|
2418
2621
|
return await this.commitTargetAggregate(stage, reservation.prepared, signal);
|
|
2419
2622
|
}
|
|
2623
|
+
/**
|
|
2624
|
+
* The one target authority settlement (Location runtime §10, spec 28
|
|
2625
|
+
* §4.4-§4.5). Only a verified cutover lets the target submit its NewOwner
|
|
2626
|
+
* CAS; a transient or indeterminate result resubmits the same RelocationId
|
|
2627
|
+
* and expected StoreVersion while the target owner lease is valid (no
|
|
2628
|
+
* separate timeout, no fixed count). Before a verified cutover the target
|
|
2629
|
+
* only reads. It returns the committed authority, or undefined when a
|
|
2630
|
+
* definitive Conflict excludes this commit after reconciliation, a source
|
|
2631
|
+
* Preserve fence moved the source row, or the target lease ended; the
|
|
2632
|
+
* caller then discards the staging.
|
|
2633
|
+
*/
|
|
2420
2634
|
async commitTargetAggregate(stage, prepared, signal) {
|
|
2421
|
-
|
|
2635
|
+
const target = stage.offer.prepare.target;
|
|
2422
2636
|
for (;;) {
|
|
2637
|
+
if (stage.phase === 'failed')
|
|
2638
|
+
return undefined;
|
|
2423
2639
|
signal?.throwIfAborted();
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2640
|
+
if (this.targetStageSealed(stage))
|
|
2641
|
+
return undefined;
|
|
2642
|
+
let conflict = false;
|
|
2643
|
+
if (stage.cutoverReceived) {
|
|
2644
|
+
// The boundary relay span is applied to the temporary queue before
|
|
2645
|
+
// the owner CAS so the merged order is saved work, pre-boundary relay,
|
|
2646
|
+
// then post-boundary temporary records (spec 28 §4.6).
|
|
2647
|
+
for (const relay of stage.boundaryRelay.splice(0)) {
|
|
2648
|
+
await this.applyRelocationData(stage, relay, signal);
|
|
2649
|
+
}
|
|
2650
|
+
// A thrown provider call has an unknown result; the read below decides.
|
|
2651
|
+
const result = await this.requireLocationStore()
|
|
2652
|
+
.commitAggregate(prepared.fence, signal)
|
|
2653
|
+
.catch((error) => {
|
|
2654
|
+
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error, signal))
|
|
2655
|
+
throw error;
|
|
2656
|
+
return undefined;
|
|
2657
|
+
});
|
|
2658
|
+
if (result?.kind === 'generationExhausted') {
|
|
2659
|
+
throw new Error('location_update_failed: relocation aggregate commit exhausted.');
|
|
2660
|
+
}
|
|
2661
|
+
conflict = result?.kind === 'stale';
|
|
2433
2662
|
}
|
|
2434
2663
|
const observed = await this.readAggregateForCommitRetry(prepared, signal);
|
|
2664
|
+
if (stage.phase === 'failed')
|
|
2665
|
+
return undefined;
|
|
2435
2666
|
if (observed.kind === 'committed') {
|
|
2436
2667
|
const primary = observed.authorities.get(stage.staging.primaryAuthorityKey.value);
|
|
2437
2668
|
if (primary === undefined) {
|
|
@@ -2439,12 +2670,17 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2439
2670
|
}
|
|
2440
2671
|
return primary;
|
|
2441
2672
|
}
|
|
2442
|
-
if (observed.kind === 'stale')
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2673
|
+
if (observed.kind === 'stale' || (conflict && observed.kind === 'source'))
|
|
2674
|
+
return undefined;
|
|
2675
|
+
// An unreadable lease is an uncertain result, not an expiry.
|
|
2676
|
+
if (await this.exactSourceLeaseExpired({ ownerId: target.ownerId, leaseGeneration: target.ownerLeaseGeneration }, signal).catch((error) => {
|
|
2677
|
+
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error, signal))
|
|
2678
|
+
throw error;
|
|
2679
|
+
return false;
|
|
2680
|
+
})) {
|
|
2681
|
+
return undefined;
|
|
2446
2682
|
}
|
|
2447
|
-
await
|
|
2683
|
+
await waitForRelocationRetry(RELOCATION_AUTHORITY_RETRY_DELAY_MS, signal);
|
|
2448
2684
|
}
|
|
2449
2685
|
}
|
|
2450
2686
|
async readAggregateForCommitRetry(prepared, signal) {
|
|
@@ -2473,7 +2709,9 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2473
2709
|
}
|
|
2474
2710
|
}
|
|
2475
2711
|
}
|
|
2476
|
-
catch {
|
|
2712
|
+
catch (error) {
|
|
2713
|
+
if (!(0, location_store_failure_1.isIndeterminateLocationStoreFailure)(error, signal))
|
|
2714
|
+
throw error;
|
|
2477
2715
|
return { kind: 'unknown' };
|
|
2478
2716
|
}
|
|
2479
2717
|
if (committed === prepared.plan.participants.length) {
|
|
@@ -2505,7 +2743,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2505
2743
|
if (request.object.kind === 'userSpot' || request.object.kind === 'instanceSpot') {
|
|
2506
2744
|
let cursor;
|
|
2507
2745
|
for (;;) {
|
|
2508
|
-
const page = await store.listAuthorities('', cursor,
|
|
2746
|
+
const page = await store.listAuthorities('', cursor, Stores_1.ZLINK_PROVIDER_MAX_PAGE_SIZE, signal);
|
|
2509
2747
|
if (page.kind === 'scanExpired') {
|
|
2510
2748
|
throw new Error('Location Store authority scan expired during relocation.');
|
|
2511
2749
|
}
|
|
@@ -2612,7 +2850,7 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2612
2850
|
const adapter = await this.createRelocationAdapter(policy.adapterType);
|
|
2613
2851
|
const captured = await (0, relocation_state_adapter_1.captureRelocationAdapterState)(adapter, value, signal ?? new AbortController().signal);
|
|
2614
2852
|
if (captured.byteLength > RELOCATION_PARTICIPANT_STATE_LIMIT_BYTES) {
|
|
2615
|
-
throw new ZLinkRelocationStateIncompatibleError(
|
|
2853
|
+
throw new ZLinkRelocationStateIncompatibleError(`Relocation application state exceeds the ${RELOCATION_PARTICIPANT_STATE_LIMIT_BYTES / (1024 * 1024)} MiB participant limit.`);
|
|
2616
2854
|
}
|
|
2617
2855
|
return captured;
|
|
2618
2856
|
}
|
|
@@ -2620,13 +2858,13 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2620
2858
|
return (await (0, spot_provider_1.createProviderInstance)(adapterType, this.options.providerResolver));
|
|
2621
2859
|
}
|
|
2622
2860
|
recordRelocationInterruption(meshName, unitKind, executionMode, startedAt, outcome) {
|
|
2623
|
-
const seconds = Math.max(0, performance.now() - startedAt) /
|
|
2861
|
+
const seconds = Math.max(0, performance.now() - startedAt) / runtime_metrics_1.MILLISECONDS_PER_SECOND;
|
|
2624
2862
|
const attributes = {
|
|
2625
2863
|
unit_kind: unitKind,
|
|
2626
2864
|
...(executionMode === undefined ? {} : { execution_mode: executionMode })
|
|
2627
2865
|
};
|
|
2628
|
-
this.options.metrics?.duration(
|
|
2629
|
-
this.options.metrics?.count(
|
|
2866
|
+
this.options.metrics?.duration(runtime_metrics_1.METRIC_NAMES.RelocationInterruption, seconds, attributes);
|
|
2867
|
+
this.options.metrics?.count(runtime_metrics_1.METRIC_NAMES.RelocationCompleted, 1, {
|
|
2630
2868
|
mesh_name: meshName,
|
|
2631
2869
|
object_kind: unitKind,
|
|
2632
2870
|
policy: unitKind === 'user_spot' && executionMode === 'per_actor' ? 'recreate' : 'configured',
|
|
@@ -2708,6 +2946,17 @@ class ZLinkHostServiceRelocationRuntime {
|
|
|
2708
2946
|
throw new Error('Host relocation requires the Spot manager.');
|
|
2709
2947
|
return value;
|
|
2710
2948
|
}
|
|
2949
|
+
afterSpotCloseDecision(meshName, spotId, seal) {
|
|
2950
|
+
const manager = this.requireSpotManager();
|
|
2951
|
+
const sealAfterDecision = () => {
|
|
2952
|
+
if (manager.isSpotClosing(meshName, spotId)) {
|
|
2953
|
+
throw (0, framework_errors_internal_1.createInternalFrameworkException)(framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotMoving, `Spot '${String(spotId)}' is closing.`, true);
|
|
2954
|
+
}
|
|
2955
|
+
return seal();
|
|
2956
|
+
};
|
|
2957
|
+
const decision = manager.pendingSpotCloseDecision(meshName, spotId);
|
|
2958
|
+
return decision === undefined ? sealAfterDecision() : decision.then(sealAfterDecision);
|
|
2959
|
+
}
|
|
2711
2960
|
requireActorManager() {
|
|
2712
2961
|
const value = this.options.actorManager();
|
|
2713
2962
|
if (value === undefined)
|
|
@@ -2793,16 +3042,10 @@ function actorJoinTargetMembership(manager, relocationId, participants) {
|
|
|
2793
3042
|
}
|
|
2794
3043
|
];
|
|
2795
3044
|
}
|
|
2796
|
-
function relocationDebug(marker, detail) {
|
|
2797
|
-
if (process.env.ZLINK_DEBUG_FRAMEWORK_RELOCATION !== '1')
|
|
2798
|
-
return;
|
|
2799
|
-
console.error('[zlink.runtime.relocation]', marker, detail);
|
|
2800
|
-
}
|
|
2801
3045
|
class LocalTargetPort {
|
|
2802
3046
|
options;
|
|
2803
3047
|
meshName;
|
|
2804
3048
|
envelope;
|
|
2805
|
-
deferredJoinRoots = new Map();
|
|
2806
3049
|
constructor(options, meshName, envelope) {
|
|
2807
3050
|
this.options = options;
|
|
2808
3051
|
this.meshName = meshName;
|
|
@@ -2819,7 +3062,7 @@ class LocalTargetPort {
|
|
|
2819
3062
|
}).globalId;
|
|
2820
3063
|
const localDescriptor = this.options.localDescriptor?.(this.meshName);
|
|
2821
3064
|
const nativeEntrySpot = localDescriptor?.entrySpotId === spotIdentity;
|
|
2822
|
-
const actor = await this.requireActorManager().prepareRelocationActor(identity, participant.stableType, participant.objectGeneration, participant.authorityOwnerGeneration + 1n,
|
|
3065
|
+
const actor = await this.requireActorManager().prepareRelocationActor(identity, participant.stableType, participant.objectGeneration, participant.authorityOwnerGeneration + 1n, nativeEntrySpot ? undefined : spotIdentity, membership.spotObjectGeneration, membership.membershipEpoch, signal);
|
|
2823
3066
|
return {
|
|
2824
3067
|
authorityKey: participant.key,
|
|
2825
3068
|
participant,
|
|
@@ -2905,8 +3148,14 @@ class LocalTargetPort {
|
|
|
2905
3148
|
const applicationSignaled = hidden.participant.objectKind === 'user_spot' &&
|
|
2906
3149
|
registration.options?.relocationCoordinationMode ===
|
|
2907
3150
|
contracts_1.ZLinkSpotRelocationCoordinationMode.ApplicationSignaled;
|
|
3151
|
+
// createHidden could only predict the owner generation; the native Spot
|
|
3152
|
+
// takes the committed authority, as the Actor branch does above.
|
|
3153
|
+
const node = this.options.meshNode(this.meshName);
|
|
3154
|
+
node?.restoreSpotAuthority?.(String(hidden.activation.spotId),
|
|
3155
|
+
// createHidden stages an activation only for a Spot participant.
|
|
3156
|
+
hidden.participant.objectKind, hidden.participant.stableType, authority.objectGeneration, authority.authorityOwnerGeneration);
|
|
2908
3157
|
await this.requireSpotManager().publishRelocationSpot(hidden.activation, applicationSignaled ? () => hidden.activation.notifyRelocatedBoundary() : undefined);
|
|
2909
|
-
|
|
3158
|
+
node?.rememberSpotRoute?.({
|
|
2910
3159
|
spot: {
|
|
2911
3160
|
spotId: String(hidden.activation.spotId),
|
|
2912
3161
|
generation: authority.objectGeneration
|
|
@@ -2936,15 +3185,14 @@ class LocalTargetPort {
|
|
|
2936
3185
|
hidden.initialized = true;
|
|
2937
3186
|
}
|
|
2938
3187
|
async restoreBoundSession(hidden, payload) {
|
|
2939
|
-
|
|
3188
|
+
const carried = decodeActorSession(payload);
|
|
3189
|
+
if (hidden.actor === undefined || carried === undefined)
|
|
2940
3190
|
return;
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
state.setBoundSessionBindingGeneration(target.bindingGeneration);
|
|
2947
|
-
}
|
|
3191
|
+
// The relocation delivers the binding it carried; the target installs it
|
|
3192
|
+
// through the one install rule (Session–Actor binding §6, §8).
|
|
3193
|
+
this.requireActorManager()
|
|
3194
|
+
.getState(hidden.actor.context.actorId)
|
|
3195
|
+
.installBoundSessionBinding(carried);
|
|
2948
3196
|
}
|
|
2949
3197
|
async replayQueuedMessage(hidden, message) {
|
|
2950
3198
|
if (hidden.actor === undefined) {
|
|
@@ -2955,10 +3203,7 @@ class LocalTargetPort {
|
|
|
2955
3203
|
if (recovery.request.actorId !== hidden.actor.context.actorId) {
|
|
2956
3204
|
throw new Error('Canonical Actor Join recovery names a different staged Actor.');
|
|
2957
3205
|
}
|
|
2958
|
-
|
|
2959
|
-
if (deferred !== undefined) {
|
|
2960
|
-
this.deferredJoinRoots.set(hidden.authorityKey, deferred);
|
|
2961
|
-
}
|
|
3206
|
+
await this.requireSpotManager().restoreCanonicalActorJoinRecovery(recovery);
|
|
2962
3207
|
return;
|
|
2963
3208
|
}
|
|
2964
3209
|
const packet = decodeQueuedHandoffPacket(message);
|
|
@@ -2967,9 +3212,6 @@ class LocalTargetPort {
|
|
|
2967
3212
|
throw new Error('Relocated Actor state is not staged.');
|
|
2968
3213
|
hidden.replayPackets.push(packet);
|
|
2969
3214
|
}
|
|
2970
|
-
deferredJoinRoot(authorityKey) {
|
|
2971
|
-
return this.deferredJoinRoots.get(authorityKey);
|
|
2972
|
-
}
|
|
2973
3215
|
async openAdmission(hidden) {
|
|
2974
3216
|
if (hidden.actor === undefined)
|
|
2975
3217
|
return;
|
|
@@ -3283,9 +3525,9 @@ function relocationWireId(value) {
|
|
|
3283
3525
|
}
|
|
3284
3526
|
function createServiceRelocationId(isInUse, entropy = node_crypto_1.randomBytes) {
|
|
3285
3527
|
for (;;) {
|
|
3286
|
-
const bytes = entropy(
|
|
3287
|
-
if (bytes.byteLength !==
|
|
3288
|
-
throw new Error(
|
|
3528
|
+
const bytes = entropy(operation_identity_1.OPERATION_IDENTITY_BYTES);
|
|
3529
|
+
if (bytes.byteLength !== operation_identity_1.OPERATION_IDENTITY_BYTES) {
|
|
3530
|
+
throw new Error(`Relocation identity entropy must return exactly ${operation_identity_1.OPERATION_IDENTITY_BYTES} bytes.`);
|
|
3289
3531
|
}
|
|
3290
3532
|
let nonZero = false;
|
|
3291
3533
|
for (const byte of bytes) {
|
|
@@ -3318,14 +3560,40 @@ function sameWireId(left, right) {
|
|
|
3318
3560
|
}
|
|
3319
3561
|
function sameCoordinator(left, right) {
|
|
3320
3562
|
return (left.ownerId === right.ownerId &&
|
|
3321
|
-
left.leaseGeneration === right.leaseGeneration &&
|
|
3563
|
+
BigInt(left.leaseGeneration) === right.leaseGeneration &&
|
|
3322
3564
|
left.nodeRid === right.nodeRid &&
|
|
3323
|
-
left.nodeGeneration === right.nodeGeneration &&
|
|
3565
|
+
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
3324
3566
|
left.expectedAuthorityStoreVersion === right.expectedAuthorityStoreVersion);
|
|
3325
3567
|
}
|
|
3568
|
+
function sameRelocationTarget(left, right) {
|
|
3569
|
+
return (left.nodeRid === right.nodeRid &&
|
|
3570
|
+
BigInt(left.nodeGeneration) === right.nodeGeneration &&
|
|
3571
|
+
left.ownerId === right.ownerId &&
|
|
3572
|
+
BigInt(left.ownerLeaseGeneration) === right.ownerLeaseGeneration);
|
|
3573
|
+
}
|
|
3574
|
+
function sameRelocationObject(left, right) {
|
|
3575
|
+
if (left.kind !== right.kind || BigInt(left.objectGeneration) !== right.objectGeneration) {
|
|
3576
|
+
return false;
|
|
3577
|
+
}
|
|
3578
|
+
if (left.kind === 'actor' && right.kind === 'actor') {
|
|
3579
|
+
return (left.actorId === right.actorId &&
|
|
3580
|
+
BigInt(left.expectedAuthorityOwnerGeneration) === right.expectedAuthorityOwnerGeneration);
|
|
3581
|
+
}
|
|
3582
|
+
if (left.kind === 'userSpot' && right.kind === 'userSpot') {
|
|
3583
|
+
return (left.spotId === right.spotId &&
|
|
3584
|
+
BigInt(left.expectedAuthorityOwnerGeneration) === right.expectedAuthorityOwnerGeneration);
|
|
3585
|
+
}
|
|
3586
|
+
return (left.kind === 'instanceSpot' &&
|
|
3587
|
+
right.kind === 'instanceSpot' &&
|
|
3588
|
+
left.spotId === right.spotId &&
|
|
3589
|
+
left.stableType === right.stableType);
|
|
3590
|
+
}
|
|
3326
3591
|
function isExactSourceAuthority(current, expected) {
|
|
3327
|
-
return (current.storeVersion.value === expected.storeVersion.value &&
|
|
3328
|
-
|
|
3592
|
+
return (current.storeVersion.value === expected.storeVersion.value && isSourceOwner(current, expected));
|
|
3593
|
+
}
|
|
3594
|
+
/** The same source tenure owns the row, at any StoreVersion. */
|
|
3595
|
+
function isSourceOwner(current, expected) {
|
|
3596
|
+
return (current.objectGeneration === expected.objectGeneration &&
|
|
3329
3597
|
current.authorityOwnerGeneration === expected.authorityOwnerGeneration &&
|
|
3330
3598
|
current.ownerId === expected.ownerId &&
|
|
3331
3599
|
current.ownerLeaseGeneration === expected.ownerLeaseGeneration &&
|
|
@@ -3345,26 +3613,6 @@ function isExactCommittedTarget(current, expected, target, payload) {
|
|
|
3345
3613
|
current.allocation.descriptorLifecycleGeneration === target.nodeGeneration &&
|
|
3346
3614
|
Buffer.from(current.payload).equals(Buffer.from(payload)));
|
|
3347
3615
|
}
|
|
3348
|
-
async function waitForLocationRetry(deadlineAtMs, signal, cause) {
|
|
3349
|
-
signal?.throwIfAborted();
|
|
3350
|
-
if (performance.now() >= deadlineAtMs) {
|
|
3351
|
-
throw new Error('location_update_failed: relocation Restore validity expired before exact target owner confirmation.', { cause });
|
|
3352
|
-
}
|
|
3353
|
-
await new Promise((resolve, reject) => {
|
|
3354
|
-
let timer;
|
|
3355
|
-
const aborted = () => {
|
|
3356
|
-
if (timer !== undefined)
|
|
3357
|
-
clearTimeout(timer);
|
|
3358
|
-
reject(signal?.reason);
|
|
3359
|
-
};
|
|
3360
|
-
signal?.addEventListener('abort', aborted, { once: true });
|
|
3361
|
-
timer = setTimeout(() => {
|
|
3362
|
-
signal?.removeEventListener('abort', aborted);
|
|
3363
|
-
resolve();
|
|
3364
|
-
}, Math.min(25, Math.max(1, deadlineAtMs - performance.now())));
|
|
3365
|
-
timer.unref();
|
|
3366
|
-
});
|
|
3367
|
-
}
|
|
3368
3616
|
async function waitForRelocationRetry(delayMs, signal) {
|
|
3369
3617
|
signal?.throwIfAborted();
|
|
3370
3618
|
await new Promise((resolve, reject) => {
|
|
@@ -3379,7 +3627,6 @@ async function waitForRelocationRetry(delayMs, signal) {
|
|
|
3379
3627
|
signal?.removeEventListener('abort', aborted);
|
|
3380
3628
|
resolve();
|
|
3381
3629
|
}, delayMs);
|
|
3382
|
-
timer.unref();
|
|
3383
3630
|
});
|
|
3384
3631
|
}
|
|
3385
3632
|
function sameRequestSource(left, right) {
|
|
@@ -3406,14 +3653,16 @@ function maintenanceReplyRelay(stage, coordinator, participantId, completion) {
|
|
|
3406
3653
|
replyRouteId: BigInt(completion.source.replyRouteId),
|
|
3407
3654
|
participantId,
|
|
3408
3655
|
sequence: BigInt(completion.index + 1),
|
|
3409
|
-
terminalResult: completion.result.ok ?
|
|
3410
|
-
failureCode: completion.result.ok
|
|
3656
|
+
terminalResult: completion.result.ok ? runtime_values_1.RequestResult.Ok : runtime_values_1.RequestResult.InternalError,
|
|
3657
|
+
failureCode: completion.result.ok
|
|
3658
|
+
? service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.none
|
|
3659
|
+
: service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.requestFailed,
|
|
3411
3660
|
...(value === undefined
|
|
3412
3661
|
? {}
|
|
3413
3662
|
: {
|
|
3414
3663
|
payload: {
|
|
3415
|
-
packetName:
|
|
3416
|
-
contentType:
|
|
3664
|
+
packetName: SERVICE_RELOCATION_REPLY_PACKET_NAME,
|
|
3665
|
+
contentType: stream_wire_1.ZlinkStreamContentType.Json,
|
|
3417
3666
|
bytes: Buffer.from(JSON.stringify(value), 'utf8')
|
|
3418
3667
|
}
|
|
3419
3668
|
})
|
|
@@ -3425,15 +3674,15 @@ function handoffResultFromRelay(relay) {
|
|
|
3425
3674
|
}
|
|
3426
3675
|
let value;
|
|
3427
3676
|
if (relay.payload !== undefined) {
|
|
3428
|
-
if (relay.terminalResult !==
|
|
3429
|
-
relay.payload.packetName !==
|
|
3430
|
-
relay.payload.contentType !==
|
|
3677
|
+
if (relay.terminalResult !== runtime_values_1.RequestResult.Ok ||
|
|
3678
|
+
relay.payload.packetName !== SERVICE_RELOCATION_REPLY_PACKET_NAME ||
|
|
3679
|
+
relay.payload.contentType !== stream_wire_1.ZlinkStreamContentType.Json) {
|
|
3431
3680
|
throw new TypeError('Relocation terminal payload boundary is invalid.');
|
|
3432
3681
|
}
|
|
3433
3682
|
value = JSON.parse(Buffer.from(relay.payload.bytes).toString('utf8'));
|
|
3434
3683
|
}
|
|
3435
3684
|
const index = Number(relay.sequence - 1n);
|
|
3436
|
-
return relay.terminalResult ===
|
|
3685
|
+
return relay.terminalResult === runtime_values_1.RequestResult.Ok
|
|
3437
3686
|
? { index, ok: true, ...(value === undefined ? {} : { response: value }) }
|
|
3438
3687
|
: { index, ok: false, error: `Actor handoff failed with framework code ${relay.failureCode}.` };
|
|
3439
3688
|
}
|
|
@@ -3448,73 +3697,15 @@ function wireIdText(value) {
|
|
|
3448
3697
|
class ServiceRelocationDataLostError extends Error {
|
|
3449
3698
|
}
|
|
3450
3699
|
exports.ServiceRelocationDataLostError = ServiceRelocationDataLostError;
|
|
3451
|
-
/**
|
|
3452
|
-
|
|
3453
|
-
* framework-error code the generated schema
|
|
3454
|
-
* ({@link ServiceWireFrameworkErrorCode}) defines — mirrors the java
|
|
3455
|
-
* reference mapping (commit 97fc074058) and the dotnet/cpp ports. The wire
|
|
3456
|
-
* vocabulary predates the framework's typed error kinds and has no
|
|
3457
|
-
* one-to-one code for every kind, so several kinds share the nearest fit
|
|
3458
|
-
* (documented per case below); unresolvable vocabulary gaps belong at the
|
|
3459
|
-
* schema level, not invented here. {@link ServiceRelocationDataLostError}
|
|
3460
|
-
* is this runtime's dedicated tag for a verified checksum/assembly/digest
|
|
3461
|
-
* integrity failure — the case that keeps relocationDataLost(35). An
|
|
3462
|
-
* unclassified error carries no evidence of integrity loss and takes the
|
|
3463
|
-
* generic opaque requestFailed(17). `objectKind` (the Prepare's
|
|
3464
|
-
* ObjectFence.kind, spec 28 §4.2) picks between an Actor- and
|
|
3465
|
-
* Spot-specific code where the schema splits by object kind.
|
|
3466
|
-
*/
|
|
3467
|
-
function relocationFailedFailureCode(error, objectKind) {
|
|
3700
|
+
/** Classifies relocation failure with the error model §2.1 mapping. */
|
|
3701
|
+
function relocationFailedFailureCode(error) {
|
|
3468
3702
|
if (error instanceof ServiceRelocationDataLostError) {
|
|
3469
|
-
return
|
|
3703
|
+
return (0, framework_errors_internal_1.frameworkRelocationFailureCode)(contracts_1.ZLinkFrameworkErrorKind.DataLost);
|
|
3470
3704
|
}
|
|
3471
3705
|
if (!(error instanceof contracts_1.ZLinkFrameworkException)) {
|
|
3472
|
-
return
|
|
3473
|
-
}
|
|
3474
|
-
switch (error.kind) {
|
|
3475
|
-
case contracts_1.ZLinkFrameworkErrorKind.DataLost:
|
|
3476
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.relocationDataLost;
|
|
3477
|
-
case contracts_1.ZLinkFrameworkErrorKind.Rejected:
|
|
3478
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.requestRejected;
|
|
3479
|
-
case contracts_1.ZLinkFrameworkErrorKind.ProtocolError:
|
|
3480
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.requestProtocolError;
|
|
3481
|
-
// No dedicated "deadline exceeded" wire code exists; a worker timeout
|
|
3482
|
-
// is the closest timeout-shaped signal.
|
|
3483
|
-
case contracts_1.ZLinkFrameworkErrorKind.DeadlineExceeded:
|
|
3484
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.workerTimedOut;
|
|
3485
|
-
// A stale generation/fence is the concrete cause of InvalidOperation
|
|
3486
|
-
// along this path (spec 15 failure table); pick the
|
|
3487
|
-
// object-kind-specific stale code.
|
|
3488
|
-
case contracts_1.ZLinkFrameworkErrorKind.InvalidOperation:
|
|
3489
|
-
return objectKind === 'actor'
|
|
3490
|
-
? service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.actorLocationStale
|
|
3491
|
-
: service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.spotGenerationStale;
|
|
3492
|
-
// No dedicated generic "unavailable" wire code exists; a disconnected
|
|
3493
|
-
// route is the closest "cannot reach/use the target" signal.
|
|
3494
|
-
case contracts_1.ZLinkFrameworkErrorKind.Unavailable:
|
|
3495
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.routeNotConnected;
|
|
3496
|
-
case contracts_1.ZLinkFrameworkErrorKind.NotFound:
|
|
3497
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.requestTargetNotFound;
|
|
3498
|
-
// The only "already exists" wire code is Actor-specific; not expected
|
|
3499
|
-
// along this target-failure path, mapped for completeness.
|
|
3500
|
-
case contracts_1.ZLinkFrameworkErrorKind.AlreadyExists:
|
|
3501
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.actorAlreadyExists;
|
|
3502
|
-
case contracts_1.ZLinkFrameworkErrorKind.TypeMismatch:
|
|
3503
|
-
return objectKind === 'actor'
|
|
3504
|
-
? service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.actorTypeMismatch
|
|
3505
|
-
: service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.spotTypeMismatch;
|
|
3506
|
-
// No dedicated "not configured" wire code exists; a missing configured
|
|
3507
|
-
// handler is the closest analog.
|
|
3508
|
-
case contracts_1.ZLinkFrameworkErrorKind.NotConfigured:
|
|
3509
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.handlerNotFound;
|
|
3510
|
-
// No dedicated generic "internal failure" or "shutting down" wire code
|
|
3511
|
-
// exists; the generic opaque request-failure code is the closest fit
|
|
3512
|
-
// for both (ShuttingDown included — re-judged 2026-08-19, C-10).
|
|
3513
|
-
case contracts_1.ZLinkFrameworkErrorKind.ShuttingDown:
|
|
3514
|
-
case contracts_1.ZLinkFrameworkErrorKind.InternalFailure:
|
|
3515
|
-
default:
|
|
3516
|
-
return service_wire_constants_generated_1.ServiceWireFrameworkErrorCode.requestFailed;
|
|
3706
|
+
return (0, framework_errors_internal_1.frameworkRelocationFailureCode)(contracts_1.ZLinkFrameworkErrorKind.InternalFailure);
|
|
3517
3707
|
}
|
|
3708
|
+
return (0, framework_errors_internal_1.frameworkRelocationFailureCode)(error);
|
|
3518
3709
|
}
|
|
3519
3710
|
function relocationStagingId(value) {
|
|
3520
3711
|
return `${value.relocation.high}:${value.relocation.low}:${value.targetAttemptGeneration}`;
|
|
@@ -3605,7 +3796,7 @@ function relocationFailed(request, failureCode) {
|
|
|
3605
3796
|
function validatePrepareEnvelope(request, envelope) {
|
|
3606
3797
|
if (!sameWireId(request.relocation, relocationWireId(envelope.aggregateId)) ||
|
|
3607
3798
|
request.applicationVersion !== envelope.applicationVersion ||
|
|
3608
|
-
|
|
3799
|
+
!sameRelocationObject(request.object, relocationObject(envelope))) {
|
|
3609
3800
|
throw new Error('Relocation Prepare does not match its immutable root identity.');
|
|
3610
3801
|
}
|
|
3611
3802
|
}
|
|
@@ -3615,14 +3806,9 @@ function validateTargetOneWayControl(stage, request, sourceNodeRid) {
|
|
|
3615
3806
|
!sameWireId(request.relocation, prepare.relocation) ||
|
|
3616
3807
|
request.targetAttemptGeneration !== prepare.targetAttemptGeneration ||
|
|
3617
3808
|
!sameCoordinator(request.coordinator, prepare.coordinator) ||
|
|
3618
|
-
|
|
3619
|
-
stringifyWire(request.kind === 'data' ? request.object : prepare.object)) {
|
|
3809
|
+
(request.kind === 'cutover' && !sameRelocationObject(request.object, prepare.object))) {
|
|
3620
3810
|
throw new Error('Relocation one-way control changed its exact Prepare fence.');
|
|
3621
3811
|
}
|
|
3622
|
-
if (request.kind === 'cutover' &&
|
|
3623
|
-
stringifyWire(request.object) !== stringifyWire(prepare.object)) {
|
|
3624
|
-
throw new Error('Relocation Cutover changed its prepared object.');
|
|
3625
|
-
}
|
|
3626
3812
|
}
|
|
3627
3813
|
function relocationPublication(prepare, envelope) {
|
|
3628
3814
|
// The direct transfer has no durable store root; the authority wrapper
|
|
@@ -3705,7 +3891,7 @@ function canonicalQueuedFrozenRecord(participant, message, coordinator, target)
|
|
|
3705
3891
|
...(packet.returnResponse ? { replyRouteId: BigInt(packet.source.replyRouteId) } : {}),
|
|
3706
3892
|
payload: {
|
|
3707
3893
|
packetName: '__zlink.actor.handoff.accepted',
|
|
3708
|
-
contentType:
|
|
3894
|
+
contentType: stream_wire_1.ZlinkStreamContentType.Json,
|
|
3709
3895
|
bytes: message.payload
|
|
3710
3896
|
}
|
|
3711
3897
|
});
|
|
@@ -3726,14 +3912,22 @@ function parseHandoffOperationId(value) {
|
|
|
3726
3912
|
function stringifyWire(value) {
|
|
3727
3913
|
return JSON.stringify(value, (_key, item) => (typeof item === 'bigint' ? item.toString() : item));
|
|
3728
3914
|
}
|
|
3915
|
+
function samePrepareIdentity(acceptedFingerprint, incoming) {
|
|
3916
|
+
const accepted = JSON.parse(acceptedFingerprint);
|
|
3917
|
+
return (accepted.sourceNodeRid === incoming.sourceNodeRid &&
|
|
3918
|
+
BigInt(accepted.sourceNodeGeneration) === incoming.sourceNodeGeneration &&
|
|
3919
|
+
sameCoordinator(accepted.coordinator, incoming.coordinator) &&
|
|
3920
|
+
sameRelocationTarget(accepted.target, incoming.target) &&
|
|
3921
|
+
sameRelocationObject(accepted.object, incoming.object));
|
|
3922
|
+
}
|
|
3729
3923
|
function validateControlResponse(request, response) {
|
|
3730
3924
|
if (request.kind !== 'prepare' ||
|
|
3731
3925
|
response.kind !== 'ready' ||
|
|
3732
3926
|
!sameWireId(response.relocation, request.relocation) ||
|
|
3733
3927
|
response.targetAttemptGeneration !== request.targetAttemptGeneration ||
|
|
3734
3928
|
!sameCoordinator(response.coordinator, request.coordinator) ||
|
|
3735
|
-
|
|
3736
|
-
|
|
3929
|
+
!sameRelocationTarget(response.target, request.target) ||
|
|
3930
|
+
!sameRelocationObject(response.object, request.object) ||
|
|
3737
3931
|
response.senderRole !== 'target') {
|
|
3738
3932
|
throw new Error('Relocation control response does not match the request fence.');
|
|
3739
3933
|
}
|
|
@@ -3750,78 +3944,16 @@ function validateControlFailureResponse(request, response) {
|
|
|
3750
3944
|
!sameWireId(response.relocation, request.relocation) ||
|
|
3751
3945
|
response.targetAttemptGeneration !== request.targetAttemptGeneration ||
|
|
3752
3946
|
!sameCoordinator(response.coordinator, request.coordinator) ||
|
|
3753
|
-
|
|
3754
|
-
|
|
3947
|
+
!sameRelocationTarget(response.target, request.target) ||
|
|
3948
|
+
!sameRelocationObject(response.object, request.object) ||
|
|
3755
3949
|
response.senderRole !== 'target') {
|
|
3756
3950
|
throw new Error('Relocation control response does not match the request fence.');
|
|
3757
3951
|
}
|
|
3758
3952
|
}
|
|
3759
|
-
/**
|
|
3760
|
-
* Maps a relocationFailed(53) wire failureCode (the generated
|
|
3761
|
-
* ServiceWireFrameworkErrorCode vocabulary) to its internal error kind, so a
|
|
3762
|
-
* source-side reject carries the same classification the target chose
|
|
3763
|
-
* instead of a generic Error. Direct 1:1 correspondence with the generated
|
|
3764
|
-
* names; an unrecognised or absent code falls back to RequestFailed
|
|
3765
|
-
* (InternalFailure) — the closest existing kind (spec 15 §"Failed.Kind").
|
|
3766
|
-
*/
|
|
3767
|
-
function relocationFailureCodeKind(failureCode) {
|
|
3768
|
-
switch (failureCode) {
|
|
3769
|
-
case 1:
|
|
3770
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorRouteNotFound;
|
|
3771
|
-
case 2:
|
|
3772
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorCreateFailed;
|
|
3773
|
-
case 3:
|
|
3774
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorAlreadyExists;
|
|
3775
|
-
case 4:
|
|
3776
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorTypeMismatch;
|
|
3777
|
-
case 5:
|
|
3778
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotCreateFailed;
|
|
3779
|
-
case 6:
|
|
3780
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotRouteNotFound;
|
|
3781
|
-
case 7:
|
|
3782
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotTypeMismatch;
|
|
3783
|
-
case 8:
|
|
3784
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorSessionNotBound;
|
|
3785
|
-
case 9:
|
|
3786
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.HandlerNotFound;
|
|
3787
|
-
case 10:
|
|
3788
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteHandlerNotFound;
|
|
3789
|
-
case 11:
|
|
3790
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorDispatchHandlerNotFound;
|
|
3791
|
-
case 12:
|
|
3792
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.PayloadDecodeFailed;
|
|
3793
|
-
case 13:
|
|
3794
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RouteNotConnected;
|
|
3795
|
-
case 14:
|
|
3796
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RequestTargetNotFound;
|
|
3797
|
-
case 15:
|
|
3798
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RequestRejected;
|
|
3799
|
-
case 16:
|
|
3800
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RequestProtocolError;
|
|
3801
|
-
case 18:
|
|
3802
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.WorkerQueueFull;
|
|
3803
|
-
case 19:
|
|
3804
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.WorkerTimedOut;
|
|
3805
|
-
case 20:
|
|
3806
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.WorkerFailed;
|
|
3807
|
-
case 21:
|
|
3808
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorLocationStale;
|
|
3809
|
-
case 22:
|
|
3810
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.ActorCreateRejected;
|
|
3811
|
-
case 33:
|
|
3812
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotGenerationStale;
|
|
3813
|
-
case 34:
|
|
3814
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.SpotMoving;
|
|
3815
|
-
case 35:
|
|
3816
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RelocationDataLost;
|
|
3817
|
-
case 17:
|
|
3818
|
-
default:
|
|
3819
|
-
return framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RequestFailed;
|
|
3820
|
-
}
|
|
3821
|
-
}
|
|
3822
3953
|
/** Builds the typed exception a Prepare waiter rejects with on an explicit Failed(53). */
|
|
3823
3954
|
function relocationFailureException(response) {
|
|
3824
|
-
return (0, framework_errors_internal_1.createInternalFrameworkException)(
|
|
3955
|
+
return (0, framework_errors_internal_1.createInternalFrameworkException)((0, framework_errors_internal_1.internalFrameworkErrorKindFromWireFailureCode)(response.failureCode) ??
|
|
3956
|
+
framework_errors_internal_1.ZLinkFrameworkInternalErrorKind.RequestFailed, `Relocation '${response.relocation.high}:${response.relocation.low}:` +
|
|
3825
3957
|
`${response.targetAttemptGeneration}' failed with wire failureCode ${response.failureCode}.`, true);
|
|
3826
3958
|
}
|
|
3827
3959
|
function controlAckKey(request) {
|
|
@@ -3950,7 +4082,7 @@ function queuedMessageU64(value, field) {
|
|
|
3950
4082
|
throw new TypeError(`Relocation Actor Message Follow ${field} is invalid.`);
|
|
3951
4083
|
}
|
|
3952
4084
|
const result = BigInt(value);
|
|
3953
|
-
if (result >
|
|
4085
|
+
if (result > stream_wire_1.UINT64_MAX) {
|
|
3954
4086
|
throw new TypeError(`Relocation Actor Message Follow ${field} is outside u64.`);
|
|
3955
4087
|
}
|
|
3956
4088
|
return result;
|
|
@@ -3970,6 +4102,14 @@ function toServiceTimer(value) {
|
|
|
3970
4102
|
pendingTicks: value.pendingTicks
|
|
3971
4103
|
};
|
|
3972
4104
|
}
|
|
4105
|
+
const PREPARE_SIDEBAND_MAGIC = 'ZLNI';
|
|
4106
|
+
const PREPARE_SIDEBAND_VERSION = 1;
|
|
4107
|
+
const PREPARE_SIDEBAND_HEADER_BYTES = 11;
|
|
4108
|
+
const PREPARE_SIDEBAND_VERSION_OFFSET = 4;
|
|
4109
|
+
const PREPARE_SIDEBAND_KEY_LENGTH_OFFSET = 5;
|
|
4110
|
+
const PREPARE_SIDEBAND_PAYLOAD_LENGTH_OFFSET = 7;
|
|
4111
|
+
const ACTOR_SESSION_JOURNAL_VERSION = 1;
|
|
4112
|
+
const SERVICE_RELOCATION_REPLY_PACKET_NAME = 'zlink.relocation.reply';
|
|
3973
4113
|
/**
|
|
3974
4114
|
* NODE-INTERNAL Prepare sideband. Frame zero remains the exact canonical
|
|
3975
4115
|
* command-40 schema record; each following ZLNI frame carries one sealed
|
|
@@ -3985,11 +4125,11 @@ function encodePrepareSideband(envelope) {
|
|
|
3985
4125
|
if (key.byteLength === 0 || key.byteLength > 0xffff || payload.byteLength > 0xffff_ffff) {
|
|
3986
4126
|
throw new TypeError('Bound-session relocation sideband is out of bounds.');
|
|
3987
4127
|
}
|
|
3988
|
-
const header = Buffer.allocUnsafe(
|
|
3989
|
-
header.write(
|
|
3990
|
-
header[
|
|
3991
|
-
header.writeUInt16BE(key.byteLength,
|
|
3992
|
-
header.writeUInt32BE(payload.byteLength,
|
|
4128
|
+
const header = Buffer.allocUnsafe(PREPARE_SIDEBAND_HEADER_BYTES);
|
|
4129
|
+
header.write(PREPARE_SIDEBAND_MAGIC, 0, 'ascii');
|
|
4130
|
+
header[PREPARE_SIDEBAND_VERSION_OFFSET] = PREPARE_SIDEBAND_VERSION;
|
|
4131
|
+
header.writeUInt16BE(key.byteLength, PREPARE_SIDEBAND_KEY_LENGTH_OFFSET);
|
|
4132
|
+
header.writeUInt32BE(payload.byteLength, PREPARE_SIDEBAND_PAYLOAD_LENGTH_OFFSET);
|
|
3993
4133
|
return Buffer.concat([header, key, payload]);
|
|
3994
4134
|
});
|
|
3995
4135
|
}
|
|
@@ -3998,20 +4138,27 @@ function decodePrepareSideband(frames) {
|
|
|
3998
4138
|
const keys = new Set();
|
|
3999
4139
|
for (const frame of frames) {
|
|
4000
4140
|
const bytes = Buffer.from(frame);
|
|
4001
|
-
if (bytes.byteLength <
|
|
4002
|
-
bytes.subarray(0,
|
|
4003
|
-
|
|
4141
|
+
if (bytes.byteLength < PREPARE_SIDEBAND_HEADER_BYTES ||
|
|
4142
|
+
bytes.subarray(0, PREPARE_SIDEBAND_VERSION_OFFSET).toString('ascii') !==
|
|
4143
|
+
PREPARE_SIDEBAND_MAGIC ||
|
|
4144
|
+
bytes[PREPARE_SIDEBAND_VERSION_OFFSET] !== PREPARE_SIDEBAND_VERSION) {
|
|
4004
4145
|
return undefined;
|
|
4005
4146
|
}
|
|
4006
|
-
const keyLength = bytes.readUInt16BE(
|
|
4007
|
-
const payloadLength = bytes.readUInt32BE(
|
|
4008
|
-
if (
|
|
4147
|
+
const keyLength = bytes.readUInt16BE(PREPARE_SIDEBAND_KEY_LENGTH_OFFSET);
|
|
4148
|
+
const payloadLength = bytes.readUInt32BE(PREPARE_SIDEBAND_PAYLOAD_LENGTH_OFFSET);
|
|
4149
|
+
if (PREPARE_SIDEBAND_HEADER_BYTES + keyLength + payloadLength !== bytes.byteLength ||
|
|
4150
|
+
keyLength === 0)
|
|
4009
4151
|
return undefined;
|
|
4010
|
-
const participantKey = bytes
|
|
4152
|
+
const participantKey = bytes
|
|
4153
|
+
.subarray(PREPARE_SIDEBAND_HEADER_BYTES, PREPARE_SIDEBAND_HEADER_BYTES + keyLength)
|
|
4154
|
+
.toString('utf8');
|
|
4011
4155
|
if (keys.has(participantKey))
|
|
4012
4156
|
return undefined;
|
|
4013
4157
|
keys.add(participantKey);
|
|
4014
|
-
result.push({
|
|
4158
|
+
result.push({
|
|
4159
|
+
participantKey,
|
|
4160
|
+
payload: Buffer.from(bytes.subarray(PREPARE_SIDEBAND_HEADER_BYTES + keyLength))
|
|
4161
|
+
});
|
|
4015
4162
|
}
|
|
4016
4163
|
return result;
|
|
4017
4164
|
}
|
|
@@ -4037,12 +4184,15 @@ function encodeActorSession(target) {
|
|
|
4037
4184
|
if (target === undefined)
|
|
4038
4185
|
return Buffer.alloc(0);
|
|
4039
4186
|
return Buffer.from(JSON.stringify({
|
|
4040
|
-
version:
|
|
4187
|
+
version: ACTOR_SESSION_JOURNAL_VERSION,
|
|
4041
4188
|
routerChannelId: target.routerChannelId,
|
|
4042
4189
|
targetNodeRid: String(target.targetNodeRid),
|
|
4043
4190
|
spotId: String(target.spotId),
|
|
4044
4191
|
sessionNodeRid: target.sessionNodeRid === undefined ? undefined : String(target.sessionNodeRid),
|
|
4045
4192
|
sessionRid: target.sessionRid === undefined ? undefined : String(target.sessionRid),
|
|
4193
|
+
sessionOwnerNodeGeneration: target.sessionOwnerNodeGeneration?.toString(),
|
|
4194
|
+
sessionOwnerId: target.sessionOwnerId,
|
|
4195
|
+
sessionOwnerLeaseGeneration: target.sessionOwnerLeaseGeneration?.toString(),
|
|
4046
4196
|
bindingGeneration: target.bindingGeneration?.toString(),
|
|
4047
4197
|
previousAuthorityOwnerGeneration: target.previousAuthorityOwnerGeneration?.toString(),
|
|
4048
4198
|
previousOwnerLeaseGeneration: target.previousOwnerLeaseGeneration?.toString(),
|
|
@@ -4067,9 +4217,11 @@ function encodeActorSession(target) {
|
|
|
4067
4217
|
}), 'utf8');
|
|
4068
4218
|
}
|
|
4069
4219
|
function decodeActorSession(payload) {
|
|
4220
|
+
if (payload.byteLength === 0)
|
|
4221
|
+
return undefined;
|
|
4070
4222
|
const value = JSON.parse(Buffer.from(payload).toString('utf8'));
|
|
4071
4223
|
const optionalBigInt = (field) => typeof value[field] === 'string' ? BigInt(value[field]) : undefined;
|
|
4072
|
-
if (value.version !==
|
|
4224
|
+
if (value.version !== ACTOR_SESSION_JOURNAL_VERSION ||
|
|
4073
4225
|
typeof value.routerChannelId !== 'string' ||
|
|
4074
4226
|
typeof value.targetNodeRid !== 'string' ||
|
|
4075
4227
|
typeof value.spotId !== 'string') {
|
|
@@ -4084,6 +4236,13 @@ function decodeActorSession(payload) {
|
|
|
4084
4236
|
? { sessionNodeRid: value.sessionNodeRid }
|
|
4085
4237
|
: {}),
|
|
4086
4238
|
...(typeof value.sessionRid === 'string' ? { sessionRid: value.sessionRid } : {}),
|
|
4239
|
+
...(optionalBigInt('sessionOwnerNodeGeneration') === undefined
|
|
4240
|
+
? {}
|
|
4241
|
+
: { sessionOwnerNodeGeneration: optionalBigInt('sessionOwnerNodeGeneration') }),
|
|
4242
|
+
...(typeof value.sessionOwnerId === 'string' ? { sessionOwnerId: value.sessionOwnerId } : {}),
|
|
4243
|
+
...(optionalBigInt('sessionOwnerLeaseGeneration') === undefined
|
|
4244
|
+
? {}
|
|
4245
|
+
: { sessionOwnerLeaseGeneration: optionalBigInt('sessionOwnerLeaseGeneration') }),
|
|
4087
4246
|
...(optionalBigInt('bindingGeneration') === undefined
|
|
4088
4247
|
? {}
|
|
4089
4248
|
: { bindingGeneration: optionalBigInt('bindingGeneration') }),
|
|
@@ -4182,3 +4341,9 @@ function addCapacity(left, right) {
|
|
|
4182
4341
|
})
|
|
4183
4342
|
};
|
|
4184
4343
|
}
|
|
4344
|
+
if ((0, state_lane_1.isStructuralGuardEnabled)()) {
|
|
4345
|
+
const relocateActorJoin = ZLinkHostServiceRelocationRuntime.prototype.relocateActorJoin;
|
|
4346
|
+
ZLinkHostServiceRelocationRuntime.prototype.relocateActorJoin = function (input) {
|
|
4347
|
+
return (0, state_lane_1.trackDiagnosticCompletion)(relocateActorJoin.call(this, input), `relocation join actor:${input.state.actorId}`);
|
|
4348
|
+
};
|
|
4349
|
+
}
|