@zlink-systems/framework 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/contracts/Common/ZLinkMessage.js +5 -5
- package/dist/contracts/Configuration/CodecContentType.js +5 -7
- package/dist/contracts/Configuration/DiagnosticsValidation.js +1 -6
- package/dist/contracts/Configuration/EndpointNotation.js +4 -8
- package/dist/contracts/Configuration/Registration.js +14 -11
- package/dist/contracts/Configuration/RegistrationBuilders.js +32 -25
- package/dist/contracts/Configuration/RegistrationNormalizers.js +25 -26
- package/dist/contracts/Configuration/RegistrationValidators.js +36 -38
- package/dist/contracts/Configuration/SendTimeoutValidation.js +4 -4
- package/dist/contracts/Configuration/TimerRegistrationValidator.js +7 -7
- package/dist/contracts/Handlers/JsonContract.js +23 -11
- package/dist/contracts/Spots/SpotKind.js +16 -8
- package/dist/location-store-integration.js +16 -16
- package/dist/nest-integration.js +17 -11
- package/dist/runtime/abort.js +9 -5
- package/dist/runtime/actors/actor-authority-payload-codec.js +12 -16
- package/dist/runtime/actors/actor-authority-publication.js +18 -17
- package/dist/runtime/actors/actor-client.js +83 -63
- package/dist/runtime/actors/actor-context.js +24 -18
- package/dist/runtime/actors/actor-creation.js +4 -3
- package/dist/runtime/actors/actor-handoff.js +89 -92
- package/dist/runtime/actors/actor-local-native-join.js +18 -19
- package/dist/runtime/actors/actor-mailbox.js +1 -1
- package/dist/runtime/actors/actor-message-follow-context.js +24 -21
- package/dist/runtime/actors/actor-packet-relay-wire.js +1 -3
- package/dist/runtime/actors/actor-remote-joiner.js +20 -17
- package/dist/runtime/actors/actor-remote-wire.js +6 -7
- package/dist/runtime/actors/actor-runtime-state.js +87 -81
- package/dist/runtime/actors/actor-transfer-registry.js +3 -5
- package/dist/runtime/actors/bound-session-wire.js +14 -9
- package/dist/runtime/actors/deferred-join-accepted-journal.js +158 -124
- package/dist/runtime/actors/index.js +2 -5
- package/dist/runtime/actors/post-commit-actor-binder.js +3 -6
- package/dist/runtime/actors/post-commit-actor-location.js +1 -1
- package/dist/runtime/actors/spot-actor-dispatch.js +5 -5
- package/dist/runtime/actors/transferred-actor-rollback.js +4 -4
- package/dist/runtime/admission.js +2 -6
- package/dist/runtime/application-jobs/application-ingress-record-owner.js +2 -6
- package/dist/runtime/application-jobs/receive-flow-controller.js +11 -12
- package/dist/runtime/backend/mesh-completion-table.js +1 -3
- package/dist/runtime/backend/mesh-dispatch-pump.js +12 -16
- package/dist/runtime/backend/node/node-backend-adapter-factory.js +1 -3
- package/dist/runtime/backend/node/node-backend-adapter-support.js +26 -17
- package/dist/runtime/backend/node/node-mesh-backend-adapter.js +4 -3
- package/dist/runtime/backend/node/node-raw-binding-port.js +7 -6
- package/dist/runtime/backend/node/node-raw-mesh-backend.js +116 -113
- package/dist/runtime/backend/node/node-socket-backend-adapter.js +12 -13
- package/dist/runtime/backend/runtime-message.js +3 -3
- package/dist/runtime/backend/runtime-values.d.ts +2 -2
- package/dist/runtime/backend/runtime-values.js +4 -5
- package/dist/runtime/channels/channel-clients.js +27 -24
- package/dist/runtime/channels/channel-dispatch-pipeline.js +10 -18
- package/dist/runtime/channels/channel-dispatch-services.js +7 -4
- package/dist/runtime/channels/channel-dispatchers.js +9 -9
- package/dist/runtime/channels/channel-envelope-inspection.js +16 -13
- package/dist/runtime/channels/channel-envelope.js +83 -47
- package/dist/runtime/channels/channel-outbound-operations.js +8 -8
- package/dist/runtime/channels/channel-receive-loops.js +20 -22
- package/dist/runtime/channels/channel-receive-task-tracker.js +1 -1
- package/dist/runtime/channels/channel-runtime-lifecycle.js +61 -56
- package/dist/runtime/channels/channel-runtime-manager.js +18 -9
- package/dist/runtime/channels/channel-socket-registry.js +126 -123
- package/dist/runtime/channels/channel-transports.js +43 -33
- package/dist/runtime/channels/client-server-location-runtime.js +32 -40
- package/dist/runtime/channels/client-server-service-wire.js +29 -22
- package/dist/runtime/channels/dispatch-error-reporter.js +9 -7
- package/dist/runtime/channels/fanout-location-runtime.js +34 -47
- package/dist/runtime/channels/fanout-service-wire.js +3 -5
- package/dist/runtime/channels/local-spot-route-transport.js +7 -1
- package/dist/runtime/channels/route-member-snapshot.js +3 -3
- package/dist/runtime/channels/source-spot-router.js +6 -2
- package/dist/runtime/channels/spot-route-bridge-transport.js +5 -2
- package/dist/runtime/channels/spot-route-dispatch-strategy.js +2 -2
- package/dist/runtime/channels/spot-route-target-resolver.js +6 -6
- package/dist/runtime/diagnostics/index.js +38 -10
- package/dist/runtime/diagnostics/message-flow.js +47 -39
- package/dist/runtime/diagnostics/runtime-metrics.js +18 -9
- package/dist/runtime/diagnostics/runtime-observation-queue.js +8 -10
- package/dist/runtime/diagnostics/topology-runtime-projections.js +17 -10
- package/dist/runtime/execution/index.js +7 -6
- package/dist/runtime/execution/serial-execution-queue.js +12 -18
- package/dist/runtime/execution/state-lane.js +2 -2
- package/dist/runtime/foundation/actor-join-recovery-codec.js +31 -23
- package/dist/runtime/foundation/event-loop-resources.js +4 -4
- package/dist/runtime/foundation/message-follow-suppression-registry.js +4 -4
- package/dist/runtime/foundation/raw-service-mesh-runtime.js +136 -136
- package/dist/runtime/foundation/route-mesh-connection-policy.js +4 -4
- package/dist/runtime/foundation/runtime-state-projections.js +22 -11
- package/dist/runtime/foundation/service-authority-payload-codec.js +13 -15
- package/dist/runtime/foundation/service-discovery-registry.js +35 -33
- package/dist/runtime/foundation/service-instance-activation-recovery-codec.js +20 -22
- package/dist/runtime/foundation/service-liveness-registry.js +16 -12
- package/dist/runtime/foundation/service-maintenance-runtime.js +2 -4
- package/dist/runtime/foundation/service-relocation-aggregate-committer.js +24 -24
- package/dist/runtime/foundation/service-relocation-object-owner.js +6 -6
- package/dist/runtime/foundation/service-relocation-runtime.js +137 -94
- package/dist/runtime/foundation/service-runtime-contracts.js +5 -5
- package/dist/runtime/foundation/service-stateful-registry.js +24 -23
- package/dist/runtime/foundation/service-stateful-runtime.js +416 -438
- package/dist/runtime/foundation/service-stateful-wire-codec.js +95 -98
- package/dist/runtime/foundation/service-topology-registry.js +96 -95
- package/dist/runtime/foundation/service-weighted-selection.js +6 -11
- package/dist/runtime/foundation/service-wire-binary-primitives.js +5 -6
- package/dist/runtime/foundation/service-wire-m6a-codec.js +46 -35
- package/dist/runtime/framework-errors-internal.js +7 -10
- package/dist/runtime/handlers/handler-instance-scope.js +4 -5
- package/dist/runtime/handlers/index.js +3 -1
- package/dist/runtime/host/actor-packet-relay.js +74 -70
- package/dist/runtime/host/actor-placement-coordinator.js +29 -31
- package/dist/runtime/host/actor-runtime-options-factory.js +12 -14
- package/dist/runtime/host/actor-transfer-authority-runtime.js +7 -9
- package/dist/runtime/host/actor-transfer-runtime.js +176 -153
- package/dist/runtime/host/application-job-queue.js +18 -22
- package/dist/runtime/host/bound-session-relay.js +3 -1
- package/dist/runtime/host/channel-runtime-options-factory.js +31 -14
- package/dist/runtime/host/host-capacity-status.js +8 -4
- package/dist/runtime/host/index.js +244 -227
- package/dist/runtime/host/instance-activation-authority.js +93 -96
- package/dist/runtime/host/location-runtime-owner.js +16 -14
- package/dist/runtime/host/mesh-router-resolver.js +11 -13
- package/dist/runtime/host/relocation-direct-transfer.js +10 -9
- package/dist/runtime/host/remote-actor-join-receiver.js +20 -15
- package/dist/runtime/host/remote-actor-packet-target-store.js +18 -22
- package/dist/runtime/host/remote-bound-session-relay.js +97 -73
- package/dist/runtime/host/route-mesh-runtime.js +64 -69
- package/dist/runtime/host/runtime-shutdown.js +1 -2
- package/dist/runtime/host/service-relocation-host-runtime.js +586 -511
- package/dist/runtime/host/spot-address-transport.js +69 -71
- package/dist/runtime/host/spot-runtime-options-factory.js +30 -29
- package/dist/runtime/host/stateful-authority-route-runtime.js +65 -68
- package/dist/runtime/host/user-spot-creation-coordinator.js +61 -67
- package/dist/runtime/locations/actor-location-claims.js +26 -20
- package/dist/runtime/locations/aggregate-inventory-store.js +72 -75
- package/dist/runtime/locations/authority-key-codec.js +7 -7
- package/dist/runtime/locations/auto-connect-loop.js +3 -3
- package/dist/runtime/locations/auto-connect-planner.js +25 -19
- package/dist/runtime/locations/auto-connect-reconciler.js +51 -31
- package/dist/runtime/locations/in-memory-authority-store.js +128 -118
- package/dist/runtime/locations/in-memory-location-store.js +195 -168
- package/dist/runtime/locations/in-memory-provider-location-store.js +8 -10
- package/dist/runtime/locations/key-codec.js +1 -1
- package/dist/runtime/locations/lease-tracker.js +23 -21
- package/dist/runtime/locations/lifecycle.js +4 -5
- package/dist/runtime/locations/location-store-repository.js +368 -324
- package/dist/runtime/locations/opaque-record-key.js +2 -2
- package/dist/runtime/locations/resolvers.js +73 -82
- package/dist/runtime/locations/runtime.js +68 -63
- package/dist/runtime/locations/spot-location-claims.js +39 -35
- package/dist/runtime/messaging/framework-json-v1.js +7 -7
- package/dist/runtime/messaging/packet-name.js +7 -6
- package/dist/runtime/messaging/payload-codec.js +21 -18
- package/dist/runtime/protocol/service_wire_codec.generated.d.ts +30 -2
- package/dist/runtime/protocol/service_wire_codec.generated.js +1671 -234
- package/dist/runtime/routing-id.js +3 -5
- package/dist/runtime/spots/formal-remote-actor-admission-registry.js +35 -31
- package/dist/runtime/spots/formal-remote-actor-transfer-registry.js +1 -2
- package/dist/runtime/spots/index.js +208 -219
- package/dist/runtime/spots/spot-activation-registry.js +15 -14
- package/dist/runtime/spots/spot-activation-state.js +17 -15
- package/dist/runtime/spots/spot-activation.js +33 -16
- package/dist/runtime/spots/spot-actor-admission-coordinator.js +18 -15
- package/dist/runtime/spots/spot-actor-membership.js +1 -3
- package/dist/runtime/spots/spot-actor-packet-dispatch.js +23 -20
- package/dist/runtime/spots/spot-actor-packet-drain.js +12 -10
- package/dist/runtime/spots/spot-actor-packet-relay-dispatch.js +10 -6
- package/dist/runtime/spots/spot-closing.js +3 -7
- package/dist/runtime/spots/spot-context.js +2 -2
- package/dist/runtime/spots/spot-entry-activation.js +13 -14
- package/dist/runtime/spots/spot-handle.js +6 -3
- package/dist/runtime/spots/spot-handler-registry.js +12 -12
- package/dist/runtime/spots/spot-manager-public.js +5 -6
- package/dist/runtime/spots/spot-native-actor-join-admission.js +4 -3
- package/dist/runtime/spots/spot-node-autoconnect.js +15 -18
- package/dist/runtime/spots/spot-node-runtime-manager.js +74 -85
- package/dist/runtime/spots/spot-outbound.js +10 -14
- package/dist/runtime/spots/spot-remote-codec.js +22 -26
- package/dist/runtime/spots/spot-remote-route-codec.js +5 -7
- package/dist/runtime/spots/spot-route-packet-dispatch.js +16 -14
- package/dist/runtime/spots/spot-route-replies.js +4 -6
- package/dist/runtime/spots/spot-routed-actor-admission.js +9 -7
- package/dist/runtime/spots/spot-routed-bound-session-dispatch.js +1 -2
- package/dist/runtime/spots/spot-routed-frame-dispatch.js +5 -5
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.js +14 -10
- package/dist/runtime/spots/spot-serial-executor.js +5 -8
- package/dist/runtime/spots/spot-serial-turn-executor.js +4 -6
- package/dist/runtime/spots/spot-subscription-dispatch.js +5 -5
- package/dist/runtime/spots/spot-timer.js +25 -29
- package/dist/runtime/streams/actor-session-binding-registry.js +98 -107
- package/dist/runtime/streams/actor-session-lifecycle-coordinator.js +3 -1
- package/dist/runtime/streams/bound-actor-relay-sender.js +18 -16
- package/dist/runtime/streams/bound-session-service.js +8 -9
- package/dist/runtime/streams/index.js +20 -19
- package/dist/runtime/streams/managed-stream.js +32 -30
- package/dist/runtime/streams/native-fallback-bound-session.js +21 -17
- package/dist/runtime/streams/protocol.js +29 -18
- package/dist/runtime/streams/session-actor-coordinator.js +34 -37
- package/dist/runtime/streams/session-context.js +30 -28
- package/dist/runtime/streams/session-provider.js +2 -2
- package/dist/runtime/streams/session-serial-executor.js +1 -2
- package/dist/runtime/streams/stream-frame-factory.js +11 -5
- package/dist/runtime/streams/stream-session-runtime.js +70 -66
- package/dist/runtime/workers/index.js +15 -4
- package/package.json +2 -2
- package/src/contracts/Channels/RouteCalls.ts +6 -1
- package/src/contracts/Common/ZLinkMessage.ts +5 -5
- package/src/contracts/Common/ZLinkMessageMetadata.ts +4 -3
- package/src/contracts/Configuration/Builders.ts +14 -14
- package/src/contracts/Configuration/CodecContentType.ts +8 -10
- package/src/contracts/Configuration/DiagnosticsValidation.ts +1 -6
- package/src/contracts/Configuration/EndpointNotation.ts +11 -11
- package/src/contracts/Configuration/ObjectRoles.ts +1 -3
- package/src/contracts/Configuration/Registration.ts +46 -17
- package/src/contracts/Configuration/RegistrationBuilderPolicy.ts +10 -8
- package/src/contracts/Configuration/RegistrationBuilders.ts +107 -96
- package/src/contracts/Configuration/RegistrationCodecRegistry.ts +9 -3
- package/src/contracts/Configuration/RegistrationNormalizers.ts +116 -84
- package/src/contracts/Configuration/RegistrationTypes.ts +21 -25
- package/src/contracts/Configuration/RegistrationValidators.ts +122 -73
- package/src/contracts/Configuration/RouteChannelInternalState.ts +1 -4
- package/src/contracts/Configuration/RuntimeEndpointConnections.ts +4 -1
- package/src/contracts/Configuration/SendTimeoutValidation.ts +6 -4
- package/src/contracts/Configuration/TimerRegistrationValidator.ts +8 -13
- package/src/contracts/Dispatch/ZLinkDispatchOptions.ts +1 -6
- package/src/contracts/Errors/index.ts +1 -4
- package/src/contracts/Handlers/Attributes.ts +15 -9
- package/src/contracts/Handlers/IZLinkChannelHandlers.ts +24 -4
- package/src/contracts/Handlers/JsonContract.ts +53 -19
- package/src/contracts/Locations/ActorTransfer.ts +1 -5
- package/src/contracts/Locations/Authority.ts +4 -11
- package/src/contracts/Locations/Keys.ts +6 -1
- package/src/contracts/Locations/Options.ts +3 -2
- package/src/contracts/Locations/RelocationStore.ts +1 -4
- package/src/contracts/Locations/Resolvers.ts +5 -5
- package/src/contracts/Locations/RuntimeQuery.ts +6 -2
- package/src/contracts/Locations/Stores.ts +1 -2
- package/src/contracts/Locations/index.ts +2 -8
- package/src/contracts/RouteMesh/Contracts.ts +1 -7
- package/src/contracts/RouteMesh/RuntimeTopology.ts +1 -4
- package/src/contracts/Spots/Contracts.ts +2 -8
- package/src/contracts/Spots/SpotKind.ts +16 -8
- package/src/contracts/Spots/ZLinkSpot.ts +8 -11
- package/src/contracts/Streams/IZLinkSession.ts +0 -1
- package/src/contracts/Streams/IZLinkSessionActor.ts +5 -1
- package/src/contracts/Streams/IZLinkSessionPacketHandler.ts +5 -1
- package/src/internal.ts +5 -4
- package/src/location-store-integration.ts +24 -16
- package/src/nest-integration.ts +40 -25
- package/src/runtime/abort.ts +10 -4
- package/src/runtime/actors/actor-authority-payload-codec.ts +20 -29
- package/src/runtime/actors/actor-authority-publication.ts +92 -77
- package/src/runtime/actors/actor-client.ts +162 -133
- package/src/runtime/actors/actor-context.ts +138 -130
- package/src/runtime/actors/actor-creation.ts +47 -44
- package/src/runtime/actors/actor-execution-context.ts +6 -4
- package/src/runtime/actors/actor-handoff.ts +269 -293
- package/src/runtime/actors/actor-join-deferred-scope.ts +5 -4
- package/src/runtime/actors/actor-lifecycle-snapshot.ts +5 -8
- package/src/runtime/actors/actor-local-native-join.ts +131 -159
- package/src/runtime/actors/actor-mailbox.ts +16 -19
- package/src/runtime/actors/actor-message-follow-context.ts +46 -36
- package/src/runtime/actors/actor-packet-relay-wire.ts +32 -21
- package/src/runtime/actors/actor-remote-joiner.ts +50 -54
- package/src/runtime/actors/actor-remote-wire.ts +9 -13
- package/src/runtime/actors/actor-routed-join-transport.ts +10 -2
- package/src/runtime/actors/actor-runtime-contracts.ts +2 -5
- package/src/runtime/actors/actor-runtime-state.ts +127 -124
- package/src/runtime/actors/actor-transfer-registry.ts +9 -14
- package/src/runtime/actors/actor-transport-delivery-gate.ts +4 -6
- package/src/runtime/actors/bound-session-wire.ts +37 -17
- package/src/runtime/actors/deferred-join-accepted-journal.ts +351 -320
- package/src/runtime/actors/index.ts +43 -86
- package/src/runtime/actors/post-commit-actor-binder.ts +10 -10
- package/src/runtime/actors/post-commit-actor-location.ts +30 -23
- package/src/runtime/actors/spot-actor-dispatch.ts +78 -53
- package/src/runtime/actors/transferred-actor-rollback.ts +18 -13
- package/src/runtime/admission.ts +9 -9
- package/src/runtime/application-jobs/application-ingress-record-owner.ts +5 -15
- package/src/runtime/application-jobs/application-job-queue-handler-start-gate.ts +3 -2
- package/src/runtime/application-jobs/receive-flow-controller.ts +17 -14
- package/src/runtime/backend/contracts/index.ts +73 -52
- package/src/runtime/backend/mesh-actor-session-node-adapter.ts +4 -14
- package/src/runtime/backend/mesh-completion-table.ts +6 -9
- package/src/runtime/backend/mesh-dispatch-pump.ts +38 -30
- package/src/runtime/backend/node/node-backend-adapter-factory.ts +23 -21
- package/src/runtime/backend/node/node-backend-adapter-support.ts +36 -22
- package/src/runtime/backend/node/node-dealer-channel-client-transport.ts +59 -58
- package/src/runtime/backend/node/node-event-loop-poller.ts +2 -10
- package/src/runtime/backend/node/node-mesh-backend-adapter.ts +12 -5
- package/src/runtime/backend/node/node-monitor-backend-adapter.ts +4 -3
- package/src/runtime/backend/node/node-raw-binding-port.ts +29 -29
- package/src/runtime/backend/node/node-raw-mesh-backend.ts +335 -383
- package/src/runtime/backend/node/node-socket-backend-adapter.ts +101 -54
- package/src/runtime/backend/runtime-message.ts +5 -3
- package/src/runtime/backend/runtime-values.ts +10 -7
- package/src/runtime/channels/channel-autoconnect.ts +1 -4
- package/src/runtime/channels/channel-clients.ts +93 -73
- package/src/runtime/channels/channel-dispatch-pipeline.ts +40 -42
- package/src/runtime/channels/channel-dispatch-services.ts +51 -49
- package/src/runtime/channels/channel-dispatchers.ts +152 -96
- package/src/runtime/channels/channel-envelope-inspection.ts +20 -16
- package/src/runtime/channels/channel-envelope.ts +256 -150
- package/src/runtime/channels/channel-multipart.ts +13 -5
- package/src/runtime/channels/channel-outbound-operations.ts +37 -41
- package/src/runtime/channels/channel-receive-loops.ts +102 -98
- package/src/runtime/channels/channel-receive-task-tracker.ts +1 -1
- package/src/runtime/channels/channel-runtime-lifecycle.ts +248 -190
- package/src/runtime/channels/channel-runtime-manager.ts +63 -37
- package/src/runtime/channels/channel-socket-options.ts +3 -10
- package/src/runtime/channels/channel-socket-registry.ts +331 -273
- package/src/runtime/channels/channel-transports.ts +215 -197
- package/src/runtime/channels/client-server-location-runtime.ts +116 -100
- package/src/runtime/channels/client-server-service-wire.ts +37 -36
- package/src/runtime/channels/dispatch-error-reporter.ts +11 -8
- package/src/runtime/channels/fanout-location-runtime.ts +91 -94
- package/src/runtime/channels/fanout-service-wire.ts +5 -12
- package/src/runtime/channels/handler-filter-scope.ts +1 -4
- package/src/runtime/channels/index.ts +1 -4
- package/src/runtime/channels/local-spot-route-transport.ts +20 -5
- package/src/runtime/channels/route-member-snapshot.ts +11 -5
- package/src/runtime/channels/router-socket-spot-route-transport.ts +1 -7
- package/src/runtime/channels/source-spot-router.ts +26 -22
- package/src/runtime/channels/spot-direct-envelope.ts +10 -8
- package/src/runtime/channels/spot-node-route-transport.ts +13 -16
- package/src/runtime/channels/spot-route-bridge-transport.ts +26 -9
- package/src/runtime/channels/spot-route-dispatch-strategy.ts +40 -13
- package/src/runtime/channels/spot-route-target-resolver.ts +18 -9
- package/src/runtime/codecs/index.ts +3 -2
- package/src/runtime/diagnostics/flow-context.ts +12 -3
- package/src/runtime/diagnostics/index.ts +96 -61
- package/src/runtime/diagnostics/internal-event-contracts.ts +87 -33
- package/src/runtime/diagnostics/message-flow.ts +95 -76
- package/src/runtime/diagnostics/runtime-metrics.ts +46 -24
- package/src/runtime/diagnostics/runtime-observation-queue.ts +17 -28
- package/src/runtime/diagnostics/topology-runtime-projections.ts +62 -43
- package/src/runtime/execution/index.ts +18 -26
- package/src/runtime/execution/serial-execution-queue.ts +31 -40
- package/src/runtime/execution/state-lane.ts +2 -2
- package/src/runtime/foundation/actor-join-recovery-codec.ts +69 -51
- package/src/runtime/foundation/event-loop-resources.ts +8 -4
- package/src/runtime/foundation/message-follow-suppression-registry.ts +7 -14
- package/src/runtime/foundation/operation-identity.ts +1 -2
- package/src/runtime/foundation/operation-registry.ts +8 -2
- package/src/runtime/foundation/raw-service-mesh-runtime.ts +312 -385
- package/src/runtime/foundation/route-mesh-connection-policy.ts +7 -8
- package/src/runtime/foundation/runtime-state-projections.ts +24 -23
- package/src/runtime/foundation/service-authority-payload-codec.ts +54 -57
- package/src/runtime/foundation/service-discovery-registry.ts +67 -59
- package/src/runtime/foundation/service-instance-activation-recovery-codec.ts +28 -29
- package/src/runtime/foundation/service-liveness-registry.ts +18 -13
- package/src/runtime/foundation/service-mailbox.ts +14 -14
- package/src/runtime/foundation/service-maintenance-runtime.ts +22 -21
- package/src/runtime/foundation/service-metadata-codec.ts +2 -9
- package/src/runtime/foundation/service-relocation-aggregate-committer.ts +39 -36
- package/src/runtime/foundation/service-relocation-coordinator.ts +1 -4
- package/src/runtime/foundation/service-relocation-object-owner.ts +40 -58
- package/src/runtime/foundation/service-relocation-runtime.ts +292 -216
- package/src/runtime/foundation/service-runtime-contracts.ts +8 -8
- package/src/runtime/foundation/service-session-binding-ingress-port.ts +1 -4
- package/src/runtime/foundation/service-stateful-registry.ts +69 -59
- package/src/runtime/foundation/service-stateful-runtime.ts +1174 -1112
- package/src/runtime/foundation/service-stateful-wire-codec.ts +318 -270
- package/src/runtime/foundation/service-topology-registry.ts +157 -159
- package/src/runtime/foundation/service-weighted-selection.ts +9 -15
- package/src/runtime/foundation/service-wire-binary-primitives.ts +6 -12
- package/src/runtime/foundation/service-wire-m6a-codec.ts +69 -47
- package/src/runtime/framework-errors-internal.ts +31 -30
- package/src/runtime/handlers/handler-instance-scope.ts +18 -29
- package/src/runtime/handlers/index.ts +18 -8
- package/src/runtime/host/actor-packet-relay.ts +290 -261
- package/src/runtime/host/actor-placement-coordinator.ts +165 -150
- package/src/runtime/host/actor-runtime-options-factory.ts +89 -87
- package/src/runtime/host/actor-transfer-authority-runtime.ts +46 -38
- package/src/runtime/host/actor-transfer-runtime.ts +391 -395
- package/src/runtime/host/application-job-queue.ts +32 -48
- package/src/runtime/host/bound-session-relay.ts +7 -9
- package/src/runtime/host/channel-runtime-options-factory.ts +75 -35
- package/src/runtime/host/host-capacity-status.ts +10 -9
- package/src/runtime/host/index.ts +861 -789
- package/src/runtime/host/instance-activation-authority.ts +125 -143
- package/src/runtime/host/location-runtime-owner.ts +38 -26
- package/src/runtime/host/mesh-router-resolver.ts +30 -23
- package/src/runtime/host/relocation-direct-transfer.ts +15 -15
- package/src/runtime/host/relocation-integrity-fault-gate.ts +3 -2
- package/src/runtime/host/remote-actor-join-receiver.ts +43 -34
- package/src/runtime/host/remote-actor-packet-target-store.ts +54 -52
- package/src/runtime/host/remote-bound-session-relay.ts +274 -247
- package/src/runtime/host/route-mesh-runtime-options.ts +1 -3
- package/src/runtime/host/route-mesh-runtime.ts +227 -181
- package/src/runtime/host/runtime-shutdown.ts +4 -3
- package/src/runtime/host/service-relocation-control.ts +14 -5
- package/src/runtime/host/service-relocation-host-runtime.ts +1388 -1165
- package/src/runtime/host/spot-address-transport.ts +284 -237
- package/src/runtime/host/spot-node-runtime-options-factory.ts +14 -8
- package/src/runtime/host/spot-runtime-options-factory.ts +71 -62
- package/src/runtime/host/stateful-authority-route-runtime.ts +117 -116
- package/src/runtime/host/user-spot-creation-coordinator.ts +225 -220
- package/src/runtime/locations/actor-location-claims.ts +61 -49
- package/src/runtime/locations/actor-session-route-claims.ts +1 -4
- package/src/runtime/locations/aggregate-inventory-store.ts +105 -134
- package/src/runtime/locations/authority-key-codec.ts +13 -10
- package/src/runtime/locations/auto-connect-loop.ts +14 -11
- package/src/runtime/locations/auto-connect-planner.ts +42 -31
- package/src/runtime/locations/auto-connect-reconciler.ts +106 -63
- package/src/runtime/locations/canonical-codec.ts +3 -1
- package/src/runtime/locations/creation-operation-id.ts +4 -3
- package/src/runtime/locations/domain-store-contract.ts +13 -7
- package/src/runtime/locations/in-memory-authority-store.ts +232 -199
- package/src/runtime/locations/in-memory-location-store.ts +432 -250
- package/src/runtime/locations/in-memory-provider-location-store.ts +16 -17
- package/src/runtime/locations/internal-store-contracts.ts +95 -38
- package/src/runtime/locations/key-codec.ts +3 -4
- package/src/runtime/locations/lease-tracker.ts +36 -27
- package/src/runtime/locations/lifecycle-runtime.ts +15 -3
- package/src/runtime/locations/lifecycle.ts +33 -28
- package/src/runtime/locations/location-store-repository.ts +1329 -1185
- package/src/runtime/locations/opaque-record-key.ts +4 -9
- package/src/runtime/locations/resolvers.ts +201 -178
- package/src/runtime/locations/runtime.ts +251 -159
- package/src/runtime/locations/spot-location-claims.ts +110 -112
- package/src/runtime/messaging/creation-payload-codec.ts +4 -1
- package/src/runtime/messaging/framework-json-v1.ts +34 -15
- package/src/runtime/messaging/packet-name.ts +8 -6
- package/src/runtime/messaging/payload-codec.ts +70 -50
- package/src/runtime/messaging/submission-result.ts +6 -9
- package/src/runtime/protocol/service_wire_codec.generated.ts +1376 -159
- package/src/runtime/protocol/service_wire_pilot_codec.generated.ts +1 -1
- package/src/runtime/routing-id.ts +10 -7
- package/src/runtime/spots/formal-remote-actor-admission-registry.ts +56 -62
- package/src/runtime/spots/formal-remote-actor-transfer-registry.ts +23 -20
- package/src/runtime/spots/index.ts +593 -563
- package/src/runtime/spots/spot-activation-registry.ts +38 -34
- package/src/runtime/spots/spot-activation-state.ts +43 -51
- package/src/runtime/spots/spot-activation.ts +179 -116
- package/src/runtime/spots/spot-actor-admission-coordinator.ts +110 -78
- package/src/runtime/spots/spot-actor-join-dispatch.ts +57 -41
- package/src/runtime/spots/spot-actor-lifecycle-drain.ts +5 -7
- package/src/runtime/spots/spot-actor-membership.ts +25 -27
- package/src/runtime/spots/spot-actor-packet-dispatch.ts +184 -153
- package/src/runtime/spots/spot-actor-packet-drain.ts +54 -42
- package/src/runtime/spots/spot-actor-packet-relay-dispatch.ts +37 -39
- package/src/runtime/spots/spot-closing.ts +11 -12
- package/src/runtime/spots/spot-context.ts +46 -26
- package/src/runtime/spots/spot-entry-activation.ts +84 -65
- package/src/runtime/spots/spot-handle.ts +17 -15
- package/src/runtime/spots/spot-handler-registry.ts +44 -31
- package/src/runtime/spots/spot-location-claim.ts +5 -4
- package/src/runtime/spots/spot-manager-public.ts +62 -82
- package/src/runtime/spots/spot-native-actor-join-admission.ts +20 -16
- package/src/runtime/spots/spot-node-autoconnect.ts +54 -61
- package/src/runtime/spots/spot-node-runtime-manager.ts +230 -221
- package/src/runtime/spots/spot-outbound.ts +47 -59
- package/src/runtime/spots/spot-publisher-transport.ts +5 -4
- package/src/runtime/spots/spot-remote-codec.ts +66 -65
- package/src/runtime/spots/spot-remote-route-codec.ts +61 -41
- package/src/runtime/spots/spot-route-packet-dispatch.ts +128 -95
- package/src/runtime/spots/spot-route-replies.ts +23 -24
- package/src/runtime/spots/spot-route-reply-wire.ts +3 -2
- package/src/runtime/spots/spot-routed-actor-admission.ts +37 -27
- package/src/runtime/spots/spot-routed-bound-session-dispatch.ts +11 -17
- package/src/runtime/spots/spot-routed-frame-dispatch.ts +18 -15
- package/src/runtime/spots/spot-routed-spot-packet-dispatch.ts +35 -26
- package/src/runtime/spots/spot-routing-internal.ts +1 -4
- package/src/runtime/spots/spot-runtime-ports.ts +5 -5
- package/src/runtime/spots/spot-serial-executor.ts +5 -9
- package/src/runtime/spots/spot-serial-turn-executor.ts +19 -26
- package/src/runtime/spots/spot-subscription-dispatch.ts +56 -46
- package/src/runtime/spots/spot-timer.ts +98 -91
- package/src/runtime/streams/actor-session-binding-registry.ts +277 -235
- package/src/runtime/streams/actor-session-binding-runtime-owner.ts +7 -4
- package/src/runtime/streams/actor-session-lifecycle-coordinator.ts +3 -1
- package/src/runtime/streams/bound-actor-relay-sender.ts +53 -51
- package/src/runtime/streams/bound-session-response-target.ts +29 -8
- package/src/runtime/streams/bound-session-service.ts +39 -39
- package/src/runtime/streams/index.ts +176 -100
- package/src/runtime/streams/managed-stream.ts +98 -87
- package/src/runtime/streams/native-fallback-bound-session.ts +57 -52
- package/src/runtime/streams/protocol.ts +73 -43
- package/src/runtime/streams/session-actor-coordinator.ts +97 -89
- package/src/runtime/streams/session-calls.ts +2 -8
- package/src/runtime/streams/session-context.ts +66 -48
- package/src/runtime/streams/session-provider.ts +5 -7
- package/src/runtime/streams/session-serial-executor.ts +7 -6
- package/src/runtime/streams/stream-binding-runtime-ports.ts +79 -29
- package/src/runtime/streams/stream-frame-factory.ts +55 -38
- package/src/runtime/streams/stream-session-runtime.ts +268 -210
- package/src/runtime/workers/index.ts +68 -29
|
@@ -8,11 +8,7 @@ import {
|
|
|
8
8
|
internalFrameworkWireReply,
|
|
9
9
|
translateWireReplyDecodeError
|
|
10
10
|
} from '../framework-errors-internal';
|
|
11
|
-
import {
|
|
12
|
-
RequestResult,
|
|
13
|
-
SubmitResult,
|
|
14
|
-
isZLinkBackendResultError
|
|
15
|
-
} from '../backend/runtime-values';
|
|
11
|
+
import { RequestResult, SubmitResult, isZLinkBackendResultError } from '../backend/runtime-values';
|
|
16
12
|
import type {
|
|
17
13
|
RawServiceIngressRecord,
|
|
18
14
|
RawServiceMeshRuntime,
|
|
@@ -84,10 +80,7 @@ import {
|
|
|
84
80
|
encodeActorJoin28,
|
|
85
81
|
type ActorJoin28
|
|
86
82
|
} from '../protocol/service_wire_pilot_codec.generated';
|
|
87
|
-
import {
|
|
88
|
-
canonicalActorJoinHandoffId,
|
|
89
|
-
routingIdBytes
|
|
90
|
-
} from './actor-join-recovery-codec';
|
|
83
|
+
import { canonicalActorJoinHandoffId, routingIdBytes } from './actor-join-recovery-codec';
|
|
91
84
|
|
|
92
85
|
import {
|
|
93
86
|
decodeApplicationPayload,
|
|
@@ -95,14 +88,10 @@ import {
|
|
|
95
88
|
type ServiceApplicationPayload,
|
|
96
89
|
ServiceWireProtocolError
|
|
97
90
|
} from './service-wire-m6a-codec';
|
|
98
|
-
import type {
|
|
99
|
-
ServiceInstanceActivationRecoveryEnvelope
|
|
100
|
-
} from './service-instance-activation-recovery-codec';
|
|
91
|
+
import type { ServiceInstanceActivationRecoveryEnvelope } from './service-instance-activation-recovery-codec';
|
|
101
92
|
import { validateServiceMetadataFrame } from './service-metadata-codec';
|
|
102
93
|
import type { ServiceSessionBindingIngressPort } from './service-session-binding-ingress-port';
|
|
103
|
-
import {
|
|
104
|
-
ZLinkFrameworkException
|
|
105
|
-
} from '../../contracts';
|
|
94
|
+
import { ZLinkFrameworkException } from '../../contracts';
|
|
106
95
|
|
|
107
96
|
const ACTOR_ROUTE_STALE = 21;
|
|
108
97
|
const SPOT_MOVING = 34;
|
|
@@ -123,10 +112,7 @@ export interface ServiceBoundSessionSendFence {
|
|
|
123
112
|
|
|
124
113
|
interface ServiceSpotMessageFollowRecord {
|
|
125
114
|
readonly ingress: RawServiceIngressRecord;
|
|
126
|
-
readonly wire: Extract<
|
|
127
|
-
ServiceStatefulWireRecord,
|
|
128
|
-
{ readonly kind: 'spotSend' | 'spotRequest' }
|
|
129
|
-
>;
|
|
115
|
+
readonly wire: Extract<ServiceStatefulWireRecord, { readonly kind: 'spotSend' | 'spotRequest' }>;
|
|
130
116
|
readonly bytes: number;
|
|
131
117
|
}
|
|
132
118
|
|
|
@@ -294,7 +280,10 @@ export interface ServiceInstanceActivationAuthority {
|
|
|
294
280
|
reservation: ServiceInstanceActivationReservation,
|
|
295
281
|
spot: ServiceSpotState
|
|
296
282
|
): { readonly kind: 'committed' | 'lost'; readonly route: ServiceInstanceRouteFence };
|
|
297
|
-
abort(
|
|
283
|
+
abort(
|
|
284
|
+
target: ServiceInstanceActivationTarget,
|
|
285
|
+
reservation: ServiceInstanceActivationReservation
|
|
286
|
+
): void;
|
|
298
287
|
}
|
|
299
288
|
|
|
300
289
|
/**
|
|
@@ -344,9 +333,18 @@ export interface ServiceUserSpotOperationResult {
|
|
|
344
333
|
}
|
|
345
334
|
|
|
346
335
|
export interface ServiceUserSpotOperationHandler {
|
|
347
|
-
create(
|
|
348
|
-
|
|
349
|
-
|
|
336
|
+
create(
|
|
337
|
+
record: ServiceUserSpotCreateRecord,
|
|
338
|
+
signal: AbortSignal
|
|
339
|
+
): Promise<ServiceUserSpotOperationResult>;
|
|
340
|
+
close(
|
|
341
|
+
record: ServiceUserSpotCloseRecord,
|
|
342
|
+
signal: AbortSignal
|
|
343
|
+
): Promise<ServiceUserSpotOperationResult>;
|
|
344
|
+
createActor?(
|
|
345
|
+
record: ServiceActorCreateRecord,
|
|
346
|
+
signal: AbortSignal
|
|
347
|
+
): Promise<ServiceUserSpotOperationResult>;
|
|
350
348
|
}
|
|
351
349
|
|
|
352
350
|
/**
|
|
@@ -403,16 +401,22 @@ export class ServiceStatefulRuntime {
|
|
|
403
401
|
readonly kind: 'spot_multicast' | 'actor_control' | 'actor_binding';
|
|
404
402
|
readonly owner: string;
|
|
405
403
|
}) => void;
|
|
406
|
-
private readonly admittedUserSpotOperations = new Map<
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
404
|
+
private readonly admittedUserSpotOperations = new Map<
|
|
405
|
+
string,
|
|
406
|
+
{
|
|
407
|
+
readonly request: string;
|
|
408
|
+
readonly replayExpiresAtMs: number;
|
|
409
|
+
readonly result: Promise<ServiceUserSpotOperationResult>;
|
|
410
|
+
settled: boolean;
|
|
411
|
+
}
|
|
412
|
+
>();
|
|
413
|
+
private readonly pendingInstanceActivations = new Map<
|
|
414
|
+
string,
|
|
415
|
+
Promise<{
|
|
416
|
+
readonly spot: ServiceSpotState;
|
|
417
|
+
readonly route: ServiceInstanceRouteFence;
|
|
418
|
+
}>
|
|
419
|
+
>();
|
|
416
420
|
private closed = false;
|
|
417
421
|
|
|
418
422
|
constructor(
|
|
@@ -422,17 +426,23 @@ export class ServiceStatefulRuntime {
|
|
|
422
426
|
) {
|
|
423
427
|
this.registry = new ServiceStatefulRegistry(nodeRid, nodeGeneration);
|
|
424
428
|
this.registry.createEntrySpot(nodeRid);
|
|
425
|
-
raw.setServiceIngress(record => this.ingress(record));
|
|
429
|
+
raw.setServiceIngress((record) => this.ingress(record));
|
|
426
430
|
}
|
|
427
431
|
|
|
428
|
-
setMailboxDropHandler(
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
+
setMailboxDropHandler(
|
|
433
|
+
handler: (record: {
|
|
434
|
+
readonly kind: 'spot_multicast' | 'actor_control' | 'actor_binding';
|
|
435
|
+
readonly owner: string;
|
|
436
|
+
}) => void
|
|
437
|
+
): void {
|
|
432
438
|
this.mailboxDropHandler = handler;
|
|
433
439
|
}
|
|
434
440
|
|
|
435
|
-
createSpot(
|
|
441
|
+
createSpot(
|
|
442
|
+
routingId?: string,
|
|
443
|
+
kind: 'user' | 'instance' = 'user',
|
|
444
|
+
stableType = kind
|
|
445
|
+
): ServiceSpotState {
|
|
436
446
|
this.requireOpen();
|
|
437
447
|
return this.registry.createSpot(
|
|
438
448
|
routingId ?? `spot-${this.nodeRid}-${this.nextSpotId++}`,
|
|
@@ -514,17 +524,17 @@ export class ServiceStatefulRuntime {
|
|
|
514
524
|
this.requireOpen();
|
|
515
525
|
const actor = this.registry.actor(actorRef.actorId);
|
|
516
526
|
if (
|
|
517
|
-
actor === undefined
|
|
518
|
-
|
|
519
|
-
|
|
527
|
+
actor === undefined ||
|
|
528
|
+
actor.ref.generation !== actorRef.generation ||
|
|
529
|
+
actor.ref.nodeRid !== actorRef.nodeRid
|
|
520
530
|
) {
|
|
521
531
|
throw new ServiceStaleGenerationError('actor', actorRef.actorId);
|
|
522
532
|
}
|
|
523
533
|
const current = this.registry.binding(actor.ref);
|
|
524
534
|
if (
|
|
525
|
-
current?.bindingGeneration === bindingGeneration
|
|
526
|
-
|
|
527
|
-
|
|
535
|
+
current?.bindingGeneration === bindingGeneration &&
|
|
536
|
+
current.sessionOwnerNodeRid === sessionNodeRid &&
|
|
537
|
+
current.sessionRid === sessionRid
|
|
528
538
|
) {
|
|
529
539
|
return;
|
|
530
540
|
}
|
|
@@ -554,14 +564,15 @@ export class ServiceStatefulRuntime {
|
|
|
554
564
|
}
|
|
555
565
|
if (current.ref.generation === attempt) {
|
|
556
566
|
return {
|
|
557
|
-
spot:
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
567
|
+
spot:
|
|
568
|
+
current.authorityOwnerGeneration === authorityOwnerGeneration
|
|
569
|
+
? current
|
|
570
|
+
: this.registry.restoreSpot(
|
|
571
|
+
current.ref,
|
|
572
|
+
'instance',
|
|
573
|
+
instanceType,
|
|
574
|
+
authorityOwnerGeneration
|
|
575
|
+
),
|
|
565
576
|
created: false
|
|
566
577
|
};
|
|
567
578
|
}
|
|
@@ -585,7 +596,10 @@ export class ServiceStatefulRuntime {
|
|
|
585
596
|
route: ServiceInstanceRouteFence,
|
|
586
597
|
expectedCurrentRoute?: ServiceInstanceRouteFence | null
|
|
587
598
|
): void {
|
|
588
|
-
if (
|
|
599
|
+
if (
|
|
600
|
+
route.targetNodeRid !== this.nodeRid ||
|
|
601
|
+
route.targetNodeGeneration !== this.nodeGeneration
|
|
602
|
+
) {
|
|
589
603
|
throw new ServiceStaleGenerationError('spot', route.targetSpotId);
|
|
590
604
|
}
|
|
591
605
|
const current = this.instanceIntents.get(route.targetSpotId);
|
|
@@ -595,7 +609,10 @@ export class ServiceStatefulRuntime {
|
|
|
595
609
|
}
|
|
596
610
|
if (!matchesExpectedInstanceRoute(current.route, expectedCurrentRoute)) return;
|
|
597
611
|
if (current.route.objectGeneration < route.objectGeneration) {
|
|
598
|
-
this.instanceIntents.set(
|
|
612
|
+
this.instanceIntents.set(
|
|
613
|
+
route.targetSpotId,
|
|
614
|
+
Object.freeze({ instanceType, route: { ...route } })
|
|
615
|
+
);
|
|
599
616
|
return;
|
|
600
617
|
}
|
|
601
618
|
if (current.route.objectGeneration > route.objectGeneration) {
|
|
@@ -615,7 +632,10 @@ export class ServiceStatefulRuntime {
|
|
|
615
632
|
} else if (!matchesExpectedInstanceRoute(undefined, expectedCurrentRoute)) {
|
|
616
633
|
return;
|
|
617
634
|
}
|
|
618
|
-
this.instanceIntents.set(
|
|
635
|
+
this.instanceIntents.set(
|
|
636
|
+
route.targetSpotId,
|
|
637
|
+
Object.freeze({ instanceType, route: { ...route } })
|
|
638
|
+
);
|
|
619
639
|
}
|
|
620
640
|
|
|
621
641
|
instanceSpotApplicationTarget(
|
|
@@ -624,10 +644,10 @@ export class ServiceStatefulRuntime {
|
|
|
624
644
|
const intent = this.instanceIntents.get(spotId);
|
|
625
645
|
const spot = this.registry.spot(spotId);
|
|
626
646
|
if (
|
|
627
|
-
intent === undefined
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
647
|
+
intent === undefined ||
|
|
648
|
+
spot?.kind !== 'instance' ||
|
|
649
|
+
spot.state !== 'ready' ||
|
|
650
|
+
spot.ref.generation !== intent.route.objectGeneration
|
|
631
651
|
) {
|
|
632
652
|
return undefined;
|
|
633
653
|
}
|
|
@@ -645,9 +665,9 @@ export class ServiceStatefulRuntime {
|
|
|
645
665
|
): void {
|
|
646
666
|
const current = this.instanceIntents.get(spotId);
|
|
647
667
|
if (
|
|
648
|
-
current?.route.objectGeneration === objectGeneration
|
|
649
|
-
|
|
650
|
-
|
|
668
|
+
current?.route.objectGeneration === objectGeneration &&
|
|
669
|
+
current.route.authorityOwnerGeneration === authorityOwnerGeneration &&
|
|
670
|
+
(storeVersion === undefined || current.route.storeVersion === storeVersion)
|
|
651
671
|
) {
|
|
652
672
|
this.instanceIntents.delete(spotId);
|
|
653
673
|
}
|
|
@@ -656,8 +676,8 @@ export class ServiceStatefulRuntime {
|
|
|
656
676
|
registerInstanceActivationAuthority(authority: ServiceInstanceActivationAuthority): void {
|
|
657
677
|
this.requireOpen();
|
|
658
678
|
if (
|
|
659
|
-
this.asyncInstanceAuthority !== undefined
|
|
660
|
-
|
|
679
|
+
this.asyncInstanceAuthority !== undefined ||
|
|
680
|
+
(this.instanceAuthority !== undefined && this.instanceAuthority !== authority)
|
|
661
681
|
) {
|
|
662
682
|
throw new Error('Instance activation authority is already registered.');
|
|
663
683
|
}
|
|
@@ -669,18 +689,15 @@ export class ServiceStatefulRuntime {
|
|
|
669
689
|
): void {
|
|
670
690
|
this.requireOpen();
|
|
671
691
|
if (
|
|
672
|
-
this.instanceAuthority !== undefined
|
|
673
|
-
|
|
674
|
-
&& this.asyncInstanceAuthority !== authority
|
|
692
|
+
this.instanceAuthority !== undefined ||
|
|
693
|
+
(this.asyncInstanceAuthority !== undefined && this.asyncInstanceAuthority !== authority)
|
|
675
694
|
) {
|
|
676
695
|
throw new Error('Instance activation authority is already registered.');
|
|
677
696
|
}
|
|
678
697
|
this.asyncInstanceAuthority = authority;
|
|
679
698
|
}
|
|
680
699
|
|
|
681
|
-
registerInstanceApplicationLifecycle(
|
|
682
|
-
lifecycle: ServiceInstanceApplicationLifecycle
|
|
683
|
-
): void {
|
|
700
|
+
registerInstanceApplicationLifecycle(lifecycle: ServiceInstanceApplicationLifecycle): void {
|
|
684
701
|
if (this.instanceApplicationLifecycle !== undefined) {
|
|
685
702
|
throw new Error('Instance application lifecycle is already registered.');
|
|
686
703
|
}
|
|
@@ -689,10 +706,7 @@ export class ServiceStatefulRuntime {
|
|
|
689
706
|
|
|
690
707
|
registerUserSpotOperationHandler(handler: ServiceUserSpotOperationHandler): void {
|
|
691
708
|
this.requireOpen();
|
|
692
|
-
if (
|
|
693
|
-
this.userSpotOperationHandler !== undefined
|
|
694
|
-
&& this.userSpotOperationHandler !== handler
|
|
695
|
-
) {
|
|
709
|
+
if (this.userSpotOperationHandler !== undefined && this.userSpotOperationHandler !== handler) {
|
|
696
710
|
throw new Error('User Spot operation handler is already registered.');
|
|
697
711
|
}
|
|
698
712
|
this.userSpotOperationHandler = handler;
|
|
@@ -775,16 +789,19 @@ export class ServiceStatefulRuntime {
|
|
|
775
789
|
expectedCurrentRoute?: ServiceInstanceRouteFence | null
|
|
776
790
|
): Promise<boolean> {
|
|
777
791
|
const target = envelope.target;
|
|
778
|
-
if (!matchesExpectedInstanceRoute(
|
|
779
|
-
this.instanceIntents.get(target.targetSpotId)?.route,
|
|
780
|
-
expectedCurrentRoute
|
|
781
|
-
)) return false;
|
|
782
792
|
if (
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
793
|
+
!matchesExpectedInstanceRoute(
|
|
794
|
+
this.instanceIntents.get(target.targetSpotId)?.route,
|
|
795
|
+
expectedCurrentRoute
|
|
796
|
+
)
|
|
797
|
+
)
|
|
798
|
+
return false;
|
|
799
|
+
if (
|
|
800
|
+
target.targetNodeRid !== this.nodeRid ||
|
|
801
|
+
target.targetNodeGeneration !== this.nodeGeneration ||
|
|
802
|
+
target.targetSpotId !== route.targetSpotId ||
|
|
803
|
+
route.targetNodeRid !== this.nodeRid ||
|
|
804
|
+
route.targetNodeGeneration !== this.nodeGeneration
|
|
788
805
|
) {
|
|
789
806
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
790
807
|
}
|
|
@@ -802,8 +819,9 @@ export class ServiceStatefulRuntime {
|
|
|
802
819
|
expectedCurrentRoute?: ServiceInstanceRouteFence | null
|
|
803
820
|
): Promise<boolean> {
|
|
804
821
|
const target = envelope.target;
|
|
805
|
-
const spot =
|
|
806
|
-
|
|
822
|
+
const spot =
|
|
823
|
+
this.registry.spot(target.targetSpotId) ??
|
|
824
|
+
this.registry.restoreSpot(
|
|
807
825
|
{
|
|
808
826
|
spotId: target.targetSpotId,
|
|
809
827
|
generation: route.objectGeneration
|
|
@@ -813,17 +831,20 @@ export class ServiceStatefulRuntime {
|
|
|
813
831
|
route.authorityOwnerGeneration
|
|
814
832
|
);
|
|
815
833
|
if (
|
|
816
|
-
spot.kind !== 'instance'
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
834
|
+
spot.kind !== 'instance' ||
|
|
835
|
+
spot.stableType !== target.stableType ||
|
|
836
|
+
spot.ref.generation !== route.objectGeneration ||
|
|
837
|
+
spot.authorityOwnerGeneration !== route.authorityOwnerGeneration
|
|
820
838
|
) {
|
|
821
839
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
822
840
|
}
|
|
823
|
-
if (
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
841
|
+
if (
|
|
842
|
+
!matchesExpectedInstanceRoute(
|
|
843
|
+
this.instanceIntents.get(target.targetSpotId)?.route,
|
|
844
|
+
expectedCurrentRoute
|
|
845
|
+
)
|
|
846
|
+
)
|
|
847
|
+
return false;
|
|
827
848
|
this.publishCommittedInstanceRoute(target.stableType, route, expectedCurrentRoute);
|
|
828
849
|
const record = {
|
|
829
850
|
kind: 'instanceSpot' as const,
|
|
@@ -831,9 +852,7 @@ export class ServiceStatefulRuntime {
|
|
|
831
852
|
target,
|
|
832
853
|
sourceNodeGeneration: envelope.sourceNodeGeneration,
|
|
833
854
|
sourceNodeRid: envelope.sourceNodeRid,
|
|
834
|
-
...(envelope.sourceSpotId === undefined
|
|
835
|
-
? {}
|
|
836
|
-
: { sourceSpotId: envelope.sourceSpotId }),
|
|
855
|
+
...(envelope.sourceSpotId === undefined ? {} : { sourceSpotId: envelope.sourceSpotId }),
|
|
837
856
|
operationKind: envelope.operationKind,
|
|
838
857
|
operation: envelope.operation,
|
|
839
858
|
deadlineUnixMs: envelope.deadlineUnixMs,
|
|
@@ -843,23 +862,23 @@ export class ServiceStatefulRuntime {
|
|
|
843
862
|
try {
|
|
844
863
|
const admitted = this.enqueueActivatedInstanceSpot(
|
|
845
864
|
{
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
865
|
+
command: M6bServiceWireCommand.instanceSpot,
|
|
866
|
+
flags: envelope.metadataFrame === undefined ? 0 : M6bServiceWireFlag.metadata,
|
|
867
|
+
sourceRoutingId: envelope.sourceNodeRid,
|
|
868
|
+
parts: [
|
|
869
|
+
encodeInstanceSpotActivationHeader(
|
|
870
|
+
target,
|
|
871
|
+
envelope.sourceNodeGeneration,
|
|
872
|
+
envelope.sourceNodeRid,
|
|
873
|
+
envelope.sourceSpotId,
|
|
874
|
+
envelope.operationKind,
|
|
875
|
+
envelope.operation,
|
|
876
|
+
envelope.deadlineUnixMs,
|
|
877
|
+
envelope.replyRouteId,
|
|
878
|
+
envelope.metadataFrame !== undefined
|
|
879
|
+
)
|
|
880
|
+
],
|
|
881
|
+
applicationJobOwner
|
|
863
882
|
},
|
|
864
883
|
record,
|
|
865
884
|
Buffer.from(envelope.applicationPayloadFrame),
|
|
@@ -887,16 +906,19 @@ export class ServiceStatefulRuntime {
|
|
|
887
906
|
expectedCurrentRoute?: ServiceInstanceRouteFence | null
|
|
888
907
|
): Promise<boolean> {
|
|
889
908
|
const target = envelope.target;
|
|
890
|
-
if (!matchesExpectedInstanceRoute(
|
|
891
|
-
this.instanceIntents.get(target.targetSpotId)?.route,
|
|
892
|
-
expectedCurrentRoute
|
|
893
|
-
)) return false;
|
|
894
909
|
if (
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
910
|
+
!matchesExpectedInstanceRoute(
|
|
911
|
+
this.instanceIntents.get(target.targetSpotId)?.route,
|
|
912
|
+
expectedCurrentRoute
|
|
913
|
+
)
|
|
914
|
+
)
|
|
915
|
+
return false;
|
|
916
|
+
if (
|
|
917
|
+
target.targetNodeRid !== this.nodeRid ||
|
|
918
|
+
target.targetNodeGeneration !== this.nodeGeneration ||
|
|
919
|
+
pending.nodeRid !== this.nodeRid ||
|
|
920
|
+
pending.nodeGeneration !== this.nodeGeneration ||
|
|
921
|
+
pending.meshName !== envelope.targetMeshName
|
|
900
922
|
) {
|
|
901
923
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
902
924
|
}
|
|
@@ -928,10 +950,13 @@ export class ServiceStatefulRuntime {
|
|
|
928
950
|
route: ServiceInstanceRouteFence,
|
|
929
951
|
expectedCurrentRoute?: ServiceInstanceRouteFence | null
|
|
930
952
|
): Promise<ServiceInstanceRouteFence | undefined> {
|
|
931
|
-
if (
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
953
|
+
if (
|
|
954
|
+
!matchesExpectedInstanceRoute(
|
|
955
|
+
this.instanceIntents.get(target.targetSpotId)?.route,
|
|
956
|
+
expectedCurrentRoute
|
|
957
|
+
)
|
|
958
|
+
)
|
|
959
|
+
return undefined;
|
|
935
960
|
const authority = this.asyncInstanceAuthority;
|
|
936
961
|
if (authority === undefined) {
|
|
937
962
|
throw new Error('Async Instance activation authority is not registered.');
|
|
@@ -940,17 +965,17 @@ export class ServiceStatefulRuntime {
|
|
|
940
965
|
const currentRoute = this.instanceIntents.get(target.targetSpotId)?.route;
|
|
941
966
|
if (!matchesExpectedInstanceRoute(currentRoute, expectedCurrentRoute)) return undefined;
|
|
942
967
|
if (currentRoute !== undefined && !sameInstanceRoute(currentRoute, route)) return undefined;
|
|
943
|
-
const authorityReleased =
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
?? false;
|
|
968
|
+
const authorityReleased =
|
|
969
|
+
(await this.instanceApplicationLifecycle?.completeTerminal({
|
|
970
|
+
targetSpotId: target.targetSpotId,
|
|
971
|
+
stableType: target.stableType,
|
|
972
|
+
objectGeneration: route.objectGeneration
|
|
973
|
+
})) ?? false;
|
|
949
974
|
if (!authorityReleased) {
|
|
950
975
|
if (currentRoute === undefined) {
|
|
951
976
|
this.publishCommittedInstanceRoute(target.stableType, released, null);
|
|
952
977
|
} else if (!this.replaceCommittedInstanceRoute(target.stableType, route, released)) {
|
|
953
|
-
|
|
978
|
+
return undefined;
|
|
954
979
|
}
|
|
955
980
|
} else {
|
|
956
981
|
this.forgetClosedInstanceRoute(released);
|
|
@@ -958,14 +983,11 @@ export class ServiceStatefulRuntime {
|
|
|
958
983
|
return released;
|
|
959
984
|
}
|
|
960
985
|
|
|
961
|
-
waitForInstanceApplicationQuiescence(
|
|
962
|
-
spotId: string,
|
|
963
|
-
signal?: AbortSignal
|
|
964
|
-
): Promise<void> {
|
|
986
|
+
waitForInstanceApplicationQuiescence(spotId: string, signal?: AbortSignal): Promise<void> {
|
|
965
987
|
const key = String(spotId);
|
|
966
988
|
if (
|
|
967
|
-
(this.pendingInstanceTerminals.get(key) ?? 0) === 0
|
|
968
|
-
|
|
989
|
+
(this.pendingInstanceTerminals.get(key) ?? 0) === 0 &&
|
|
990
|
+
(this.pendingInstanceAuthorityTerminals.get(key) ?? 0) === 0
|
|
969
991
|
) {
|
|
970
992
|
return Promise.resolve();
|
|
971
993
|
}
|
|
@@ -989,17 +1011,19 @@ export class ServiceStatefulRuntime {
|
|
|
989
1011
|
});
|
|
990
1012
|
}
|
|
991
1013
|
|
|
992
|
-
|
|
993
1014
|
rememberActorRoute(route: ServiceActorRouteFence): void {
|
|
994
1015
|
if (route.actor.nodeRid.length === 0) {
|
|
995
1016
|
throw new TypeError('Actor route requires a target node.');
|
|
996
1017
|
}
|
|
997
|
-
this.actorRoutes.set(
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1018
|
+
this.actorRoutes.set(
|
|
1019
|
+
actorKey(route.actor),
|
|
1020
|
+
Object.freeze({
|
|
1021
|
+
actor: Object.freeze({ ...route.actor }),
|
|
1022
|
+
targetNodeGeneration: route.targetNodeGeneration,
|
|
1023
|
+
authorityOwnerGeneration: route.authorityOwnerGeneration,
|
|
1024
|
+
ownerLeaseGeneration: route.ownerLeaseGeneration
|
|
1025
|
+
})
|
|
1026
|
+
);
|
|
1003
1027
|
}
|
|
1004
1028
|
|
|
1005
1029
|
rememberSpotRoute(
|
|
@@ -1017,9 +1041,10 @@ export class ServiceStatefulRuntime {
|
|
|
1017
1041
|
return;
|
|
1018
1042
|
}
|
|
1019
1043
|
if (
|
|
1020
|
-
expectedCurrentRoute !== undefined
|
|
1021
|
-
|
|
1022
|
-
)
|
|
1044
|
+
expectedCurrentRoute !== undefined &&
|
|
1045
|
+
!matchesExpectedDirectSpotRoute(current, expectedCurrentRoute)
|
|
1046
|
+
)
|
|
1047
|
+
return;
|
|
1023
1048
|
if (current !== undefined) {
|
|
1024
1049
|
if (current.spot.generation < route.spot.generation) {
|
|
1025
1050
|
const next = freezeDirectSpotRoute(route);
|
|
@@ -1061,9 +1086,9 @@ export class ServiceStatefulRuntime {
|
|
|
1061
1086
|
return await this.raw.sendService(targetNodeRid, [encodeMessageFollowHeader(record)]);
|
|
1062
1087
|
}
|
|
1063
1088
|
|
|
1064
|
-
setMessageFollowHandler(
|
|
1065
|
-
record: import('./service-stateful-wire-codec').ServiceMessageFollowRecord
|
|
1066
|
-
)
|
|
1089
|
+
setMessageFollowHandler(
|
|
1090
|
+
handler: (record: import('./service-stateful-wire-codec').ServiceMessageFollowRecord) => void
|
|
1091
|
+
): void {
|
|
1067
1092
|
this.messageFollowHandler = handler;
|
|
1068
1093
|
}
|
|
1069
1094
|
|
|
@@ -1072,8 +1097,8 @@ export class ServiceStatefulRuntime {
|
|
|
1072
1097
|
): ServiceSpotMessageFollowSeal | undefined {
|
|
1073
1098
|
this.requireOpen();
|
|
1074
1099
|
if (
|
|
1075
|
-
source.targetNodeRid !== this.nodeRid
|
|
1076
|
-
|
|
1100
|
+
source.targetNodeRid !== this.nodeRid ||
|
|
1101
|
+
source.targetNodeGeneration !== this.nodeGeneration
|
|
1077
1102
|
) {
|
|
1078
1103
|
return undefined;
|
|
1079
1104
|
}
|
|
@@ -1125,10 +1150,10 @@ export class ServiceStatefulRuntime {
|
|
|
1125
1150
|
throw new RangeError('Message Follow duration must be a non-negative safe integer.');
|
|
1126
1151
|
}
|
|
1127
1152
|
if (
|
|
1128
|
-
target.spot.spotId !== state.source.spot.spotId
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1153
|
+
target.spot.spotId !== state.source.spot.spotId ||
|
|
1154
|
+
target.spot.generation !== state.source.spot.generation ||
|
|
1155
|
+
target.authorityOwnerGeneration <= state.source.authorityOwnerGeneration ||
|
|
1156
|
+
target.targetNodeRid === this.nodeRid
|
|
1132
1157
|
) {
|
|
1133
1158
|
throw new ServiceStaleGenerationError('spot', state.source.spot.spotId);
|
|
1134
1159
|
}
|
|
@@ -1139,9 +1164,7 @@ export class ServiceStatefulRuntime {
|
|
|
1139
1164
|
state.target
|
|
1140
1165
|
);
|
|
1141
1166
|
this.spotMessageFollowSuppression.retainRoute(state.suppressionFence);
|
|
1142
|
-
state.expiresAtMs = durationMs === 0
|
|
1143
|
-
? Number.MAX_SAFE_INTEGER
|
|
1144
|
-
: performance.now() + durationMs;
|
|
1167
|
+
state.expiresAtMs = durationMs === 0 ? Number.MAX_SAFE_INTEGER : performance.now() + durationMs;
|
|
1145
1168
|
await this.drainSpotMessageFollow(state);
|
|
1146
1169
|
if (durationMs === 0) this.removeSpotMessageFollow(state);
|
|
1147
1170
|
return true;
|
|
@@ -1156,16 +1179,16 @@ export class ServiceStatefulRuntime {
|
|
|
1156
1179
|
const exact = this.spotRoutes.get(exactKey);
|
|
1157
1180
|
const current = this.directSpotRoutes.get(directSpotKey(spot.spotId));
|
|
1158
1181
|
if (
|
|
1159
|
-
exact?.authorityOwnerGeneration === authorityOwnerGeneration
|
|
1160
|
-
|
|
1182
|
+
exact?.authorityOwnerGeneration === authorityOwnerGeneration &&
|
|
1183
|
+
(storeVersion === undefined || exact.storeVersion === storeVersion)
|
|
1161
1184
|
) {
|
|
1162
1185
|
this.spotRoutes.delete(exactKey);
|
|
1163
1186
|
}
|
|
1164
1187
|
const key = directSpotKey(spot.spotId);
|
|
1165
1188
|
if (
|
|
1166
|
-
current?.authorityOwnerGeneration === authorityOwnerGeneration
|
|
1167
|
-
|
|
1168
|
-
|
|
1189
|
+
current?.authorityOwnerGeneration === authorityOwnerGeneration &&
|
|
1190
|
+
current.spot.generation === spot.generation &&
|
|
1191
|
+
(storeVersion === undefined || current.storeVersion === storeVersion)
|
|
1169
1192
|
) {
|
|
1170
1193
|
this.directSpotRoutes.delete(key);
|
|
1171
1194
|
}
|
|
@@ -1204,10 +1227,13 @@ export class ServiceStatefulRuntime {
|
|
|
1204
1227
|
sourceSpotId = this.nodeRid
|
|
1205
1228
|
): Promise<void> {
|
|
1206
1229
|
await this.enqueueLogicalMulticast(channelName, topic, sourceSpotId, payload);
|
|
1207
|
-
const targets = this.raw.topology
|
|
1208
|
-
.
|
|
1209
|
-
|
|
1210
|
-
|
|
1230
|
+
const targets = this.raw.topology
|
|
1231
|
+
.peers()
|
|
1232
|
+
.filter((peer) =>
|
|
1233
|
+
peer.descriptor.channels.some(
|
|
1234
|
+
(channel) => channel.name === channelName && channel.weight > 0
|
|
1235
|
+
)
|
|
1236
|
+
);
|
|
1211
1237
|
const header = encodeLogicalMulticastHeader(channelName, topic, sourceSpotId);
|
|
1212
1238
|
const payloadFrame = encodeApplicationPayload(payload);
|
|
1213
1239
|
for (const target of targets) {
|
|
@@ -1327,19 +1353,25 @@ export class ServiceStatefulRuntime {
|
|
|
1327
1353
|
sourceSpotId?: string,
|
|
1328
1354
|
metadataFrame?: Uint8Array
|
|
1329
1355
|
): Promise<number> {
|
|
1330
|
-
return this.submitOneWay(
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1356
|
+
return this.submitOneWay(
|
|
1357
|
+
route.targetNodeRid,
|
|
1358
|
+
instanceOperationParts(
|
|
1359
|
+
[
|
|
1360
|
+
encodeInstanceSpotHeader(
|
|
1361
|
+
route,
|
|
1362
|
+
this.nodeGeneration,
|
|
1363
|
+
this.nodeRid,
|
|
1364
|
+
sourceSpotId,
|
|
1365
|
+
'send',
|
|
1366
|
+
{ high: 0n, low: 0n },
|
|
1367
|
+
undefined,
|
|
1368
|
+
metadataFrame !== undefined
|
|
1369
|
+
),
|
|
1370
|
+
encodeApplicationPayload(payload)
|
|
1371
|
+
],
|
|
1372
|
+
metadataFrame
|
|
1373
|
+
)
|
|
1374
|
+
);
|
|
1343
1375
|
}
|
|
1344
1376
|
|
|
1345
1377
|
async sendToMissingInstanceSpot(
|
|
@@ -1366,20 +1398,26 @@ export class ServiceStatefulRuntime {
|
|
|
1366
1398
|
metadataFrame?: Uint8Array
|
|
1367
1399
|
): Promise<number> {
|
|
1368
1400
|
const operation = { high: this.nodeGeneration, low: this.nextInstanceOperation++ };
|
|
1369
|
-
return await this.submitOneWay(
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1401
|
+
return await this.submitOneWay(
|
|
1402
|
+
target.targetNodeRid,
|
|
1403
|
+
instanceOperationParts(
|
|
1404
|
+
[
|
|
1405
|
+
encodeInstanceSpotActivationHeader(
|
|
1406
|
+
target,
|
|
1407
|
+
this.nodeGeneration,
|
|
1408
|
+
this.nodeRid,
|
|
1409
|
+
sourceSpotId,
|
|
1410
|
+
'send',
|
|
1411
|
+
operation,
|
|
1412
|
+
deadlineUnixMs,
|
|
1413
|
+
undefined,
|
|
1414
|
+
metadataFrame !== undefined
|
|
1415
|
+
),
|
|
1416
|
+
payloadFrame
|
|
1417
|
+
],
|
|
1418
|
+
metadataFrame
|
|
1419
|
+
)
|
|
1420
|
+
);
|
|
1383
1421
|
}
|
|
1384
1422
|
|
|
1385
1423
|
requestToInstanceSpot(
|
|
@@ -1393,19 +1431,22 @@ export class ServiceStatefulRuntime {
|
|
|
1393
1431
|
this.submitRequest(
|
|
1394
1432
|
pending,
|
|
1395
1433
|
route.targetNodeRid,
|
|
1396
|
-
instanceOperationParts(
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1434
|
+
instanceOperationParts(
|
|
1435
|
+
[
|
|
1436
|
+
encodeInstanceSpotHeader(
|
|
1437
|
+
route,
|
|
1438
|
+
this.nodeGeneration,
|
|
1439
|
+
this.nodeRid,
|
|
1440
|
+
sourceSpotId,
|
|
1441
|
+
'request',
|
|
1442
|
+
{ high: 2n, low: pending.id },
|
|
1443
|
+
pending.id,
|
|
1444
|
+
metadataFrame !== undefined
|
|
1445
|
+
),
|
|
1446
|
+
encodeApplicationPayload(payload)
|
|
1447
|
+
],
|
|
1448
|
+
metadataFrame
|
|
1449
|
+
),
|
|
1409
1450
|
timeoutMs,
|
|
1410
1451
|
'instanceSpotRequest'
|
|
1411
1452
|
);
|
|
@@ -1442,20 +1483,23 @@ export class ServiceStatefulRuntime {
|
|
|
1442
1483
|
this.submitRequest(
|
|
1443
1484
|
pending,
|
|
1444
1485
|
target.targetNodeRid,
|
|
1445
|
-
instanceOperationParts(
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1486
|
+
instanceOperationParts(
|
|
1487
|
+
[
|
|
1488
|
+
encodeInstanceSpotActivationHeader(
|
|
1489
|
+
target,
|
|
1490
|
+
this.nodeGeneration,
|
|
1491
|
+
this.nodeRid,
|
|
1492
|
+
sourceSpotId,
|
|
1493
|
+
'request',
|
|
1494
|
+
{ high: this.nodeGeneration, low: pending.id },
|
|
1495
|
+
deadlineUnixMs,
|
|
1496
|
+
pending.id,
|
|
1497
|
+
metadataFrame !== undefined
|
|
1498
|
+
),
|
|
1499
|
+
payloadFrame
|
|
1500
|
+
],
|
|
1501
|
+
metadataFrame
|
|
1502
|
+
),
|
|
1459
1503
|
timeoutMs,
|
|
1460
1504
|
'instanceSpotRequest'
|
|
1461
1505
|
);
|
|
@@ -1506,19 +1550,15 @@ export class ServiceStatefulRuntime {
|
|
|
1506
1550
|
timeoutMs: number
|
|
1507
1551
|
): ServiceStatefulPendingOperation {
|
|
1508
1552
|
const deadlineMs = performance.now() + timeoutMs;
|
|
1509
|
-
const pending = this.operations.register(
|
|
1553
|
+
const pending = this.operations.register(
|
|
1554
|
+
timeoutMs,
|
|
1555
|
+
targetNodeRid === this.nodeRid ? 'registry' : 'sender'
|
|
1556
|
+
);
|
|
1510
1557
|
const target = this.trySpotFence(targetNodeRid, {
|
|
1511
1558
|
...targetSpot,
|
|
1512
1559
|
generation: targetSpotGeneration
|
|
1513
1560
|
});
|
|
1514
|
-
this.submitActorJoin(
|
|
1515
|
-
pending,
|
|
1516
|
-
actor,
|
|
1517
|
-
targetNodeRid,
|
|
1518
|
-
target,
|
|
1519
|
-
payload,
|
|
1520
|
-
deadlineMs
|
|
1521
|
-
);
|
|
1561
|
+
this.submitActorJoin(pending, actor, targetNodeRid, target, payload, deadlineMs);
|
|
1522
1562
|
return pending;
|
|
1523
1563
|
}
|
|
1524
1564
|
|
|
@@ -1533,20 +1573,19 @@ export class ServiceStatefulRuntime {
|
|
|
1533
1573
|
timeoutMs: number
|
|
1534
1574
|
): ServiceStatefulPendingOperation {
|
|
1535
1575
|
const deadlineMs = performance.now() + timeoutMs;
|
|
1536
|
-
const pending = this.operations.register(
|
|
1576
|
+
const pending = this.operations.register(
|
|
1577
|
+
timeoutMs,
|
|
1578
|
+
targetNodeRid === this.nodeRid ? 'registry' : 'sender'
|
|
1579
|
+
);
|
|
1537
1580
|
const target = this.trySpotFence(targetNodeRid, {
|
|
1538
1581
|
...targetSpot,
|
|
1539
1582
|
generation: targetSpotGeneration
|
|
1540
1583
|
});
|
|
1541
|
-
this.submitActorJoin(
|
|
1542
|
-
pending,
|
|
1543
|
-
actor,
|
|
1544
|
-
targetNodeRid,
|
|
1545
|
-
target,
|
|
1584
|
+
this.submitActorJoin(pending, actor, targetNodeRid, target, request, deadlineMs, {
|
|
1546
1585
|
request,
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
);
|
|
1586
|
+
actorFence,
|
|
1587
|
+
local
|
|
1588
|
+
});
|
|
1550
1589
|
return pending;
|
|
1551
1590
|
}
|
|
1552
1591
|
|
|
@@ -1557,7 +1596,10 @@ export class ServiceStatefulRuntime {
|
|
|
1557
1596
|
timeoutMs: number
|
|
1558
1597
|
): ServiceStatefulPendingOperation {
|
|
1559
1598
|
const deadlineMs = performance.now() + timeoutMs;
|
|
1560
|
-
const pending = this.operations.register(
|
|
1599
|
+
const pending = this.operations.register(
|
|
1600
|
+
timeoutMs,
|
|
1601
|
+
targetNodeRid === this.nodeRid ? 'registry' : 'sender'
|
|
1602
|
+
);
|
|
1561
1603
|
this.submitActorJoin(
|
|
1562
1604
|
pending,
|
|
1563
1605
|
actor,
|
|
@@ -1578,7 +1620,10 @@ export class ServiceStatefulRuntime {
|
|
|
1578
1620
|
timeoutMs: number
|
|
1579
1621
|
): ServiceStatefulPendingOperation {
|
|
1580
1622
|
const deadlineMs = performance.now() + timeoutMs;
|
|
1581
|
-
const pending = this.operations.register(
|
|
1623
|
+
const pending = this.operations.register(
|
|
1624
|
+
timeoutMs,
|
|
1625
|
+
targetNodeRid === this.nodeRid ? 'registry' : 'sender'
|
|
1626
|
+
);
|
|
1582
1627
|
this.submitActorJoin(
|
|
1583
1628
|
pending,
|
|
1584
1629
|
actor,
|
|
@@ -1656,7 +1701,10 @@ export class ServiceStatefulRuntime {
|
|
|
1656
1701
|
const previous = this.registry.binding(actor);
|
|
1657
1702
|
if (previous !== undefined && sameSessionIdentity(previous, localBinding)) {
|
|
1658
1703
|
const currentDelivery = this.sessionDeliveries.get(deliveryKey);
|
|
1659
|
-
if (
|
|
1704
|
+
if (
|
|
1705
|
+
currentDelivery !== undefined &&
|
|
1706
|
+
sameSessionBinding(currentDelivery.binding, previous)
|
|
1707
|
+
) {
|
|
1660
1708
|
this.sessionDeliveries.set(deliveryKey, {
|
|
1661
1709
|
...delivery,
|
|
1662
1710
|
binding: previous
|
|
@@ -1680,10 +1728,7 @@ export class ServiceStatefulRuntime {
|
|
|
1680
1728
|
this.nodeGeneration
|
|
1681
1729
|
);
|
|
1682
1730
|
if (previousDelivery !== undefined && previous !== undefined) {
|
|
1683
|
-
this.retiredSessionDeliveries.set(
|
|
1684
|
-
sessionDeliveryKey(previous),
|
|
1685
|
-
previousDelivery
|
|
1686
|
-
);
|
|
1731
|
+
this.retiredSessionDeliveries.set(sessionDeliveryKey(previous), previousDelivery);
|
|
1687
1732
|
}
|
|
1688
1733
|
installedDelivery = { ...delivery, binding };
|
|
1689
1734
|
this.sessionDeliveries.set(deliveryKey, installedDelivery);
|
|
@@ -1693,18 +1738,24 @@ export class ServiceStatefulRuntime {
|
|
|
1693
1738
|
kindData: undefined
|
|
1694
1739
|
});
|
|
1695
1740
|
if (previous !== undefined) {
|
|
1696
|
-
this.sendBoundSessionReplacement(
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1741
|
+
this.sendBoundSessionReplacement(
|
|
1742
|
+
previous,
|
|
1743
|
+
actorAuthorityFromRoute(
|
|
1744
|
+
this.tryActorFence(actor) ?? {
|
|
1745
|
+
actor,
|
|
1746
|
+
targetNodeGeneration: this.nodeGeneration,
|
|
1747
|
+
authorityOwnerGeneration:
|
|
1748
|
+
this.registry.actor(actor.actorId)?.authorityOwnerGeneration ?? actor.generation,
|
|
1749
|
+
ownerLeaseGeneration: this.nodeGeneration
|
|
1750
|
+
}
|
|
1751
|
+
)
|
|
1752
|
+
);
|
|
1705
1753
|
}
|
|
1706
1754
|
} catch (error) {
|
|
1707
|
-
if (
|
|
1755
|
+
if (
|
|
1756
|
+
installedDelivery !== undefined &&
|
|
1757
|
+
this.sessionDeliveries.get(deliveryKey) === installedDelivery
|
|
1758
|
+
) {
|
|
1708
1759
|
this.sessionDeliveries.delete(deliveryKey);
|
|
1709
1760
|
}
|
|
1710
1761
|
this.operations.reply(pending.id, failure(error));
|
|
@@ -1715,28 +1766,29 @@ export class ServiceStatefulRuntime {
|
|
|
1715
1766
|
let bindingActorFence: ServiceActorRouteFence;
|
|
1716
1767
|
let header: Buffer;
|
|
1717
1768
|
try {
|
|
1718
|
-
bindingActorFence =
|
|
1719
|
-
? this.actorFence(actor)
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
sessionRid,
|
|
1725
|
-
{ state: 'active', generation }
|
|
1726
|
-
);
|
|
1769
|
+
bindingActorFence =
|
|
1770
|
+
actorAuthority === undefined ? this.actorFence(actor) : { actor, ...actorAuthority };
|
|
1771
|
+
header = encodeBoundSessionBindHeader(pending.id, bindingActorFence, sessionRid, {
|
|
1772
|
+
state: 'active',
|
|
1773
|
+
generation
|
|
1774
|
+
});
|
|
1727
1775
|
} catch (error) {
|
|
1728
1776
|
this.operations.fail(pending.id, error);
|
|
1729
1777
|
return pending;
|
|
1730
1778
|
}
|
|
1731
1779
|
void this.requestDurableOperation(
|
|
1732
|
-
actor.nodeRid,
|
|
1780
|
+
actor.nodeRid,
|
|
1781
|
+
[header],
|
|
1782
|
+
pending.id,
|
|
1783
|
+
'streamBind',
|
|
1784
|
+
deadlineMs
|
|
1733
1785
|
).then(
|
|
1734
|
-
reply => this.completeRemoteReply(pending, actor.nodeRid, 'streamBind', undefined, reply),
|
|
1735
|
-
error => this.operations.fail(pending.id, error)
|
|
1786
|
+
(reply) => this.completeRemoteReply(pending, actor.nodeRid, 'streamBind', undefined, reply),
|
|
1787
|
+
(error) => this.operations.fail(pending.id, error)
|
|
1736
1788
|
);
|
|
1737
1789
|
return {
|
|
1738
1790
|
id: pending.id,
|
|
1739
|
-
promise: pending.promise.then(result => {
|
|
1791
|
+
promise: pending.promise.then((result) => {
|
|
1740
1792
|
if (result.terminalResult === RequestResult.Ok && result.streamBinding !== undefined) {
|
|
1741
1793
|
const committed: ServiceSessionDelivery = {
|
|
1742
1794
|
...delivery,
|
|
@@ -1747,10 +1799,10 @@ export class ServiceStatefulRuntime {
|
|
|
1747
1799
|
};
|
|
1748
1800
|
const current = this.sessionDeliveries.get(deliveryKey);
|
|
1749
1801
|
if (
|
|
1750
|
-
current === previousDelivery
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1802
|
+
current === previousDelivery ||
|
|
1803
|
+
current === undefined ||
|
|
1804
|
+
(sameSessionOwnerIdentity(current.binding, committed.binding) &&
|
|
1805
|
+
current.binding.bindingGeneration < committed.binding.bindingGeneration)
|
|
1754
1806
|
) {
|
|
1755
1807
|
if (current !== undefined && !sameSessionBinding(current.binding, committed.binding)) {
|
|
1756
1808
|
this.retiredSessionDeliveries.set(sessionDeliveryKey(current.binding), current);
|
|
@@ -1776,9 +1828,9 @@ export class ServiceStatefulRuntime {
|
|
|
1776
1828
|
const pending = this.operations.register(timeoutMs);
|
|
1777
1829
|
const delivery = this.sessionDeliveries.get(actorKey(actor));
|
|
1778
1830
|
if (
|
|
1779
|
-
delivery === undefined
|
|
1780
|
-
|
|
1781
|
-
|
|
1831
|
+
delivery === undefined ||
|
|
1832
|
+
delivery.binding.sessionRid !== sessionRid ||
|
|
1833
|
+
delivery.binding.bindingGeneration !== expectedBindingGeneration
|
|
1782
1834
|
) {
|
|
1783
1835
|
this.operations.reply(pending.id, {
|
|
1784
1836
|
terminalResult: RequestResult.NotFound,
|
|
@@ -1796,24 +1848,22 @@ export class ServiceStatefulRuntime {
|
|
|
1796
1848
|
}
|
|
1797
1849
|
return pending;
|
|
1798
1850
|
}
|
|
1799
|
-
const header = encodeBoundSessionBindHeader(
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
{ state: 'tombstone', retiredGeneration: expectedBindingGeneration }
|
|
1804
|
-
);
|
|
1851
|
+
const header = encodeBoundSessionBindHeader(pending.id, this.actorFence(actor), sessionRid, {
|
|
1852
|
+
state: 'tombstone',
|
|
1853
|
+
retiredGeneration: expectedBindingGeneration
|
|
1854
|
+
});
|
|
1805
1855
|
this.submitRequest(pending, actor.nodeRid, [header], timeoutMs, 'streamUnbind');
|
|
1806
1856
|
return pending;
|
|
1807
1857
|
}
|
|
1808
1858
|
|
|
1809
1859
|
sessionBindings(sessionRid: string): readonly ServiceSessionBinding[] {
|
|
1810
1860
|
return [...this.sessionDeliveries.values()]
|
|
1811
|
-
.map(value => value.binding)
|
|
1812
|
-
.filter(binding => binding.sessionRid === sessionRid);
|
|
1861
|
+
.map((value) => value.binding)
|
|
1862
|
+
.filter((binding) => binding.sessionRid === sessionRid);
|
|
1813
1863
|
}
|
|
1814
1864
|
|
|
1815
1865
|
allSessionBindings(): readonly ServiceSessionBinding[] {
|
|
1816
|
-
return [...this.sessionDeliveries.values()].map(value => value.binding);
|
|
1866
|
+
return [...this.sessionDeliveries.values()].map((value) => value.binding);
|
|
1817
1867
|
}
|
|
1818
1868
|
|
|
1819
1869
|
async sendSessionToActor(
|
|
@@ -1852,14 +1902,11 @@ export class ServiceStatefulRuntime {
|
|
|
1852
1902
|
}
|
|
1853
1903
|
const actorFence = this.tryBoundSessionSendFence(actor, senderFence);
|
|
1854
1904
|
if (actorFence === undefined) return SubmitResult.InvalidState;
|
|
1855
|
-
const header = encodeBoundSessionSendHeader(
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
binding.sessionOwnerNodeRid,
|
|
1861
|
-
[header, encodeApplicationPayload(payload)]
|
|
1862
|
-
);
|
|
1905
|
+
const header = encodeBoundSessionSendHeader(actorFence, expectedBindingGeneration);
|
|
1906
|
+
return this.submitOneWay(binding.sessionOwnerNodeRid, [
|
|
1907
|
+
header,
|
|
1908
|
+
encodeApplicationPayload(payload)
|
|
1909
|
+
]);
|
|
1863
1910
|
}
|
|
1864
1911
|
|
|
1865
1912
|
close(): void {
|
|
@@ -1906,23 +1953,21 @@ export class ServiceStatefulRuntime {
|
|
|
1906
1953
|
// fall through to RawServiceMeshRuntime's relocation ingress instead of
|
|
1907
1954
|
// being decoded as stateful actor/spot work.
|
|
1908
1955
|
if (
|
|
1909
|
-
record.command === M6bServiceWireCommand.relocationReady
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1956
|
+
record.command === M6bServiceWireCommand.relocationReady ||
|
|
1957
|
+
record.command === M6bServiceWireCommand.relocationData ||
|
|
1958
|
+
record.command === M6bServiceWireCommand.replyRelay ||
|
|
1959
|
+
record.command === M6bServiceWireCommand.relocationCutover
|
|
1913
1960
|
) {
|
|
1914
1961
|
return undefined;
|
|
1915
1962
|
}
|
|
1916
1963
|
if (
|
|
1917
|
-
record.command < M6bServiceWireCommand.spotSend
|
|
1918
|
-
|
|
1919
|
-
record.command
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
&& record.command !== M6bServiceWireCommand.boundSessionReplaced
|
|
1925
|
-
)
|
|
1964
|
+
record.command < M6bServiceWireCommand.spotSend ||
|
|
1965
|
+
(record.command > M6bServiceWireCommand.instanceSpot &&
|
|
1966
|
+
record.command !== M6bServiceWireCommand.userSpotCreate &&
|
|
1967
|
+
record.command !== M6bServiceWireCommand.userSpotClose &&
|
|
1968
|
+
record.command !== M6bServiceWireCommand.actorCreate &&
|
|
1969
|
+
record.command !== M6bServiceWireCommand.messageFollow &&
|
|
1970
|
+
record.command !== M6bServiceWireCommand.boundSessionReplaced)
|
|
1926
1971
|
) {
|
|
1927
1972
|
return undefined;
|
|
1928
1973
|
}
|
|
@@ -1948,38 +1993,40 @@ export class ServiceStatefulRuntime {
|
|
|
1948
1993
|
'actorJoin',
|
|
1949
1994
|
'boundSessionSend'
|
|
1950
1995
|
].includes(decoded.kind);
|
|
1951
|
-
const instanceMetadata =
|
|
1952
|
-
&& (record.flags & M6bServiceWireFlag.metadata) !== 0;
|
|
1953
|
-
const userSpotInfrastructure =
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1996
|
+
const instanceMetadata =
|
|
1997
|
+
decoded.kind === 'instanceSpot' && (record.flags & M6bServiceWireFlag.metadata) !== 0;
|
|
1998
|
+
const userSpotInfrastructure =
|
|
1999
|
+
decoded.kind === 'userSpotCreate' ||
|
|
2000
|
+
decoded.kind === 'userSpotClose' ||
|
|
2001
|
+
decoded.kind === 'actorCreate' ||
|
|
2002
|
+
decoded.kind === 'messageFollow' ||
|
|
2003
|
+
decoded.kind === 'boundSessionReplaced';
|
|
1958
2004
|
if (
|
|
1959
|
-
record.parts.length < 1
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
&& record.parts.length !== (instanceMetadata ? 3 : 2)
|
|
1965
|
-
)
|
|
1966
|
-
|| (userSpotInfrastructure && record.parts.length !== 1)
|
|
2005
|
+
record.parts.length < 1 ||
|
|
2006
|
+
record.parts.length > (instanceMetadata ? 3 : 2) ||
|
|
2007
|
+
(hasPayload && decoded.kind !== 'actorJoin' && record.parts.length !== 2) ||
|
|
2008
|
+
(decoded.kind === 'instanceSpot' && record.parts.length !== (instanceMetadata ? 3 : 2)) ||
|
|
2009
|
+
(userSpotInfrastructure && record.parts.length !== 1)
|
|
1967
2010
|
) {
|
|
1968
2011
|
return 'protocolError';
|
|
1969
2012
|
}
|
|
1970
2013
|
const metadataFrame = instanceMetadata
|
|
1971
2014
|
? validateServiceMetadataFrame(record.parts[1]!)
|
|
1972
2015
|
: undefined;
|
|
1973
|
-
const canonicalPayload =
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
2016
|
+
const canonicalPayload =
|
|
2017
|
+
canonicalActorJoin?.payload === undefined
|
|
2018
|
+
? undefined
|
|
2019
|
+
: {
|
|
2020
|
+
packetName: canonicalActorJoin.payload.packetName,
|
|
2021
|
+
contentType: canonicalActorJoin.payload.contentType,
|
|
2022
|
+
payload: Buffer.from(canonicalActorJoin.payload.payload)
|
|
2023
|
+
};
|
|
2024
|
+
const payloadFrame =
|
|
2025
|
+
canonicalPayload === undefined
|
|
2026
|
+
? record.parts.length >= 2
|
|
2027
|
+
? record.parts[record.parts.length - 1]
|
|
2028
|
+
: undefined
|
|
2029
|
+
: encodeApplicationPayload(canonicalPayload);
|
|
1983
2030
|
try {
|
|
1984
2031
|
return await this.handleIngress(
|
|
1985
2032
|
record,
|
|
@@ -2000,9 +2047,9 @@ export class ServiceStatefulRuntime {
|
|
|
2000
2047
|
if (error instanceof ZLinkFrameworkException) {
|
|
2001
2048
|
const kind = internalFrameworkErrorKind(error);
|
|
2002
2049
|
if (
|
|
2003
|
-
kind === ZLinkFrameworkInternalErrorKind.RequestTargetNotFound
|
|
2004
|
-
|
|
2005
|
-
|
|
2050
|
+
kind === ZLinkFrameworkInternalErrorKind.RequestTargetNotFound ||
|
|
2051
|
+
kind === ZLinkFrameworkInternalErrorKind.SpotMoving ||
|
|
2052
|
+
kind === ZLinkFrameworkInternalErrorKind.SpotGenerationStale
|
|
2006
2053
|
) {
|
|
2007
2054
|
return 'protocolError';
|
|
2008
2055
|
}
|
|
@@ -2025,9 +2072,10 @@ export class ServiceStatefulRuntime {
|
|
|
2025
2072
|
): Promise<RawServicePumpResult> {
|
|
2026
2073
|
switch (record.kind) {
|
|
2027
2074
|
case 'messageFollow': {
|
|
2028
|
-
if (
|
|
2029
|
-
|
|
2030
|
-
|
|
2075
|
+
if (
|
|
2076
|
+
record.source.kind !== record.target.kind ||
|
|
2077
|
+
record.target.authorityOwnerGeneration <= record.source.authorityOwnerGeneration
|
|
2078
|
+
) {
|
|
2031
2079
|
throw new ServiceWireProtocolError(
|
|
2032
2080
|
'Message Follow target owner generation must increase at every hop.'
|
|
2033
2081
|
);
|
|
@@ -2043,10 +2091,10 @@ export class ServiceStatefulRuntime {
|
|
|
2043
2091
|
'Message Follow source route is not the current admitted peer.'
|
|
2044
2092
|
);
|
|
2045
2093
|
}
|
|
2046
|
-
const targetGeneration =
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
.lifecycleGeneration;
|
|
2094
|
+
const targetGeneration =
|
|
2095
|
+
record.target.targetNodeRid === this.nodeRid
|
|
2096
|
+
? this.nodeGeneration
|
|
2097
|
+
: this.raw.topology.peer(record.target.targetNodeRid)?.descriptor.lifecycleGeneration;
|
|
2050
2098
|
if (targetGeneration !== record.target.targetNodeGeneration) {
|
|
2051
2099
|
throw new ServiceWireProtocolError(
|
|
2052
2100
|
'Message Follow target route is not the current topology target.'
|
|
@@ -2057,8 +2105,7 @@ export class ServiceStatefulRuntime {
|
|
|
2057
2105
|
} else {
|
|
2058
2106
|
const directKey = directSpotKey(record.source.spot.spotId);
|
|
2059
2107
|
const current = this.directSpotRoutes.get(directKey);
|
|
2060
|
-
if (current !== undefined
|
|
2061
|
-
&& sameMessageFollowSpotSource(current, record.source)) {
|
|
2108
|
+
if (current !== undefined && sameMessageFollowSpotSource(current, record.source)) {
|
|
2062
2109
|
this.directSpotRoutes.delete(directKey);
|
|
2063
2110
|
const exactKey = spotKey(record.source.spot);
|
|
2064
2111
|
const exact = this.spotRoutes.get(exactKey);
|
|
@@ -2079,7 +2126,8 @@ export class ServiceStatefulRuntime {
|
|
|
2079
2126
|
`spot:${record.target.spot.spotId}`,
|
|
2080
2127
|
payloadFrame!,
|
|
2081
2128
|
{
|
|
2082
|
-
receiveKind:
|
|
2129
|
+
receiveKind:
|
|
2130
|
+
record.kind === 'spotSend' ? ReceiveKind.SpotSend : ReceiveKind.SpotRequest,
|
|
2083
2131
|
operationKind: record.kind === 'spotRequest' ? OperationKind.SpotRequest : 0,
|
|
2084
2132
|
...(record.correlation === undefined ? {} : { correlation: record.correlation }),
|
|
2085
2133
|
sourceSpotId: record.sourceSpotId,
|
|
@@ -2092,17 +2140,18 @@ export class ServiceStatefulRuntime {
|
|
|
2092
2140
|
}
|
|
2093
2141
|
case 'actorSend':
|
|
2094
2142
|
case 'actorRequest': {
|
|
2095
|
-
const actor =
|
|
2096
|
-
|
|
2097
|
-
|
|
2143
|
+
const actor =
|
|
2144
|
+
record.boundSession === undefined
|
|
2145
|
+
? this.validateActorMessageFence(record.target)
|
|
2146
|
+
: this.validateActorFence(record.target);
|
|
2098
2147
|
if (record.boundSession !== undefined) {
|
|
2099
2148
|
const binding = this.registry.validateBoundSession(
|
|
2100
2149
|
record.target.actor,
|
|
2101
2150
|
record.boundSession.bindingGeneration
|
|
2102
2151
|
);
|
|
2103
2152
|
if (
|
|
2104
|
-
binding.sessionRid !== record.boundSession.sessionRid
|
|
2105
|
-
|
|
2153
|
+
binding.sessionRid !== record.boundSession.sessionRid ||
|
|
2154
|
+
binding.sessionOwnerNodeRid !== ingress.sourceRoutingId
|
|
2106
2155
|
) {
|
|
2107
2156
|
throw new ServiceStaleGenerationError('binding', record.target.actor.actorId);
|
|
2108
2157
|
}
|
|
@@ -2112,7 +2161,8 @@ export class ServiceStatefulRuntime {
|
|
|
2112
2161
|
`actor:${actor.ref.actorId}\0${actor.ref.generation}`,
|
|
2113
2162
|
payloadFrame!,
|
|
2114
2163
|
{
|
|
2115
|
-
receiveKind:
|
|
2164
|
+
receiveKind:
|
|
2165
|
+
record.kind === 'actorSend' ? ReceiveKind.ActorSend : ReceiveKind.ActorRequest,
|
|
2116
2166
|
operationKind: record.kind === 'actorRequest' ? OperationKind.ActorRequest : 0,
|
|
2117
2167
|
...(record.correlation === undefined ? {} : { correlation: record.correlation }),
|
|
2118
2168
|
...(record.sourceActor === undefined
|
|
@@ -2170,18 +2220,9 @@ export class ServiceStatefulRuntime {
|
|
|
2170
2220
|
case 'boundSessionReplaced':
|
|
2171
2221
|
return this.handleBoundSessionReplaced(ingress, record);
|
|
2172
2222
|
case 'boundSessionSend':
|
|
2173
|
-
return await this.deliverBoundSession(
|
|
2174
|
-
ingress,
|
|
2175
|
-
record,
|
|
2176
|
-
payloadFrame
|
|
2177
|
-
);
|
|
2223
|
+
return await this.deliverBoundSession(ingress, record, payloadFrame);
|
|
2178
2224
|
case 'instanceSpot':
|
|
2179
|
-
return this.enqueueInstanceSpot(
|
|
2180
|
-
ingress,
|
|
2181
|
-
record,
|
|
2182
|
-
payloadFrame!,
|
|
2183
|
-
metadataFrame
|
|
2184
|
-
);
|
|
2225
|
+
return this.enqueueInstanceSpot(ingress, record, payloadFrame!, metadataFrame);
|
|
2185
2226
|
case 'userSpotCreate':
|
|
2186
2227
|
case 'userSpotClose':
|
|
2187
2228
|
case 'actorCreate':
|
|
@@ -2207,8 +2248,8 @@ export class ServiceStatefulRuntime {
|
|
|
2207
2248
|
return 'infrastructure';
|
|
2208
2249
|
}
|
|
2209
2250
|
if (
|
|
2210
|
-
record.activation === 'ready'
|
|
2211
|
-
|
|
2251
|
+
record.activation === 'ready' &&
|
|
2252
|
+
this.needsInstanceApplicationMaterialization(ingress, record)
|
|
2212
2253
|
) {
|
|
2213
2254
|
const ownedIngress = shareIngressOwnership(ingress);
|
|
2214
2255
|
void this.continueReadyInstanceMaterialization(
|
|
@@ -2220,13 +2261,14 @@ export class ServiceStatefulRuntime {
|
|
|
2220
2261
|
return 'infrastructure';
|
|
2221
2262
|
}
|
|
2222
2263
|
const spot = this.requireInstanceActivation(ingress, record);
|
|
2223
|
-
const applicationTarget =
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2264
|
+
const applicationTarget =
|
|
2265
|
+
record.activation === 'missing'
|
|
2266
|
+
? {
|
|
2267
|
+
targetSpotId: record.target.targetSpotId,
|
|
2268
|
+
stableType: record.target.stableType,
|
|
2269
|
+
objectGeneration: spot.ref.generation
|
|
2270
|
+
}
|
|
2271
|
+
: this.instanceApplicationTarget(record);
|
|
2230
2272
|
return this.enqueueActivatedInstanceSpot(
|
|
2231
2273
|
ingress,
|
|
2232
2274
|
record,
|
|
@@ -2248,10 +2290,12 @@ export class ServiceStatefulRuntime {
|
|
|
2248
2290
|
const lifecycle = this.instanceApplicationLifecycle;
|
|
2249
2291
|
if (lifecycle === undefined) return false;
|
|
2250
2292
|
const { intent } = this.requireReadyInstanceApplicationRoute(ingress, record);
|
|
2251
|
-
if (
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2293
|
+
if (
|
|
2294
|
+
this.instanceApplicationLifecycle?.isIdleEvicting?.({
|
|
2295
|
+
targetSpotId: record.route.targetSpotId,
|
|
2296
|
+
stableType: intent.instanceType
|
|
2297
|
+
}) === true
|
|
2298
|
+
) {
|
|
2255
2299
|
throw createInternalFrameworkException(
|
|
2256
2300
|
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
2257
2301
|
`Instance Spot '${record.route.targetSpotId}' is entering idle eviction.`
|
|
@@ -2289,15 +2333,13 @@ export class ServiceStatefulRuntime {
|
|
|
2289
2333
|
operationKey = instanceOperationKey(record);
|
|
2290
2334
|
if (this.admittedInstanceOperations.has(operationKey)) return 'application';
|
|
2291
2335
|
}
|
|
2292
|
-
const directReply =
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
const applicationTarget = applicationTargetOverride
|
|
2300
|
-
?? (record.activation === 'missing'
|
|
2336
|
+
const directReply =
|
|
2337
|
+
record.operationKind === 'request'
|
|
2338
|
+
? (localReply ?? this.replyPort(ingress, record.replyRouteId, 'instanceSpotRequest'))
|
|
2339
|
+
: undefined;
|
|
2340
|
+
const applicationTarget =
|
|
2341
|
+
applicationTargetOverride ??
|
|
2342
|
+
(record.activation === 'missing'
|
|
2301
2343
|
? {
|
|
2302
2344
|
targetSpotId: record.target.targetSpotId,
|
|
2303
2345
|
stableType: record.target.stableType,
|
|
@@ -2305,45 +2347,42 @@ export class ServiceStatefulRuntime {
|
|
|
2305
2347
|
}
|
|
2306
2348
|
: this.instanceApplicationTarget(record));
|
|
2307
2349
|
let deferredReply: Parameters<NonNullable<ServiceStatefulMailboxData['reply']>> | undefined;
|
|
2308
|
-
const terminalCompletion =
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2350
|
+
const terminalCompletion =
|
|
2351
|
+
onTerminalCompletion === undefined
|
|
2352
|
+
? undefined
|
|
2353
|
+
: async () => {
|
|
2354
|
+
// The application turn has returned before authority cleanup starts.
|
|
2355
|
+
// Mark it quiescent first so a close requested by that turn can finish
|
|
2356
|
+
// its local cleanup and release the exact authority fence below.
|
|
2357
|
+
this.completeInstanceApplicationOperation(applicationTarget.targetSpotId);
|
|
2358
|
+
try {
|
|
2359
|
+
await onTerminalCompletion();
|
|
2360
|
+
} finally {
|
|
2361
|
+
if (deferredReply !== undefined && directReply !== undefined) {
|
|
2362
|
+
directReply(...deferredReply);
|
|
2363
|
+
}
|
|
2320
2364
|
}
|
|
2321
|
-
}
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
...(terminalCompletion === undefined ? {} : { onTerminalCompletion: terminalCompletion }),
|
|
2335
|
-
...(record.operationKind === 'request'
|
|
2336
|
-
? {
|
|
2337
|
-
reply: onTerminalCompletion === undefined
|
|
2365
|
+
};
|
|
2366
|
+
const result = this.enqueueApplicationFrame(ingress, `spot:${spot.ref.spotId}`, payloadFrame, {
|
|
2367
|
+
receiveKind: ReceiveKind.InstanceSpotActivation,
|
|
2368
|
+
operationKind: record.operationKind === 'request' ? OperationKind.InstanceSpotRequest : 0,
|
|
2369
|
+
...(record.operationKind === 'request' ? { correlation: record.replyRouteId } : {}),
|
|
2370
|
+
...(record.sourceSpotId === undefined ? {} : { sourceSpotId: record.sourceSpotId }),
|
|
2371
|
+
targetSpot: spot.ref,
|
|
2372
|
+
...(metadataFrame === undefined ? {} : { applicationMetadata: metadataFrame }),
|
|
2373
|
+
...(terminalCompletion === undefined ? {} : { onTerminalCompletion: terminalCompletion }),
|
|
2374
|
+
...(record.operationKind === 'request'
|
|
2375
|
+
? {
|
|
2376
|
+
reply:
|
|
2377
|
+
onTerminalCompletion === undefined
|
|
2338
2378
|
? directReply
|
|
2339
|
-
: (
|
|
2379
|
+
: (...reply: Parameters<NonNullable<ServiceStatefulMailboxData['reply']>>) => {
|
|
2340
2380
|
deferredReply = reply;
|
|
2341
2381
|
return true;
|
|
2342
|
-
}
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
);
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
: {})
|
|
2385
|
+
});
|
|
2347
2386
|
if (operationKey !== undefined && result === 'application' && record.activation === 'missing') {
|
|
2348
2387
|
this.admittedInstanceOperations.set(operationKey, record.deadlineUnixMs);
|
|
2349
2388
|
}
|
|
@@ -2369,7 +2408,11 @@ export class ServiceStatefulRuntime {
|
|
|
2369
2408
|
): Promise<void> {
|
|
2370
2409
|
try {
|
|
2371
2410
|
this.validateInstanceIngress(ingress, record);
|
|
2372
|
-
const activation = await this.activateMissingInstanceAsync(
|
|
2411
|
+
const activation = await this.activateMissingInstanceAsync(
|
|
2412
|
+
record,
|
|
2413
|
+
payloadFrame,
|
|
2414
|
+
metadataFrame
|
|
2415
|
+
);
|
|
2373
2416
|
if (this.closed) return;
|
|
2374
2417
|
const admitted = this.enqueueActivatedInstanceSpot(
|
|
2375
2418
|
ingress,
|
|
@@ -2408,12 +2451,7 @@ export class ServiceStatefulRuntime {
|
|
|
2408
2451
|
localReply(result.terminalResult, result.failureCode);
|
|
2409
2452
|
} else {
|
|
2410
2453
|
if (record.replyRouteId === undefined) return;
|
|
2411
|
-
this.replyWire(
|
|
2412
|
-
ingress,
|
|
2413
|
-
record.replyRouteId,
|
|
2414
|
-
result.terminalResult,
|
|
2415
|
-
result.failureCode
|
|
2416
|
-
);
|
|
2454
|
+
this.replyWire(ingress, record.replyRouteId, result.terminalResult, result.failureCode);
|
|
2417
2455
|
}
|
|
2418
2456
|
}
|
|
2419
2457
|
}
|
|
@@ -2445,10 +2483,10 @@ export class ServiceStatefulRuntime {
|
|
|
2445
2483
|
);
|
|
2446
2484
|
if (this.closed) return;
|
|
2447
2485
|
const latestRoute = this.instanceIntents.get(record.route.targetSpotId)?.route;
|
|
2448
|
-
const refreshedRecord =
|
|
2449
|
-
&& sameInstanceApplicationOwner(latestRoute, record.route)
|
|
2450
|
-
|
|
2451
|
-
|
|
2486
|
+
const refreshedRecord =
|
|
2487
|
+
latestRoute !== undefined && sameInstanceApplicationOwner(latestRoute, record.route)
|
|
2488
|
+
? { ...record, route: latestRoute }
|
|
2489
|
+
: record;
|
|
2452
2490
|
const spot = this.requireInstanceActivation(ingress, refreshedRecord);
|
|
2453
2491
|
const admitted = this.enqueueActivatedInstanceSpot(
|
|
2454
2492
|
ingress,
|
|
@@ -2465,12 +2503,7 @@ export class ServiceStatefulRuntime {
|
|
|
2465
2503
|
} catch (error) {
|
|
2466
2504
|
if (record.operationKind !== 'request' || record.replyRouteId === undefined) return;
|
|
2467
2505
|
const result = failure(error);
|
|
2468
|
-
this.replyWire(
|
|
2469
|
-
ingress,
|
|
2470
|
-
record.replyRouteId,
|
|
2471
|
-
result.terminalResult,
|
|
2472
|
-
result.failureCode
|
|
2473
|
-
);
|
|
2506
|
+
this.replyWire(ingress, record.replyRouteId, result.terminalResult, result.failureCode);
|
|
2474
2507
|
}
|
|
2475
2508
|
}
|
|
2476
2509
|
|
|
@@ -2484,40 +2517,42 @@ export class ServiceStatefulRuntime {
|
|
|
2484
2517
|
stableType: target.stableType,
|
|
2485
2518
|
objectGeneration: route.objectGeneration
|
|
2486
2519
|
};
|
|
2487
|
-
return () =>
|
|
2488
|
-
|
|
2489
|
-
|
|
2490
|
-
|
|
2491
|
-
|
|
2492
|
-
authorityCompletionSignaled = true;
|
|
2493
|
-
if (released !== undefined) {
|
|
2494
|
-
const authorityReleased = await this.instanceApplicationLifecycle?.completeTerminal(applicationTarget)
|
|
2495
|
-
?? false;
|
|
2496
|
-
if (!authorityReleased) {
|
|
2497
|
-
this.replaceCommittedInstanceRoute(target.stableType, route, released);
|
|
2498
|
-
} else {
|
|
2499
|
-
this.forgetClosedInstanceRoute(released);
|
|
2500
|
-
}
|
|
2501
|
-
}
|
|
2502
|
-
} finally {
|
|
2503
|
-
if (!authorityCompletionSignaled) {
|
|
2520
|
+
return () =>
|
|
2521
|
+
(completion ??= (async () => {
|
|
2522
|
+
let authorityCompletionSignaled = false;
|
|
2523
|
+
try {
|
|
2524
|
+
const released = await this.asyncInstanceAuthority?.complete(target, route);
|
|
2504
2525
|
this.completeInstanceAuthorityOperation(target.targetSpotId);
|
|
2526
|
+
authorityCompletionSignaled = true;
|
|
2527
|
+
if (released !== undefined) {
|
|
2528
|
+
const authorityReleased =
|
|
2529
|
+
(await this.instanceApplicationLifecycle?.completeTerminal(applicationTarget)) ??
|
|
2530
|
+
false;
|
|
2531
|
+
if (!authorityReleased) {
|
|
2532
|
+
this.replaceCommittedInstanceRoute(target.stableType, route, released);
|
|
2533
|
+
} else {
|
|
2534
|
+
this.forgetClosedInstanceRoute(released);
|
|
2535
|
+
}
|
|
2536
|
+
}
|
|
2537
|
+
} finally {
|
|
2538
|
+
if (!authorityCompletionSignaled) {
|
|
2539
|
+
this.completeInstanceAuthorityOperation(target.targetSpotId);
|
|
2540
|
+
}
|
|
2505
2541
|
}
|
|
2506
|
-
}
|
|
2507
|
-
})();
|
|
2542
|
+
})());
|
|
2508
2543
|
}
|
|
2509
2544
|
|
|
2510
2545
|
private instanceApplicationTerminalCompletion(
|
|
2511
2546
|
target: ServiceInstanceApplicationTarget
|
|
2512
2547
|
): () => Promise<void> {
|
|
2513
2548
|
return async () => {
|
|
2514
|
-
const authorityReleased =
|
|
2515
|
-
?? false;
|
|
2549
|
+
const authorityReleased =
|
|
2550
|
+
(await this.instanceApplicationLifecycle?.completeTerminal(target)) ?? false;
|
|
2516
2551
|
if (!authorityReleased) return;
|
|
2517
2552
|
const current = this.instanceIntents.get(target.targetSpotId);
|
|
2518
2553
|
if (
|
|
2519
|
-
current?.instanceType === target.stableType
|
|
2520
|
-
|
|
2554
|
+
current?.instanceType === target.stableType &&
|
|
2555
|
+
current.route.objectGeneration === target.objectGeneration
|
|
2521
2556
|
) {
|
|
2522
2557
|
this.forgetClosedInstanceRoute(current.route);
|
|
2523
2558
|
}
|
|
@@ -2551,10 +2586,7 @@ export class ServiceStatefulRuntime {
|
|
|
2551
2586
|
|
|
2552
2587
|
private beginInstanceApplicationOperation(spotId: string): void {
|
|
2553
2588
|
const key = String(spotId);
|
|
2554
|
-
this.pendingInstanceTerminals.set(
|
|
2555
|
-
key,
|
|
2556
|
-
(this.pendingInstanceTerminals.get(key) ?? 0) + 1
|
|
2557
|
-
);
|
|
2589
|
+
this.pendingInstanceTerminals.set(key, (this.pendingInstanceTerminals.get(key) ?? 0) + 1);
|
|
2558
2590
|
}
|
|
2559
2591
|
|
|
2560
2592
|
private beginInstanceAuthorityOperation(spotId: string): void {
|
|
@@ -2589,9 +2621,10 @@ export class ServiceStatefulRuntime {
|
|
|
2589
2621
|
|
|
2590
2622
|
private completeInstanceApplicationWaitersIfQuiescent(key: string): void {
|
|
2591
2623
|
if (
|
|
2592
|
-
(this.pendingInstanceTerminals.get(key) ?? 0) !== 0
|
|
2593
|
-
|
|
2594
|
-
)
|
|
2624
|
+
(this.pendingInstanceTerminals.get(key) ?? 0) !== 0 ||
|
|
2625
|
+
(this.pendingInstanceAuthorityTerminals.get(key) ?? 0) !== 0
|
|
2626
|
+
)
|
|
2627
|
+
return;
|
|
2595
2628
|
const waiters = this.instanceApplicationWaiters.get(key);
|
|
2596
2629
|
if (waiters === undefined) return;
|
|
2597
2630
|
this.instanceApplicationWaiters.delete(key);
|
|
@@ -2620,20 +2653,26 @@ export class ServiceStatefulRuntime {
|
|
|
2620
2653
|
targetSpotId: route.targetSpotId
|
|
2621
2654
|
};
|
|
2622
2655
|
if (record.operationKind === 'send') {
|
|
2623
|
-
const submitted = await this.submitOneWay(
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2656
|
+
const submitted = await this.submitOneWay(
|
|
2657
|
+
route.targetNodeRid,
|
|
2658
|
+
instanceOperationParts(
|
|
2659
|
+
[
|
|
2660
|
+
encodeInstanceSpotActivationHeader(
|
|
2661
|
+
redirectedTarget,
|
|
2662
|
+
this.nodeGeneration,
|
|
2663
|
+
this.nodeRid,
|
|
2664
|
+
record.sourceSpotId,
|
|
2665
|
+
'send',
|
|
2666
|
+
record.operation,
|
|
2667
|
+
record.deadlineUnixMs,
|
|
2668
|
+
undefined,
|
|
2669
|
+
metadataFrame !== undefined
|
|
2670
|
+
),
|
|
2671
|
+
payloadFrame
|
|
2672
|
+
],
|
|
2673
|
+
metadataFrame
|
|
2674
|
+
)
|
|
2675
|
+
);
|
|
2637
2676
|
if (submitted !== SubmitResult.Ok) {
|
|
2638
2677
|
throw new Error(`Instance activation redirect was not admitted: ${submitted}.`);
|
|
2639
2678
|
}
|
|
@@ -2644,20 +2683,23 @@ export class ServiceStatefulRuntime {
|
|
|
2644
2683
|
this.submitRequest(
|
|
2645
2684
|
pending,
|
|
2646
2685
|
route.targetNodeRid,
|
|
2647
|
-
instanceOperationParts(
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2686
|
+
instanceOperationParts(
|
|
2687
|
+
[
|
|
2688
|
+
encodeInstanceSpotActivationHeader(
|
|
2689
|
+
redirectedTarget,
|
|
2690
|
+
this.nodeGeneration,
|
|
2691
|
+
this.nodeRid,
|
|
2692
|
+
record.sourceSpotId,
|
|
2693
|
+
'request',
|
|
2694
|
+
record.operation,
|
|
2695
|
+
record.deadlineUnixMs,
|
|
2696
|
+
pending.id,
|
|
2697
|
+
metadataFrame !== undefined
|
|
2698
|
+
),
|
|
2699
|
+
payloadFrame
|
|
2700
|
+
],
|
|
2701
|
+
metadataFrame
|
|
2702
|
+
),
|
|
2661
2703
|
remainingMs,
|
|
2662
2704
|
'instanceSpotRequest'
|
|
2663
2705
|
);
|
|
@@ -2682,16 +2724,18 @@ export class ServiceStatefulRuntime {
|
|
|
2682
2724
|
record: Extract<ServiceStatefulWireRecord, { readonly kind: 'instanceSpot' }>
|
|
2683
2725
|
): ServiceSpotState {
|
|
2684
2726
|
const readyRoute = record.activation === 'ready' ? record.route : undefined;
|
|
2685
|
-
const spotId =
|
|
2686
|
-
|
|
2727
|
+
const spotId =
|
|
2728
|
+
readyRoute?.targetSpotId ??
|
|
2729
|
+
(record.activation === 'missing' ? record.target.targetSpotId : undefined);
|
|
2687
2730
|
if (spotId === undefined) throw new ServiceStaleGenerationError('spot', 'unknown');
|
|
2688
2731
|
this.validateInstanceIngress(ingress, record);
|
|
2689
2732
|
if (record.activation === 'missing') {
|
|
2690
2733
|
return this.activateMissingInstance(record);
|
|
2691
2734
|
}
|
|
2692
2735
|
const { intent, route } = this.requireReadyInstanceApplicationRoute(ingress, record);
|
|
2693
|
-
const spot =
|
|
2694
|
-
|
|
2736
|
+
const spot =
|
|
2737
|
+
this.registry.spot(route.targetSpotId) ??
|
|
2738
|
+
this.registry.restoreSpot(
|
|
2695
2739
|
{
|
|
2696
2740
|
spotId: route.targetSpotId,
|
|
2697
2741
|
generation: route.objectGeneration
|
|
@@ -2701,11 +2745,11 @@ export class ServiceStatefulRuntime {
|
|
|
2701
2745
|
route.authorityOwnerGeneration
|
|
2702
2746
|
);
|
|
2703
2747
|
if (
|
|
2704
|
-
spot.state !== 'ready'
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2748
|
+
spot.state !== 'ready' ||
|
|
2749
|
+
spot.kind !== 'instance' ||
|
|
2750
|
+
spot.stableType !== intent.instanceType ||
|
|
2751
|
+
spot.ref.generation !== route.objectGeneration ||
|
|
2752
|
+
spot.authorityOwnerGeneration !== route.authorityOwnerGeneration
|
|
2709
2753
|
) {
|
|
2710
2754
|
throw createInternalFrameworkException(
|
|
2711
2755
|
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
@@ -2733,9 +2777,9 @@ export class ServiceStatefulRuntime {
|
|
|
2733
2777
|
// request. A mismatched or absent local projection remains NotFound.
|
|
2734
2778
|
const local = this.registry.spot(record.route.targetSpotId);
|
|
2735
2779
|
if (
|
|
2736
|
-
local?.kind === 'instance'
|
|
2737
|
-
|
|
2738
|
-
|
|
2780
|
+
local?.kind === 'instance' &&
|
|
2781
|
+
local.state === 'ready' &&
|
|
2782
|
+
routeMatchesLocal(record.route, local, this.nodeRid, this.nodeGeneration)
|
|
2739
2783
|
) {
|
|
2740
2784
|
this.publishCommittedInstanceRoute(local.stableType, record.route);
|
|
2741
2785
|
intent = this.instanceIntents.get(record.route.targetSpotId);
|
|
@@ -2761,8 +2805,8 @@ export class ServiceStatefulRuntime {
|
|
|
2761
2805
|
);
|
|
2762
2806
|
}
|
|
2763
2807
|
if (
|
|
2764
|
-
current.objectGeneration === record.route.objectGeneration
|
|
2765
|
-
|
|
2808
|
+
current.objectGeneration === record.route.objectGeneration &&
|
|
2809
|
+
current.authorityOwnerGeneration !== record.route.authorityOwnerGeneration
|
|
2766
2810
|
) {
|
|
2767
2811
|
throw createInternalFrameworkException(
|
|
2768
2812
|
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
@@ -2778,8 +2822,8 @@ export class ServiceStatefulRuntime {
|
|
|
2778
2822
|
): void {
|
|
2779
2823
|
const expectedSourceGeneration = this.peerGeneration(record.sourceNodeRid);
|
|
2780
2824
|
if (
|
|
2781
|
-
record.sourceNodeRid !== ingress.sourceRoutingId
|
|
2782
|
-
|
|
2825
|
+
record.sourceNodeRid !== ingress.sourceRoutingId ||
|
|
2826
|
+
record.sourceNodeGeneration !== expectedSourceGeneration
|
|
2783
2827
|
) {
|
|
2784
2828
|
if (record.activation === 'ready') {
|
|
2785
2829
|
throw createInternalFrameworkException(
|
|
@@ -2799,9 +2843,9 @@ export class ServiceStatefulRuntime {
|
|
|
2799
2843
|
): ServiceSpotState {
|
|
2800
2844
|
const target = record.target;
|
|
2801
2845
|
if (
|
|
2802
|
-
target.targetNodeRid !== this.nodeRid
|
|
2803
|
-
|
|
2804
|
-
|
|
2846
|
+
target.targetNodeRid !== this.nodeRid ||
|
|
2847
|
+
target.targetNodeGeneration !== this.nodeGeneration ||
|
|
2848
|
+
record.deadlineUnixMs < BigInt(Date.now())
|
|
2805
2849
|
) {
|
|
2806
2850
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2807
2851
|
}
|
|
@@ -2817,10 +2861,10 @@ export class ServiceStatefulRuntime {
|
|
|
2817
2861
|
const current = authority.read(target);
|
|
2818
2862
|
if (local !== undefined) {
|
|
2819
2863
|
if (
|
|
2820
|
-
local.kind !== 'instance'
|
|
2821
|
-
|
|
2822
|
-
|
|
2823
|
-
|
|
2864
|
+
local.kind !== 'instance' ||
|
|
2865
|
+
local.stableType !== target.stableType ||
|
|
2866
|
+
current.kind !== 'ready' ||
|
|
2867
|
+
!routeMatchesLocal(current.route, local, this.nodeRid, this.nodeGeneration)
|
|
2824
2868
|
) {
|
|
2825
2869
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2826
2870
|
}
|
|
@@ -2852,12 +2896,7 @@ export class ServiceStatefulRuntime {
|
|
|
2852
2896
|
if (activation.created) this.registry.closeSpot(activation.spot.ref);
|
|
2853
2897
|
throw new ServiceInstanceActivationRedirectError(committed.route);
|
|
2854
2898
|
}
|
|
2855
|
-
if (!routeMatchesLocal(
|
|
2856
|
-
committed.route,
|
|
2857
|
-
activation.spot,
|
|
2858
|
-
this.nodeRid,
|
|
2859
|
-
this.nodeGeneration
|
|
2860
|
-
)) {
|
|
2899
|
+
if (!routeMatchesLocal(committed.route, activation.spot, this.nodeRid, this.nodeGeneration)) {
|
|
2861
2900
|
if (activation.created) this.registry.closeSpot(activation.spot.ref);
|
|
2862
2901
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2863
2902
|
}
|
|
@@ -2896,9 +2935,9 @@ export class ServiceStatefulRuntime {
|
|
|
2896
2935
|
): Promise<{ readonly spot: ServiceSpotState; readonly route: ServiceInstanceRouteFence }> {
|
|
2897
2936
|
const target = record.target;
|
|
2898
2937
|
if (
|
|
2899
|
-
target.targetNodeRid !== this.nodeRid
|
|
2900
|
-
|
|
2901
|
-
|
|
2938
|
+
target.targetNodeRid !== this.nodeRid ||
|
|
2939
|
+
target.targetNodeGeneration !== this.nodeGeneration ||
|
|
2940
|
+
record.deadlineUnixMs < BigInt(Date.now())
|
|
2902
2941
|
) {
|
|
2903
2942
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2904
2943
|
}
|
|
@@ -2925,17 +2964,14 @@ export class ServiceStatefulRuntime {
|
|
|
2925
2964
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2926
2965
|
}
|
|
2927
2966
|
if (
|
|
2928
|
-
local !== undefined
|
|
2929
|
-
|
|
2930
|
-
local.kind !== 'instance'
|
|
2931
|
-
|| local.stableType !== target.stableType
|
|
2932
|
-
)
|
|
2967
|
+
local !== undefined &&
|
|
2968
|
+
(local.kind !== 'instance' || local.stableType !== target.stableType)
|
|
2933
2969
|
) {
|
|
2934
2970
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2935
2971
|
}
|
|
2936
2972
|
if (
|
|
2937
|
-
local !== undefined
|
|
2938
|
-
|
|
2973
|
+
local !== undefined &&
|
|
2974
|
+
routeMatchesLocal(current.route, local, this.nodeRid, this.nodeGeneration)
|
|
2939
2975
|
) {
|
|
2940
2976
|
return { spot: local, route: current.route };
|
|
2941
2977
|
}
|
|
@@ -2944,26 +2980,22 @@ export class ServiceStatefulRuntime {
|
|
|
2944
2980
|
if (local !== undefined) {
|
|
2945
2981
|
const lifecycle = this.instanceApplicationLifecycle;
|
|
2946
2982
|
const closing = lifecycle?.isClosing?.(target) === true;
|
|
2947
|
-
let materialized = lifecycle === undefined
|
|
2948
|
-
? undefined
|
|
2949
|
-
: lifecycle.isMaterialized(target);
|
|
2983
|
+
let materialized = lifecycle === undefined ? undefined : lifecycle.isMaterialized(target);
|
|
2950
2984
|
const materializing = lifecycle?.isMaterializing?.(target) === true;
|
|
2951
|
-
if (
|
|
2952
|
-
local.kind !== 'instance'
|
|
2953
|
-
|| local.stableType !== target.stableType
|
|
2954
|
-
) {
|
|
2985
|
+
if (local.kind !== 'instance' || local.stableType !== target.stableType) {
|
|
2955
2986
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2956
2987
|
}
|
|
2957
|
-
const joinsCreating =
|
|
2958
|
-
|
|
2959
|
-
|
|
2988
|
+
const joinsCreating =
|
|
2989
|
+
current.kind === 'creating' &&
|
|
2990
|
+
local.ref.generation === current.objectGeneration &&
|
|
2991
|
+
local.authorityOwnerGeneration === current.authorityOwnerGeneration;
|
|
2960
2992
|
const replacesClosingProjection = current.kind === 'creating' && closing;
|
|
2961
2993
|
if (
|
|
2962
|
-
current.kind === 'missing'
|
|
2963
|
-
|
|
2964
|
-
|
|
2965
|
-
|
|
2966
|
-
|
|
2994
|
+
current.kind === 'missing' &&
|
|
2995
|
+
!closing &&
|
|
2996
|
+
!materializing &&
|
|
2997
|
+
materialized === true &&
|
|
2998
|
+
lifecycle !== undefined
|
|
2967
2999
|
) {
|
|
2968
3000
|
// The durable authority is the ownership source of truth. Its removal
|
|
2969
3001
|
// can become visible before the route reconciler has discarded the
|
|
@@ -2984,8 +3016,8 @@ export class ServiceStatefulRuntime {
|
|
|
2984
3016
|
// If disposal did not remove an orphaned materialization, keep fencing
|
|
2985
3017
|
// it instead of admitting work into two application generations.
|
|
2986
3018
|
if (
|
|
2987
|
-
(!joinsCreating && !replacesClosingProjection && current.kind !== 'missing')
|
|
2988
|
-
|
|
3019
|
+
(!joinsCreating && !replacesClosingProjection && current.kind !== 'missing') ||
|
|
3020
|
+
(!joinsCreating && !closing && materialized !== false && !materializing)
|
|
2989
3021
|
) {
|
|
2990
3022
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
2991
3023
|
}
|
|
@@ -3037,12 +3069,7 @@ export class ServiceStatefulRuntime {
|
|
|
3037
3069
|
await this.instanceApplicationLifecycle?.discard(target);
|
|
3038
3070
|
throw new ServiceInstanceActivationRedirectError(committed.route);
|
|
3039
3071
|
}
|
|
3040
|
-
if (!routeMatchesLocal(
|
|
3041
|
-
committed.route,
|
|
3042
|
-
activation.spot,
|
|
3043
|
-
this.nodeRid,
|
|
3044
|
-
this.nodeGeneration
|
|
3045
|
-
)) {
|
|
3072
|
+
if (!routeMatchesLocal(committed.route, activation.spot, this.nodeRid, this.nodeGeneration)) {
|
|
3046
3073
|
if (activation.created) this.registry.closeSpot(activation.spot.ref);
|
|
3047
3074
|
await this.instanceApplicationLifecycle?.discard(target);
|
|
3048
3075
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
@@ -3059,8 +3086,9 @@ export class ServiceStatefulRuntime {
|
|
|
3059
3086
|
};
|
|
3060
3087
|
};
|
|
3061
3088
|
const descriptor = topology.localDescriptor?.();
|
|
3062
|
-
return
|
|
3063
|
-
|| (descriptor.state === 'serving' && descriptor.placementWeight > 0)
|
|
3089
|
+
return (
|
|
3090
|
+
descriptor === undefined || (descriptor.state === 'serving' && descriptor.placementWeight > 0)
|
|
3091
|
+
);
|
|
3064
3092
|
}
|
|
3065
3093
|
|
|
3066
3094
|
private async reconcileReadyInstanceProjection(
|
|
@@ -3075,12 +3103,7 @@ export class ServiceStatefulRuntime {
|
|
|
3075
3103
|
route.objectGeneration,
|
|
3076
3104
|
route.authorityOwnerGeneration
|
|
3077
3105
|
);
|
|
3078
|
-
if (!routeMatchesLocal(
|
|
3079
|
-
route,
|
|
3080
|
-
activation.spot,
|
|
3081
|
-
this.nodeRid,
|
|
3082
|
-
this.nodeGeneration
|
|
3083
|
-
)) {
|
|
3106
|
+
if (!routeMatchesLocal(route, activation.spot, this.nodeRid, this.nodeGeneration)) {
|
|
3084
3107
|
if (activation.created) this.registry.closeSpot(activation.spot.ref);
|
|
3085
3108
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
3086
3109
|
}
|
|
@@ -3088,18 +3111,15 @@ export class ServiceStatefulRuntime {
|
|
|
3088
3111
|
|
|
3089
3112
|
const confirmed = await authority.read(target);
|
|
3090
3113
|
if (confirmed.kind !== 'ready' || !sameInstanceRoute(confirmed.route, route)) {
|
|
3091
|
-
if (
|
|
3092
|
-
confirmed.kind === 'ready'
|
|
3093
|
-
&& confirmed.route.targetNodeRid !== this.nodeRid
|
|
3094
|
-
) {
|
|
3114
|
+
if (confirmed.kind === 'ready' && confirmed.route.targetNodeRid !== this.nodeRid) {
|
|
3095
3115
|
throw new ServiceInstanceActivationRedirectError(confirmed.route);
|
|
3096
3116
|
}
|
|
3097
3117
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
3098
3118
|
}
|
|
3099
3119
|
const current = this.registry.spot(target.targetSpotId);
|
|
3100
3120
|
if (
|
|
3101
|
-
current === undefined
|
|
3102
|
-
|
|
3121
|
+
current === undefined ||
|
|
3122
|
+
!routeMatchesLocal(confirmed.route, current, this.nodeRid, this.nodeGeneration)
|
|
3103
3123
|
) {
|
|
3104
3124
|
throw new ServiceStaleGenerationError('spot', target.targetSpotId);
|
|
3105
3125
|
}
|
|
@@ -3112,11 +3132,12 @@ export class ServiceStatefulRuntime {
|
|
|
3112
3132
|
expectedCurrentRoute?: ServiceInstanceRouteFence | null
|
|
3113
3133
|
): void {
|
|
3114
3134
|
const directRoute = instanceRouteAsDirectRoute(route);
|
|
3115
|
-
const expectedDirectRoute =
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3135
|
+
const expectedDirectRoute =
|
|
3136
|
+
expectedCurrentRoute === undefined
|
|
3137
|
+
? undefined
|
|
3138
|
+
: expectedCurrentRoute === null
|
|
3139
|
+
? null
|
|
3140
|
+
: instanceRouteAsDirectRoute(expectedCurrentRoute);
|
|
3120
3141
|
this.rememberSpotRoute(directRoute, expectedDirectRoute);
|
|
3121
3142
|
this.registerInstanceIntent(stableType, route, expectedCurrentRoute);
|
|
3122
3143
|
}
|
|
@@ -3150,9 +3171,9 @@ export class ServiceStatefulRuntime {
|
|
|
3150
3171
|
private forgetClosedInstanceRoute(route: ServiceInstanceRouteFence): void {
|
|
3151
3172
|
const local = this.registry.spot(route.targetSpotId);
|
|
3152
3173
|
if (
|
|
3153
|
-
local?.kind === 'instance'
|
|
3154
|
-
|
|
3155
|
-
|
|
3174
|
+
local?.kind === 'instance' &&
|
|
3175
|
+
local.ref.generation === route.objectGeneration &&
|
|
3176
|
+
local.authorityOwnerGeneration === route.authorityOwnerGeneration
|
|
3156
3177
|
) {
|
|
3157
3178
|
this.registry.closeSpot(local.ref);
|
|
3158
3179
|
}
|
|
@@ -3279,8 +3300,8 @@ export class ServiceStatefulRuntime {
|
|
|
3279
3300
|
if (record.binding.state === 'active') {
|
|
3280
3301
|
const actor = this.validateActorFence(record.actor);
|
|
3281
3302
|
const previous = this.registry.binding(actor.ref);
|
|
3282
|
-
const sourceGeneration =
|
|
3283
|
-
?? this.peerGeneration(ingress.sourceRoutingId);
|
|
3303
|
+
const sourceGeneration =
|
|
3304
|
+
ingress.sourceNodeGeneration ?? this.peerGeneration(ingress.sourceRoutingId);
|
|
3284
3305
|
const binding = sessionBindingFromWire(
|
|
3285
3306
|
actor.ref,
|
|
3286
3307
|
record.sessionRid,
|
|
@@ -3337,9 +3358,9 @@ export class ServiceStatefulRuntime {
|
|
|
3337
3358
|
: undefined
|
|
3338
3359
|
);
|
|
3339
3360
|
if (
|
|
3340
|
-
terminalResult === RequestResult.Ok
|
|
3341
|
-
|
|
3342
|
-
|
|
3361
|
+
terminalResult === RequestResult.Ok &&
|
|
3362
|
+
replacedBinding !== undefined &&
|
|
3363
|
+
!sameSessionBinding(replacedBinding, completedBinding)
|
|
3343
3364
|
) {
|
|
3344
3365
|
this.sendBoundSessionReplacement(
|
|
3345
3366
|
replacedBinding,
|
|
@@ -3353,16 +3374,13 @@ export class ServiceStatefulRuntime {
|
|
|
3353
3374
|
}
|
|
3354
3375
|
);
|
|
3355
3376
|
if (!accepted) {
|
|
3356
|
-
const result = failure(
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
ingress,
|
|
3362
|
-
record.correlation,
|
|
3363
|
-
result.terminalResult,
|
|
3364
|
-
result.failureCode
|
|
3377
|
+
const result = failure(
|
|
3378
|
+
createInternalFrameworkException(
|
|
3379
|
+
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
3380
|
+
`Actor '${actor.ref.actorId}' no longer accepts session bindings.`
|
|
3381
|
+
)
|
|
3365
3382
|
);
|
|
3383
|
+
this.replyWire(ingress, record.correlation, result.terminalResult, result.failureCode);
|
|
3366
3384
|
}
|
|
3367
3385
|
} else {
|
|
3368
3386
|
this.enqueueSessionBindingTombstone(ingress, record);
|
|
@@ -3388,8 +3406,8 @@ export class ServiceStatefulRuntime {
|
|
|
3388
3406
|
if (authority.actor.nodeRid !== ingress.sourceRoutingId) {
|
|
3389
3407
|
return 'protocolError';
|
|
3390
3408
|
}
|
|
3391
|
-
const sourceGeneration =
|
|
3392
|
-
?? this.tryPeerGeneration(ingress.sourceRoutingId);
|
|
3409
|
+
const sourceGeneration =
|
|
3410
|
+
ingress.sourceNodeGeneration ?? this.tryPeerGeneration(ingress.sourceRoutingId);
|
|
3393
3411
|
if (sourceGeneration === undefined) return 'infrastructure';
|
|
3394
3412
|
if (sourceGeneration !== authority.targetNodeGeneration) {
|
|
3395
3413
|
// A notice from a restarted authority is stale. It must not tear down a
|
|
@@ -3397,10 +3415,10 @@ export class ServiceStatefulRuntime {
|
|
|
3397
3415
|
return 'infrastructure';
|
|
3398
3416
|
}
|
|
3399
3417
|
if (
|
|
3400
|
-
record.retiredSession.sessionOwnerNodeRid !== this.nodeRid
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
|
|
3418
|
+
record.retiredSession.sessionOwnerNodeRid !== this.nodeRid ||
|
|
3419
|
+
record.retiredSession.sessionOwnerNodeGeneration !== this.nodeGeneration ||
|
|
3420
|
+
record.retiredSession.sessionOwnerId !== this.nodeRid ||
|
|
3421
|
+
record.retiredSession.sessionOwnerLeaseGeneration !== this.nodeGeneration
|
|
3404
3422
|
) {
|
|
3405
3423
|
return 'infrastructure';
|
|
3406
3424
|
}
|
|
@@ -3417,16 +3435,15 @@ export class ServiceStatefulRuntime {
|
|
|
3417
3435
|
sessionOwnerNodeGeneration: record.retiredSession.sessionOwnerNodeGeneration,
|
|
3418
3436
|
sessionOwnerId: record.retiredSession.sessionOwnerId,
|
|
3419
3437
|
sessionOwnerLeaseGeneration: record.retiredSession.sessionOwnerLeaseGeneration,
|
|
3420
|
-
bindingGeneration: record.retiredSession.retiredBindingGeneration
|
|
3438
|
+
bindingGeneration: record.retiredSession.retiredBindingGeneration
|
|
3421
3439
|
})
|
|
3422
3440
|
);
|
|
3423
|
-
const delivery =
|
|
3424
|
-
&& exactRetiredDelivery(current.binding, record.retiredSession)
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
: undefined;
|
|
3441
|
+
const delivery =
|
|
3442
|
+
current !== undefined && exactRetiredDelivery(current.binding, record.retiredSession)
|
|
3443
|
+
? current
|
|
3444
|
+
: retired !== undefined && exactRetiredDelivery(retired.binding, record.retiredSession)
|
|
3445
|
+
? retired
|
|
3446
|
+
: undefined;
|
|
3430
3447
|
if (delivery === undefined) return 'infrastructure';
|
|
3431
3448
|
this.appliedReplacementNotices.add(key);
|
|
3432
3449
|
delivery.onBindingReplaced?.(actor.actorId, record.retiredSession, actor);
|
|
@@ -3440,20 +3457,17 @@ export class ServiceStatefulRuntime {
|
|
|
3440
3457
|
previous: ServiceSessionBinding,
|
|
3441
3458
|
actorAuthority: ServiceBoundSessionActorAuthority
|
|
3442
3459
|
): void {
|
|
3443
|
-
const ownerGeneration =
|
|
3444
|
-
?? this.tryPeerGeneration(previous.sessionOwnerNodeRid);
|
|
3460
|
+
const ownerGeneration =
|
|
3461
|
+
previous.sessionOwnerNodeGeneration ?? this.tryPeerGeneration(previous.sessionOwnerNodeRid);
|
|
3445
3462
|
if (ownerGeneration === undefined) return;
|
|
3446
|
-
const header = encodeBoundSessionReplacedHeader(
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
retiredBindingGeneration: previous.bindingGeneration
|
|
3455
|
-
}
|
|
3456
|
-
);
|
|
3463
|
+
const header = encodeBoundSessionReplacedHeader(actorAuthority, {
|
|
3464
|
+
sessionOwnerNodeRid: previous.sessionOwnerNodeRid,
|
|
3465
|
+
sessionOwnerNodeGeneration: ownerGeneration,
|
|
3466
|
+
sessionOwnerId: previous.sessionOwnerId ?? previous.sessionOwnerNodeRid,
|
|
3467
|
+
sessionOwnerLeaseGeneration: previous.sessionOwnerLeaseGeneration ?? ownerGeneration,
|
|
3468
|
+
sessionRid: previous.sessionRid,
|
|
3469
|
+
retiredBindingGeneration: previous.bindingGeneration
|
|
3470
|
+
});
|
|
3457
3471
|
// submitOneWay owns the normal transport admission timeout. A replacement
|
|
3458
3472
|
// notice is one-way, so an unsuccessful admission has no caller terminal
|
|
3459
3473
|
// to complete and must not delay the already-completed bind.
|
|
@@ -3467,26 +3481,26 @@ export class ServiceStatefulRuntime {
|
|
|
3467
3481
|
if (record.binding.state !== 'tombstone') return;
|
|
3468
3482
|
const actor = record.actor.actor;
|
|
3469
3483
|
const current = this.registry.binding(actor);
|
|
3470
|
-
const sourceGeneration =
|
|
3471
|
-
?? this.tryPeerGeneration(ingress.sourceRoutingId);
|
|
3484
|
+
const sourceGeneration =
|
|
3485
|
+
ingress.sourceNodeGeneration ?? this.tryPeerGeneration(ingress.sourceRoutingId);
|
|
3472
3486
|
if (
|
|
3473
|
-
current === undefined
|
|
3474
|
-
|
|
3475
|
-
|
|
3476
|
-
|
|
3477
|
-
|
|
3478
|
-
|
|
3479
|
-
|
|
3480
|
-
|
|
3487
|
+
current === undefined ||
|
|
3488
|
+
sourceGeneration === undefined ||
|
|
3489
|
+
current.bindingGeneration !== record.binding.retiredGeneration ||
|
|
3490
|
+
current.sessionRid !== record.sessionRid ||
|
|
3491
|
+
current.sessionOwnerNodeRid !== ingress.sourceRoutingId ||
|
|
3492
|
+
current.sessionOwnerNodeGeneration !== sourceGeneration ||
|
|
3493
|
+
current.sessionOwnerId !== ingress.sourceRoutingId ||
|
|
3494
|
+
current.sessionOwnerLeaseGeneration !== sourceGeneration
|
|
3481
3495
|
) {
|
|
3482
3496
|
this.replyWire(ingress, record.correlation, RequestResult.Ok, 0);
|
|
3483
3497
|
return;
|
|
3484
3498
|
}
|
|
3485
3499
|
const delivery = this.sessionDeliveries.get(actorKey(actor));
|
|
3486
|
-
const currentDelivery =
|
|
3487
|
-
&& sameSessionBinding(delivery.binding, current)
|
|
3488
|
-
|
|
3489
|
-
|
|
3500
|
+
const currentDelivery =
|
|
3501
|
+
delivery !== undefined && sameSessionBinding(delivery.binding, current)
|
|
3502
|
+
? delivery
|
|
3503
|
+
: undefined;
|
|
3490
3504
|
const accepted = this.enqueueSessionBindingRetirement(
|
|
3491
3505
|
ingress,
|
|
3492
3506
|
current,
|
|
@@ -3503,8 +3517,8 @@ export class ServiceStatefulRuntime {
|
|
|
3503
3517
|
);
|
|
3504
3518
|
}
|
|
3505
3519
|
if (
|
|
3506
|
-
currentDelivery !== undefined
|
|
3507
|
-
|
|
3520
|
+
currentDelivery !== undefined &&
|
|
3521
|
+
this.sessionDeliveries.get(actorKey(actor)) === currentDelivery
|
|
3508
3522
|
) {
|
|
3509
3523
|
this.sessionDeliveries.delete(actorKey(actor));
|
|
3510
3524
|
}
|
|
@@ -3514,10 +3528,12 @@ export class ServiceStatefulRuntime {
|
|
|
3514
3528
|
}
|
|
3515
3529
|
);
|
|
3516
3530
|
if (!accepted) {
|
|
3517
|
-
const result = failure(
|
|
3518
|
-
|
|
3519
|
-
|
|
3520
|
-
|
|
3531
|
+
const result = failure(
|
|
3532
|
+
createInternalFrameworkException(
|
|
3533
|
+
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
3534
|
+
`Actor '${actor.actorId}' no longer accepts session binding tombstones.`
|
|
3535
|
+
)
|
|
3536
|
+
);
|
|
3521
3537
|
this.replyWire(ingress, record.correlation, result.terminalResult, result.failureCode);
|
|
3522
3538
|
}
|
|
3523
3539
|
}
|
|
@@ -3569,12 +3585,12 @@ export class ServiceStatefulRuntime {
|
|
|
3569
3585
|
): Promise<RawServicePumpResult> {
|
|
3570
3586
|
const delivery = this.sessionDeliveries.get(actorKey(record.actor.actor));
|
|
3571
3587
|
if (
|
|
3572
|
-
delivery === undefined
|
|
3573
|
-
|
|
3574
|
-
|
|
3575
|
-
|
|
3576
|
-
|
|
3577
|
-
|
|
3588
|
+
delivery === undefined ||
|
|
3589
|
+
delivery.binding.bindingGeneration !== record.expectedBindingGeneration ||
|
|
3590
|
+
payloadFrame === undefined ||
|
|
3591
|
+
ingress.sourceNodeGeneration === undefined ||
|
|
3592
|
+
!routingIdsEqual(ingress.sourceRoutingId, record.actor.actor.nodeRid) ||
|
|
3593
|
+
ingress.sourceNodeGeneration !== record.actor.targetNodeGeneration
|
|
3578
3594
|
) {
|
|
3579
3595
|
return 'protocolError';
|
|
3580
3596
|
}
|
|
@@ -3598,18 +3614,21 @@ export class ServiceStatefulRuntime {
|
|
|
3598
3614
|
settle();
|
|
3599
3615
|
}
|
|
3600
3616
|
};
|
|
3601
|
-
const decision = await delivery.bindingIngress?.retainOutbound(
|
|
3602
|
-
|
|
3603
|
-
|
|
3604
|
-
|
|
3605
|
-
|
|
3606
|
-
|
|
3607
|
-
|
|
3608
|
-
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3617
|
+
const decision = await delivery.bindingIngress?.retainOutbound(
|
|
3618
|
+
{
|
|
3619
|
+
actorId: record.actor.actor.actorId,
|
|
3620
|
+
objectGeneration: record.actor.actor.generation,
|
|
3621
|
+
actorNodeRid: record.actor.actor.nodeRid,
|
|
3622
|
+
actorNodeGeneration: record.actor.targetNodeGeneration,
|
|
3623
|
+
authorityOwnerGeneration: record.actor.authorityOwnerGeneration,
|
|
3624
|
+
ownerLeaseGeneration: record.actor.ownerLeaseGeneration,
|
|
3625
|
+
producerNodeRid: ingress.sourceRoutingId,
|
|
3626
|
+
producerNodeGeneration: ingress.sourceNodeGeneration,
|
|
3627
|
+
sessionIdentity: sessionRid,
|
|
3628
|
+
bindingGeneration: record.expectedBindingGeneration
|
|
3629
|
+
},
|
|
3630
|
+
operation
|
|
3631
|
+
);
|
|
3613
3632
|
if (decision === 'retained') return 'application';
|
|
3614
3633
|
if (decision === 'rejected') {
|
|
3615
3634
|
return 'protocolError';
|
|
@@ -3624,9 +3643,9 @@ export class ServiceStatefulRuntime {
|
|
|
3624
3643
|
// already completed, so a later client delivery terminal is local to
|
|
3625
3644
|
// this session owner. Local fast paths still preserve the typed result.
|
|
3626
3645
|
if (
|
|
3627
|
-
ingress.sourceRoutingId !== this.nodeRid
|
|
3628
|
-
|
|
3629
|
-
|
|
3646
|
+
ingress.sourceRoutingId !== this.nodeRid &&
|
|
3647
|
+
isZLinkBackendResultError(error) &&
|
|
3648
|
+
error.operation === 'submit'
|
|
3630
3649
|
) {
|
|
3631
3650
|
return 'application';
|
|
3632
3651
|
}
|
|
@@ -3668,7 +3687,9 @@ export class ServiceStatefulRuntime {
|
|
|
3668
3687
|
sourceRoutingId: ingress.sourceRoutingId,
|
|
3669
3688
|
sourceRoute: ingress.sourceRoute,
|
|
3670
3689
|
...(ingress.reply === undefined ? {} : { reply: ingress.reply }),
|
|
3671
|
-
...(ingress.requestSequence === undefined
|
|
3690
|
+
...(ingress.requestSequence === undefined
|
|
3691
|
+
? {}
|
|
3692
|
+
: { requestSequence: ingress.requestSequence }),
|
|
3672
3693
|
...(stateful.correlation === undefined ? {} : { correlation: stateful.correlation }),
|
|
3673
3694
|
stateful,
|
|
3674
3695
|
applicationJob
|
|
@@ -3676,10 +3697,12 @@ export class ServiceStatefulRuntime {
|
|
|
3676
3697
|
if (accepted) return domain;
|
|
3677
3698
|
applicationJob.close();
|
|
3678
3699
|
if (stateful.reply !== undefined) {
|
|
3679
|
-
const result = failure(
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
|
|
3700
|
+
const result = failure(
|
|
3701
|
+
createInternalFrameworkException(
|
|
3702
|
+
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
3703
|
+
`Mailbox owner '${owner}' no longer accepts work on this runtime.`
|
|
3704
|
+
)
|
|
3705
|
+
);
|
|
3683
3706
|
stateful.reply(result.terminalResult, result.failureCode);
|
|
3684
3707
|
return 'infrastructure';
|
|
3685
3708
|
}
|
|
@@ -3717,11 +3740,15 @@ export class ServiceStatefulRuntime {
|
|
|
3717
3740
|
applicationJobOwner: NonNullable<RawServiceIngressRecord['applicationJobOwner']>
|
|
3718
3741
|
): Promise<void> {
|
|
3719
3742
|
const targets = [...this.subscriptions.entries()]
|
|
3720
|
-
.filter(([, values]) =>
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3743
|
+
.filter(([, values]) =>
|
|
3744
|
+
[...values].some((value) => {
|
|
3745
|
+
const separator = value.indexOf('\0');
|
|
3746
|
+
return (
|
|
3747
|
+
value.slice(0, separator) === channelName &&
|
|
3748
|
+
topicMatches(value.slice(separator + 1), topic)
|
|
3749
|
+
);
|
|
3750
|
+
})
|
|
3751
|
+
)
|
|
3725
3752
|
.map(([spotId]) => spotId)
|
|
3726
3753
|
.sort();
|
|
3727
3754
|
for (const spotId of targets) {
|
|
@@ -3731,10 +3758,7 @@ export class ServiceStatefulRuntime {
|
|
|
3731
3758
|
const accepted = this.raw.mailbox.tryEnqueue({
|
|
3732
3759
|
owner: `spot:${spotId}`,
|
|
3733
3760
|
domain: 'application',
|
|
3734
|
-
parts: [
|
|
3735
|
-
encodeLogicalMulticastHeader(channelName, topic, sourceSpotId),
|
|
3736
|
-
payloadFrame
|
|
3737
|
-
],
|
|
3761
|
+
parts: [encodeLogicalMulticastHeader(channelName, topic, sourceSpotId), payloadFrame],
|
|
3738
3762
|
sourceRoutingId: sourceNodeRid,
|
|
3739
3763
|
stateful: {
|
|
3740
3764
|
receiveKind: ReceiveKind.SpotMulticast,
|
|
@@ -3843,10 +3867,10 @@ export class ServiceStatefulRuntime {
|
|
|
3843
3867
|
private replyPort(
|
|
3844
3868
|
ingress: RawServiceIngressRecord,
|
|
3845
3869
|
correlation: bigint | undefined,
|
|
3846
|
-
operationKind: 'spotRequest' | 'actorRequest' | 'actorJoin'
|
|
3847
|
-
| 'instanceSpotRequest'
|
|
3870
|
+
operationKind: 'spotRequest' | 'actorRequest' | 'actorJoin' | 'instanceSpotRequest'
|
|
3848
3871
|
): NonNullable<ServiceStatefulMailboxData['reply']> {
|
|
3849
|
-
if (correlation === undefined)
|
|
3872
|
+
if (correlation === undefined)
|
|
3873
|
+
throw new ServiceWireProtocolError('Request correlation is missing.');
|
|
3850
3874
|
return (terminalResult, failureCode, payload, tail) => {
|
|
3851
3875
|
this.replyWire(ingress, correlation, terminalResult, failureCode, payload, tail);
|
|
3852
3876
|
void operationKind;
|
|
@@ -3873,26 +3897,22 @@ export class ServiceStatefulRuntime {
|
|
|
3873
3897
|
record: ServiceUserSpotCreateRecord | ServiceUserSpotCloseRecord | ServiceActorCreateRecord
|
|
3874
3898
|
): RawServicePumpResult {
|
|
3875
3899
|
if (
|
|
3876
|
-
ingress.requestSequence === undefined
|
|
3877
|
-
|
|
3878
|
-
|
|
3879
|
-
|
|
3900
|
+
ingress.requestSequence === undefined ||
|
|
3901
|
+
record.sourceNodeRid !== ingress.sourceRoutingId ||
|
|
3902
|
+
record.sourceNodeGeneration !== this.peerGeneration(ingress.sourceRoutingId) ||
|
|
3903
|
+
(record.kind === 'actorCreate' && record.reservation.pendingCapacityDelta !== 1)
|
|
3880
3904
|
) {
|
|
3881
3905
|
return 'protocolError';
|
|
3882
3906
|
}
|
|
3883
|
-
const target =
|
|
3884
|
-
|
|
3885
|
-
|
|
3907
|
+
const target =
|
|
3908
|
+
record.kind === 'userSpotCreate' || record.kind === 'actorCreate'
|
|
3909
|
+
? record.reservation
|
|
3910
|
+
: record.target;
|
|
3886
3911
|
if (
|
|
3887
|
-
target.targetNodeRid !== this.nodeRid
|
|
3888
|
-
|
|
3912
|
+
target.targetNodeRid !== this.nodeRid ||
|
|
3913
|
+
target.targetNodeGeneration !== this.nodeGeneration
|
|
3889
3914
|
) {
|
|
3890
|
-
this.replyWire(
|
|
3891
|
-
ingress,
|
|
3892
|
-
record.correlation,
|
|
3893
|
-
RequestResult.Conflict,
|
|
3894
|
-
SPOT_MOVING
|
|
3895
|
-
);
|
|
3915
|
+
this.replyWire(ingress, record.correlation, RequestResult.Conflict, SPOT_MOVING);
|
|
3896
3916
|
return 'infrastructure';
|
|
3897
3917
|
}
|
|
3898
3918
|
const operationKey = [
|
|
@@ -3903,10 +3923,7 @@ export class ServiceStatefulRuntime {
|
|
|
3903
3923
|
].join('\0');
|
|
3904
3924
|
const encodedRequest = userSpotOperationFingerprint(record);
|
|
3905
3925
|
let admitted = this.admittedUserSpotOperations.get(operationKey);
|
|
3906
|
-
if (
|
|
3907
|
-
admitted?.settled === true
|
|
3908
|
-
&& operationReplayExpired(admitted.replayExpiresAtMs)
|
|
3909
|
-
) {
|
|
3926
|
+
if (admitted?.settled === true && operationReplayExpired(admitted.replayExpiresAtMs)) {
|
|
3910
3927
|
this.admittedUserSpotOperations.delete(operationKey);
|
|
3911
3928
|
admitted = undefined;
|
|
3912
3929
|
}
|
|
@@ -3915,36 +3932,28 @@ export class ServiceStatefulRuntime {
|
|
|
3915
3932
|
}
|
|
3916
3933
|
if (admitted === undefined) {
|
|
3917
3934
|
if (record.deadlineUnixMs < BigInt(Date.now())) {
|
|
3918
|
-
this.replyWire(
|
|
3919
|
-
ingress,
|
|
3920
|
-
record.correlation,
|
|
3921
|
-
RequestResult.TimedOut,
|
|
3922
|
-
0
|
|
3923
|
-
);
|
|
3935
|
+
this.replyWire(ingress, record.correlation, RequestResult.TimedOut, 0);
|
|
3924
3936
|
return 'infrastructure';
|
|
3925
3937
|
}
|
|
3926
3938
|
if (this.admittedUserSpotOperations.size >= USER_SPOT_OPERATION_CAPACITY) {
|
|
3927
3939
|
this.releaseExpiredUserSpotOperations();
|
|
3928
3940
|
}
|
|
3929
3941
|
if (this.admittedUserSpotOperations.size >= USER_SPOT_OPERATION_CAPACITY) {
|
|
3930
|
-
this.replyWire(
|
|
3931
|
-
ingress,
|
|
3932
|
-
record.correlation,
|
|
3933
|
-
RequestResult.Busy,
|
|
3934
|
-
0
|
|
3935
|
-
);
|
|
3942
|
+
this.replyWire(ingress, record.correlation, RequestResult.Busy, 0);
|
|
3936
3943
|
return 'infrastructure';
|
|
3937
3944
|
}
|
|
3938
|
-
const replayExpiresAtMs =
|
|
3939
|
-
|
|
3940
|
-
+
|
|
3945
|
+
const replayExpiresAtMs =
|
|
3946
|
+
performance.now() +
|
|
3947
|
+
Number(record.deadlineUnixMs - BigInt(Date.now())) +
|
|
3948
|
+
USER_SPOT_OPERATION_REPLAY_RETENTION_MS;
|
|
3941
3949
|
const handler = this.userSpotOperationHandler;
|
|
3942
|
-
const result =
|
|
3943
|
-
|
|
3944
|
-
|
|
3945
|
-
|
|
3946
|
-
|
|
3947
|
-
|
|
3950
|
+
const result =
|
|
3951
|
+
handler === undefined
|
|
3952
|
+
? Promise.resolve<ServiceUserSpotOperationResult>({
|
|
3953
|
+
terminalResult: RequestResult.InvalidState,
|
|
3954
|
+
failureCode: 0
|
|
3955
|
+
})
|
|
3956
|
+
: this.executeUserSpotOperation(handler, record);
|
|
3948
3957
|
admitted = {
|
|
3949
3958
|
request: encodedRequest,
|
|
3950
3959
|
replayExpiresAtMs,
|
|
@@ -3952,13 +3961,15 @@ export class ServiceStatefulRuntime {
|
|
|
3952
3961
|
settled: false
|
|
3953
3962
|
};
|
|
3954
3963
|
this.admittedUserSpotOperations.set(operationKey, admitted);
|
|
3955
|
-
void result
|
|
3956
|
-
|
|
3957
|
-
|
|
3958
|
-
|
|
3964
|
+
void result
|
|
3965
|
+
.finally(() => {
|
|
3966
|
+
const current = this.admittedUserSpotOperations.get(operationKey);
|
|
3967
|
+
if (current?.result === result) current.settled = true;
|
|
3968
|
+
})
|
|
3969
|
+
.catch(() => undefined);
|
|
3959
3970
|
}
|
|
3960
3971
|
void admitted.result.then(
|
|
3961
|
-
result => {
|
|
3972
|
+
(result) => {
|
|
3962
3973
|
if (this.closed) return;
|
|
3963
3974
|
this.raw.replyService(ingress, [
|
|
3964
3975
|
encodeStatefulReply(
|
|
@@ -3967,20 +3978,13 @@ export class ServiceStatefulRuntime {
|
|
|
3967
3978
|
result.failureCode,
|
|
3968
3979
|
result.tail
|
|
3969
3980
|
),
|
|
3970
|
-
...(result.payload === undefined
|
|
3971
|
-
? []
|
|
3972
|
-
: [encodeApplicationPayload(result.payload)])
|
|
3981
|
+
...(result.payload === undefined ? [] : [encodeApplicationPayload(result.payload)])
|
|
3973
3982
|
]);
|
|
3974
3983
|
},
|
|
3975
|
-
error => {
|
|
3984
|
+
(error) => {
|
|
3976
3985
|
if (this.closed) return;
|
|
3977
3986
|
const failed = failure(error);
|
|
3978
|
-
this.replyWire(
|
|
3979
|
-
ingress,
|
|
3980
|
-
record.correlation,
|
|
3981
|
-
failed.terminalResult,
|
|
3982
|
-
failed.failureCode
|
|
3983
|
-
);
|
|
3987
|
+
this.replyWire(ingress, record.correlation, failed.terminalResult, failed.failureCode);
|
|
3984
3988
|
}
|
|
3985
3989
|
);
|
|
3986
3990
|
return 'infrastructure';
|
|
@@ -4000,22 +4004,20 @@ export class ServiceStatefulRuntime {
|
|
|
4000
4004
|
): Promise<ServiceUserSpotOperationResult> {
|
|
4001
4005
|
const deadline = userSpotDeadline(record.deadlineUnixMs);
|
|
4002
4006
|
try {
|
|
4003
|
-
const result =
|
|
4004
|
-
|
|
4005
|
-
|
|
4006
|
-
|
|
4007
|
-
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
|
|
4012
|
-
|
|
4007
|
+
const result =
|
|
4008
|
+
record.kind === 'userSpotCreate'
|
|
4009
|
+
? await handler.create(record, deadline.signal)
|
|
4010
|
+
: record.kind === 'userSpotClose'
|
|
4011
|
+
? await handler.close(record, deadline.signal)
|
|
4012
|
+
: handler.createActor === undefined
|
|
4013
|
+
? {
|
|
4014
|
+
terminalResult: RequestResult.InvalidState,
|
|
4015
|
+
failureCode: 0
|
|
4016
|
+
}
|
|
4017
|
+
: await handler.createActor(record, deadline.signal);
|
|
4013
4018
|
if (
|
|
4014
|
-
result.terminalResult === RequestResult.Ok
|
|
4015
|
-
|
|
4016
|
-
result.tail === undefined
|
|
4017
|
-
|| result.tail.kind !== record.kind
|
|
4018
|
-
)
|
|
4019
|
+
result.terminalResult === RequestResult.Ok &&
|
|
4020
|
+
(result.tail === undefined || result.tail.kind !== record.kind)
|
|
4019
4021
|
) {
|
|
4020
4022
|
throw new ServiceWireProtocolError('User Spot success result omits its operation tail.');
|
|
4021
4023
|
}
|
|
@@ -4036,7 +4038,11 @@ export class ServiceStatefulRuntime {
|
|
|
4036
4038
|
timeoutMs: number
|
|
4037
4039
|
): Promise<ServiceUserSpotOperationResult> {
|
|
4038
4040
|
const parts = await this.requestDurableOperation(
|
|
4039
|
-
targetNodeRid,
|
|
4041
|
+
targetNodeRid,
|
|
4042
|
+
[header],
|
|
4043
|
+
correlation,
|
|
4044
|
+
operationKind,
|
|
4045
|
+
performance.now() + Math.max(1, timeoutMs)
|
|
4040
4046
|
);
|
|
4041
4047
|
// A received envelope, including a decode failure, ends sender replay.
|
|
4042
4048
|
try {
|
|
@@ -4044,25 +4050,17 @@ export class ServiceStatefulRuntime {
|
|
|
4044
4050
|
throw new ServiceWireProtocolError('Invalid creation operation reply parts.');
|
|
4045
4051
|
}
|
|
4046
4052
|
const decoded = decodeStatefulReply(parts[0]!, correlation, operationKind, parts.length >= 2);
|
|
4047
|
-
if (
|
|
4048
|
-
decoded.terminalResult !== RequestResult.Ok
|
|
4049
|
-
&& parts.length !== 1
|
|
4050
|
-
) {
|
|
4053
|
+
if (decoded.terminalResult !== RequestResult.Ok && parts.length !== 1) {
|
|
4051
4054
|
throw new ServiceWireProtocolError('Failed creation operation reply carries a payload.');
|
|
4052
4055
|
}
|
|
4053
|
-
if (
|
|
4054
|
-
operationKind === 'userSpotClose'
|
|
4055
|
-
&& parts.length !== 1
|
|
4056
|
-
) {
|
|
4056
|
+
if (operationKind === 'userSpotClose' && parts.length !== 1) {
|
|
4057
4057
|
throw new ServiceWireProtocolError('User Spot close reply carries a payload.');
|
|
4058
4058
|
}
|
|
4059
4059
|
return {
|
|
4060
4060
|
terminalResult: decoded.terminalResult,
|
|
4061
4061
|
failureCode: decoded.failureCode,
|
|
4062
4062
|
...(decoded.tail === undefined ? {} : { tail: decoded.tail as never }),
|
|
4063
|
-
...(parts.length === 2
|
|
4064
|
-
? { payload: decodeApplicationPayload(parts[1]!) }
|
|
4065
|
-
: {})
|
|
4063
|
+
...(parts.length === 2 ? { payload: decodeApplicationPayload(parts[1]!) } : {})
|
|
4066
4064
|
};
|
|
4067
4065
|
} catch (error) {
|
|
4068
4066
|
throw translateWireReplyDecodeError(error);
|
|
@@ -4077,13 +4075,15 @@ export class ServiceStatefulRuntime {
|
|
|
4077
4075
|
deadlineMs: number
|
|
4078
4076
|
): Promise<readonly Buffer[]> {
|
|
4079
4077
|
const stop = new AbortController();
|
|
4080
|
-
const unobserve = this.raw.observePeerConnectionIntentRemoved(nodeRoutingId => {
|
|
4078
|
+
const unobserve = this.raw.observePeerConnectionIntentRemoved((nodeRoutingId) => {
|
|
4081
4079
|
if (nodeRoutingId === targetNodeRid) {
|
|
4082
|
-
stop.abort(
|
|
4083
|
-
|
|
4084
|
-
|
|
4085
|
-
|
|
4086
|
-
|
|
4080
|
+
stop.abort(
|
|
4081
|
+
createInternalFrameworkException(
|
|
4082
|
+
ZLinkFrameworkInternalErrorKind.RouteNotConnected,
|
|
4083
|
+
`${operationKind} target node lifecycle ended.`,
|
|
4084
|
+
true
|
|
4085
|
+
)
|
|
4086
|
+
);
|
|
4087
4087
|
}
|
|
4088
4088
|
});
|
|
4089
4089
|
let wasAdmitted = false;
|
|
@@ -4112,9 +4112,12 @@ export class ServiceStatefulRuntime {
|
|
|
4112
4112
|
}
|
|
4113
4113
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
4114
4114
|
try {
|
|
4115
|
-
await awaitWithAbort(
|
|
4116
|
-
|
|
4117
|
-
|
|
4115
|
+
await awaitWithAbort(
|
|
4116
|
+
new Promise<void>((resolve) => {
|
|
4117
|
+
timer = setTimeout(resolve, retryDelayMs);
|
|
4118
|
+
}),
|
|
4119
|
+
stop.signal
|
|
4120
|
+
);
|
|
4118
4121
|
} finally {
|
|
4119
4122
|
if (timer !== undefined) clearTimeout(timer);
|
|
4120
4123
|
}
|
|
@@ -4148,7 +4151,7 @@ export class ServiceStatefulRuntime {
|
|
|
4148
4151
|
if (settled) return;
|
|
4149
4152
|
settled = true;
|
|
4150
4153
|
clearTimeout(timer);
|
|
4151
|
-
resolve(parts.map(part => Buffer.from(part)));
|
|
4154
|
+
resolve(parts.map((part) => Buffer.from(part)));
|
|
4152
4155
|
};
|
|
4153
4156
|
void (async () => {
|
|
4154
4157
|
const applicationJobOwner = await this.raw.reserveLocalIngress();
|
|
@@ -4163,9 +4166,11 @@ export class ServiceStatefulRuntime {
|
|
|
4163
4166
|
applicationJobOwner
|
|
4164
4167
|
});
|
|
4165
4168
|
if (result !== 'infrastructure') {
|
|
4166
|
-
fail(
|
|
4167
|
-
|
|
4168
|
-
|
|
4169
|
+
fail(
|
|
4170
|
+
new ServiceWireProtocolError(
|
|
4171
|
+
`Local infrastructure request was rejected as '${result ?? 'unsupported'}'.`
|
|
4172
|
+
)
|
|
4173
|
+
);
|
|
4169
4174
|
}
|
|
4170
4175
|
} finally {
|
|
4171
4176
|
applicationJobOwner.close();
|
|
@@ -4174,10 +4179,7 @@ export class ServiceStatefulRuntime {
|
|
|
4174
4179
|
});
|
|
4175
4180
|
}
|
|
4176
4181
|
|
|
4177
|
-
private async submitOneWay(
|
|
4178
|
-
targetNodeRid: string,
|
|
4179
|
-
parts: readonly Buffer[]
|
|
4180
|
-
): Promise<number> {
|
|
4182
|
+
private async submitOneWay(targetNodeRid: string, parts: readonly Buffer[]): Promise<number> {
|
|
4181
4183
|
this.requireOpen();
|
|
4182
4184
|
if (targetNodeRid === this.nodeRid) {
|
|
4183
4185
|
const applicationJobOwner = await this.raw.reserveLocalIngress();
|
|
@@ -4231,7 +4233,7 @@ export class ServiceStatefulRuntime {
|
|
|
4231
4233
|
applicationJobOwner.close();
|
|
4232
4234
|
}
|
|
4233
4235
|
}
|
|
4234
|
-
return await this.raw.sendService(targetNodeRid, parts)
|
|
4236
|
+
return (await this.raw.sendService(targetNodeRid, parts))
|
|
4235
4237
|
? SubmitResult.Ok
|
|
4236
4238
|
: SubmitResult.NotConnected;
|
|
4237
4239
|
}
|
|
@@ -4241,7 +4243,13 @@ export class ServiceStatefulRuntime {
|
|
|
4241
4243
|
targetNodeRid: string,
|
|
4242
4244
|
parts: readonly Buffer[],
|
|
4243
4245
|
timeoutMs: number,
|
|
4244
|
-
operationKind:
|
|
4246
|
+
operationKind:
|
|
4247
|
+
| 'spotRequest'
|
|
4248
|
+
| 'actorRequest'
|
|
4249
|
+
| 'actorLookup'
|
|
4250
|
+
| 'actorDestroy'
|
|
4251
|
+
| 'actorJoin'
|
|
4252
|
+
| 'streamBind'
|
|
4245
4253
|
| 'streamUnbind'
|
|
4246
4254
|
| 'instanceSpotRequest',
|
|
4247
4255
|
actor?: ServiceActorRef
|
|
@@ -4259,24 +4267,18 @@ export class ServiceStatefulRuntime {
|
|
|
4259
4267
|
applicationJobOwner
|
|
4260
4268
|
};
|
|
4261
4269
|
try {
|
|
4262
|
-
this.submitLocalRequest(
|
|
4263
|
-
localIngress,
|
|
4264
|
-
pending,
|
|
4265
|
-
operationKind,
|
|
4266
|
-
actor,
|
|
4267
|
-
deadlineUnixMs
|
|
4268
|
-
);
|
|
4270
|
+
this.submitLocalRequest(localIngress, pending, operationKind, actor, deadlineUnixMs);
|
|
4269
4271
|
} catch (error) {
|
|
4270
4272
|
this.operations.reply(pending.id, failure(error));
|
|
4271
4273
|
} finally {
|
|
4272
4274
|
applicationJobOwner.close();
|
|
4273
4275
|
}
|
|
4274
|
-
})().catch(error => this.operations.reply(pending.id, failure(error)));
|
|
4276
|
+
})().catch((error) => this.operations.reply(pending.id, failure(error)));
|
|
4275
4277
|
return;
|
|
4276
4278
|
}
|
|
4277
4279
|
void this.raw.requestService(targetNodeRid, parts, timeoutMs).then(
|
|
4278
|
-
reply => this.completeRemoteReply(pending, targetNodeRid, operationKind, actor, reply),
|
|
4279
|
-
error => this.operations.fail(pending.id, error)
|
|
4280
|
+
(reply) => this.completeRemoteReply(pending, targetNodeRid, operationKind, actor, reply),
|
|
4281
|
+
(error) => this.operations.fail(pending.id, error)
|
|
4280
4282
|
);
|
|
4281
4283
|
}
|
|
4282
4284
|
|
|
@@ -4289,9 +4291,10 @@ export class ServiceStatefulRuntime {
|
|
|
4289
4291
|
deadlineMs: number,
|
|
4290
4292
|
canonical?: ServiceCanonicalActorJoinCandidate
|
|
4291
4293
|
): void {
|
|
4292
|
-
const actorRoute =
|
|
4293
|
-
|
|
4294
|
-
|
|
4294
|
+
const actorRoute =
|
|
4295
|
+
canonical === undefined
|
|
4296
|
+
? this.tryActorFence(actor)
|
|
4297
|
+
: this.acceptCanonicalActorFence(actor, canonical.actorFence);
|
|
4295
4298
|
if (target === undefined || actorRoute === undefined) {
|
|
4296
4299
|
this.operations.reply(pending.id, {
|
|
4297
4300
|
terminalResult: RequestResult.NotFound,
|
|
@@ -4300,13 +4303,16 @@ export class ServiceStatefulRuntime {
|
|
|
4300
4303
|
return;
|
|
4301
4304
|
}
|
|
4302
4305
|
if (canonical !== undefined) {
|
|
4303
|
-
this.canonicalActorJoinHandoffs.set(
|
|
4304
|
-
|
|
4305
|
-
|
|
4306
|
-
|
|
4307
|
-
|
|
4308
|
-
|
|
4309
|
-
|
|
4306
|
+
this.canonicalActorJoinHandoffs.set(
|
|
4307
|
+
pending.id,
|
|
4308
|
+
canonicalActorJoinHandoffId({
|
|
4309
|
+
sourceActorNodeRid: routingIdBytes(actor.nodeRid as never),
|
|
4310
|
+
actorId: actor.actorId,
|
|
4311
|
+
actorGeneration: actor.generation,
|
|
4312
|
+
sourceActorNodeGeneration: actorRoute.targetNodeGeneration,
|
|
4313
|
+
correlation: pending.id
|
|
4314
|
+
})
|
|
4315
|
+
);
|
|
4310
4316
|
}
|
|
4311
4317
|
let parts: readonly Buffer[];
|
|
4312
4318
|
try {
|
|
@@ -4330,7 +4336,7 @@ export class ServiceStatefulRuntime {
|
|
|
4330
4336
|
expectedOwnerLeaseGeneration: target.ownerLeaseGeneration
|
|
4331
4337
|
},
|
|
4332
4338
|
...(payload === undefined ? {} : { payload })
|
|
4333
|
-
}).map(frame => Buffer.from(frame));
|
|
4339
|
+
}).map((frame) => Buffer.from(frame));
|
|
4334
4340
|
} catch (error) {
|
|
4335
4341
|
this.canonicalActorJoinHandoffs.delete(pending.id);
|
|
4336
4342
|
this.operations.fail(pending.id, error);
|
|
@@ -4338,14 +4344,24 @@ export class ServiceStatefulRuntime {
|
|
|
4338
4344
|
}
|
|
4339
4345
|
if (targetNodeRid === this.nodeRid) {
|
|
4340
4346
|
this.submitRequest(
|
|
4341
|
-
pending,
|
|
4342
|
-
|
|
4347
|
+
pending,
|
|
4348
|
+
targetNodeRid,
|
|
4349
|
+
parts,
|
|
4350
|
+
Math.max(0, Math.ceil(deadlineMs - performance.now())),
|
|
4351
|
+
'actorJoin',
|
|
4352
|
+
actor
|
|
4343
4353
|
);
|
|
4344
4354
|
return;
|
|
4345
4355
|
}
|
|
4346
|
-
void this.requestDurableOperation(
|
|
4347
|
-
|
|
4348
|
-
|
|
4356
|
+
void this.requestDurableOperation(
|
|
4357
|
+
targetNodeRid,
|
|
4358
|
+
parts,
|
|
4359
|
+
pending.id,
|
|
4360
|
+
'actorJoin',
|
|
4361
|
+
deadlineMs
|
|
4362
|
+
).then(
|
|
4363
|
+
(reply) => this.completeRemoteReply(pending, targetNodeRid, 'actorJoin', actor, reply),
|
|
4364
|
+
(error) => {
|
|
4349
4365
|
this.canonicalActorJoinHandoffs.delete(pending.id);
|
|
4350
4366
|
this.operations.fail(pending.id, error);
|
|
4351
4367
|
}
|
|
@@ -4355,34 +4371,46 @@ export class ServiceStatefulRuntime {
|
|
|
4355
4371
|
private submitLocalRequest(
|
|
4356
4372
|
ingress: RawServiceIngressRecord,
|
|
4357
4373
|
pending: ServiceStatefulPendingOperation,
|
|
4358
|
-
operationKind:
|
|
4374
|
+
operationKind:
|
|
4375
|
+
| 'spotRequest'
|
|
4376
|
+
| 'actorRequest'
|
|
4377
|
+
| 'actorLookup'
|
|
4378
|
+
| 'actorDestroy'
|
|
4379
|
+
| 'actorJoin'
|
|
4380
|
+
| 'streamBind'
|
|
4359
4381
|
| 'streamUnbind'
|
|
4360
4382
|
| 'instanceSpotRequest',
|
|
4361
4383
|
actor?: ServiceActorRef,
|
|
4362
4384
|
deadlineUnixMs?: bigint
|
|
4363
4385
|
): void {
|
|
4364
|
-
const canonicalActorJoin =
|
|
4365
|
-
? decodeActorJoin28(ingress.parts)
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
const payloadFrame =
|
|
4371
|
-
|
|
4372
|
-
|
|
4373
|
-
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4386
|
+
const canonicalActorJoin =
|
|
4387
|
+
operationKind === 'actorJoin' ? decodeActorJoin28(ingress.parts) : undefined;
|
|
4388
|
+
const decoded =
|
|
4389
|
+
canonicalActorJoin === undefined
|
|
4390
|
+
? decodeStatefulHeader(ingress.parts[0]!)
|
|
4391
|
+
: statefulActorJoinFromCanonical(canonicalActorJoin);
|
|
4392
|
+
const payloadFrame =
|
|
4393
|
+
canonicalActorJoin?.payload === undefined
|
|
4394
|
+
? ingress.parts.length < 2
|
|
4395
|
+
? undefined
|
|
4396
|
+
: ingress.parts[1]
|
|
4397
|
+
: encodeApplicationPayload({
|
|
4398
|
+
packetName: canonicalActorJoin.payload.packetName,
|
|
4399
|
+
contentType: canonicalActorJoin.payload.contentType,
|
|
4400
|
+
payload: Buffer.from(canonicalActorJoin.payload.payload)
|
|
4401
|
+
});
|
|
4377
4402
|
if (decoded.kind === 'actorLookup') {
|
|
4378
4403
|
const found = this.registry.actor(decoded.actorId);
|
|
4379
|
-
this.operations.reply(
|
|
4380
|
-
|
|
4381
|
-
|
|
4382
|
-
|
|
4383
|
-
|
|
4384
|
-
|
|
4385
|
-
|
|
4404
|
+
this.operations.reply(
|
|
4405
|
+
pending.id,
|
|
4406
|
+
found === undefined
|
|
4407
|
+
? { terminalResult: RequestResult.NotFound, failureCode: ACTOR_ROUTE_NOT_FOUND }
|
|
4408
|
+
: {
|
|
4409
|
+
terminalResult: RequestResult.Ok,
|
|
4410
|
+
failureCode: 0,
|
|
4411
|
+
kindData: lookupKindData(found)
|
|
4412
|
+
}
|
|
4413
|
+
);
|
|
4386
4414
|
return;
|
|
4387
4415
|
}
|
|
4388
4416
|
if (decoded.kind === 'actorDestroy') {
|
|
@@ -4399,35 +4427,28 @@ export class ServiceStatefulRuntime {
|
|
|
4399
4427
|
failureCode,
|
|
4400
4428
|
replyPayload,
|
|
4401
4429
|
tail
|
|
4402
|
-
) =>
|
|
4403
|
-
|
|
4404
|
-
|
|
4405
|
-
replyPayload,
|
|
4406
|
-
|
|
4407
|
-
this.nodeRid,
|
|
4408
|
-
actor
|
|
4409
|
-
));
|
|
4430
|
+
) =>
|
|
4431
|
+
this.operations.reply(
|
|
4432
|
+
pending.id,
|
|
4433
|
+
this.resultFromReply(terminalResult, failureCode, replyPayload, tail, this.nodeRid, actor)
|
|
4434
|
+
);
|
|
4410
4435
|
if (decoded.kind === 'spotRequest') {
|
|
4411
4436
|
this.validateDirectSpotFence(decoded.target);
|
|
4412
|
-
this.enqueueApplicationFrame(
|
|
4413
|
-
|
|
4414
|
-
|
|
4415
|
-
|
|
4416
|
-
|
|
4417
|
-
|
|
4418
|
-
|
|
4419
|
-
|
|
4420
|
-
sourceSpotId: decoded.sourceSpotId,
|
|
4421
|
-
targetSpot: decoded.target.spot,
|
|
4422
|
-
reply: localReply
|
|
4423
|
-
}
|
|
4424
|
-
);
|
|
4437
|
+
this.enqueueApplicationFrame(ingress, `spot:${decoded.target.spot.spotId}`, payloadFrame!, {
|
|
4438
|
+
receiveKind: ReceiveKind.SpotRequest,
|
|
4439
|
+
operationKind: OperationKind.SpotRequest,
|
|
4440
|
+
correlation: decoded.correlation,
|
|
4441
|
+
sourceSpotId: decoded.sourceSpotId,
|
|
4442
|
+
targetSpot: decoded.target.spot,
|
|
4443
|
+
reply: localReply
|
|
4444
|
+
});
|
|
4425
4445
|
return;
|
|
4426
4446
|
}
|
|
4427
4447
|
if (decoded.kind === 'actorRequest') {
|
|
4428
|
-
const current =
|
|
4429
|
-
|
|
4430
|
-
|
|
4448
|
+
const current =
|
|
4449
|
+
decoded.boundSession === undefined
|
|
4450
|
+
? this.validateActorMessageFence(decoded.target)
|
|
4451
|
+
: this.validateActorFence(decoded.target);
|
|
4431
4452
|
this.enqueueApplicationFrame(
|
|
4432
4453
|
ingress,
|
|
4433
4454
|
`actor:${current.ref.actorId}\0${current.ref.generation}`,
|
|
@@ -4490,9 +4511,10 @@ export class ServiceStatefulRuntime {
|
|
|
4490
4511
|
onTerminalCompletion: () => completeOnJoinDispatchTerminal?.(),
|
|
4491
4512
|
reply: (terminalResult, failureCode, replyPayload, tail) => {
|
|
4492
4513
|
if (actorJoinReplyAccepted) return false;
|
|
4493
|
-
const acceptedJoin =
|
|
4494
|
-
|
|
4495
|
-
|
|
4514
|
+
const acceptedJoin =
|
|
4515
|
+
terminalResult === RequestResult.Ok &&
|
|
4516
|
+
tail?.kind === 'actorJoin' &&
|
|
4517
|
+
tail.joinResult === 0;
|
|
4496
4518
|
if (acceptedJoin && this.operations.isPending(pending.id)) {
|
|
4497
4519
|
// Commit membership now, but keep the source operation behind
|
|
4498
4520
|
// target OnJoinedActor. Entry finishes on this record; User
|
|
@@ -4517,12 +4539,7 @@ export class ServiceStatefulRuntime {
|
|
|
4517
4539
|
}
|
|
4518
4540
|
return true;
|
|
4519
4541
|
}
|
|
4520
|
-
const acceptedReply = localReply(
|
|
4521
|
-
terminalResult,
|
|
4522
|
-
failureCode,
|
|
4523
|
-
replyPayload,
|
|
4524
|
-
tail
|
|
4525
|
-
);
|
|
4542
|
+
const acceptedReply = localReply(terminalResult, failureCode, replyPayload, tail);
|
|
4526
4543
|
if (acceptedReply) actorJoinReplyAccepted = true;
|
|
4527
4544
|
return acceptedReply;
|
|
4528
4545
|
}
|
|
@@ -4534,13 +4551,15 @@ export class ServiceStatefulRuntime {
|
|
|
4534
4551
|
try {
|
|
4535
4552
|
const current = this.validateActorFence(decoded.actor);
|
|
4536
4553
|
if (decoded.binding.state === 'active') {
|
|
4537
|
-
this.registry.installSessionBinding(
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
|
|
4541
|
-
|
|
4542
|
-
|
|
4543
|
-
|
|
4554
|
+
this.registry.installSessionBinding(
|
|
4555
|
+
sessionBindingFromWire(
|
|
4556
|
+
current.ref,
|
|
4557
|
+
decoded.sessionRid,
|
|
4558
|
+
this.nodeRid,
|
|
4559
|
+
decoded.binding.generation,
|
|
4560
|
+
current.membershipEpoch
|
|
4561
|
+
)
|
|
4562
|
+
);
|
|
4544
4563
|
} else {
|
|
4545
4564
|
this.registry.unbindSession(
|
|
4546
4565
|
current.ref,
|
|
@@ -4557,29 +4576,19 @@ export class ServiceStatefulRuntime {
|
|
|
4557
4576
|
}
|
|
4558
4577
|
if (decoded.kind === 'instanceSpot') {
|
|
4559
4578
|
if (decoded.activation === 'missing' && this.asyncInstanceAuthority !== undefined) {
|
|
4560
|
-
void this.continueMissingInstanceActivation(
|
|
4561
|
-
ingress,
|
|
4562
|
-
decoded,
|
|
4563
|
-
payloadFrame!,
|
|
4564
|
-
localReply
|
|
4565
|
-
);
|
|
4579
|
+
void this.continueMissingInstanceActivation(ingress, decoded, payloadFrame!, localReply);
|
|
4566
4580
|
return;
|
|
4567
4581
|
}
|
|
4568
4582
|
try {
|
|
4569
4583
|
const activation = this.requireInstanceActivation(ingress, decoded);
|
|
4570
|
-
this.enqueueApplicationFrame(
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
|
|
4574
|
-
{
|
|
4575
|
-
|
|
4576
|
-
|
|
4577
|
-
|
|
4578
|
-
...(decoded.sourceSpotId === undefined ? {} : { sourceSpotId: decoded.sourceSpotId }),
|
|
4579
|
-
targetSpot: activation.ref,
|
|
4580
|
-
reply: localReply
|
|
4581
|
-
}
|
|
4582
|
-
);
|
|
4584
|
+
this.enqueueApplicationFrame(ingress, `spot:${activation.ref.spotId}`, payloadFrame!, {
|
|
4585
|
+
receiveKind: ReceiveKind.InstanceSpotActivation,
|
|
4586
|
+
operationKind: OperationKind.InstanceSpotRequest,
|
|
4587
|
+
correlation: pending.id,
|
|
4588
|
+
...(decoded.sourceSpotId === undefined ? {} : { sourceSpotId: decoded.sourceSpotId }),
|
|
4589
|
+
targetSpot: activation.ref,
|
|
4590
|
+
reply: localReply
|
|
4591
|
+
});
|
|
4583
4592
|
} catch (error) {
|
|
4584
4593
|
this.operations.reply(pending.id, failure(error));
|
|
4585
4594
|
}
|
|
@@ -4595,28 +4604,37 @@ export class ServiceStatefulRuntime {
|
|
|
4595
4604
|
private completeRemoteReply(
|
|
4596
4605
|
pending: ServiceStatefulPendingOperation,
|
|
4597
4606
|
targetNodeRid: string,
|
|
4598
|
-
operationKind:
|
|
4607
|
+
operationKind:
|
|
4608
|
+
| 'spotRequest'
|
|
4609
|
+
| 'actorRequest'
|
|
4610
|
+
| 'actorLookup'
|
|
4611
|
+
| 'actorDestroy'
|
|
4612
|
+
| 'actorJoin'
|
|
4613
|
+
| 'streamBind'
|
|
4599
4614
|
| 'streamUnbind'
|
|
4600
4615
|
| 'instanceSpotRequest',
|
|
4601
4616
|
actor: ServiceActorRef | undefined,
|
|
4602
4617
|
reply: readonly Buffer[]
|
|
4603
4618
|
): void {
|
|
4604
|
-
const canonicalActorJoinHandoffId =
|
|
4605
|
-
? this.canonicalActorJoinHandoffs.get(pending.id)
|
|
4606
|
-
: undefined;
|
|
4619
|
+
const canonicalActorJoinHandoffId =
|
|
4620
|
+
operationKind === 'actorJoin' ? this.canonicalActorJoinHandoffs.get(pending.id) : undefined;
|
|
4607
4621
|
try {
|
|
4608
|
-
if (reply.length < 1 || reply.length > 2)
|
|
4622
|
+
if (reply.length < 1 || reply.length > 2)
|
|
4623
|
+
throw new ServiceWireProtocolError('Invalid M6B reply parts.');
|
|
4609
4624
|
const decoded = decodeStatefulReply(reply[0]!, pending.id, operationKind, reply.length >= 2);
|
|
4610
4625
|
const payload = reply.length < 2 ? undefined : decodeApplicationPayload(reply[1]);
|
|
4611
|
-
this.operations.reply(
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4626
|
+
this.operations.reply(
|
|
4627
|
+
pending.id,
|
|
4628
|
+
this.resultFromReply(
|
|
4629
|
+
decoded.terminalResult,
|
|
4630
|
+
decoded.failureCode,
|
|
4631
|
+
payload,
|
|
4632
|
+
decoded.tail,
|
|
4633
|
+
targetNodeRid,
|
|
4634
|
+
actor,
|
|
4635
|
+
canonicalActorJoinHandoffId
|
|
4636
|
+
)
|
|
4637
|
+
);
|
|
4620
4638
|
} catch (error) {
|
|
4621
4639
|
// Spec 32-framework-error-model:58-60, 91-92 — an unprocessable remote
|
|
4622
4640
|
// reply (decode/shape failure) fails the awaiting operation with a
|
|
@@ -4695,7 +4713,10 @@ export class ServiceStatefulRuntime {
|
|
|
4695
4713
|
}
|
|
4696
4714
|
|
|
4697
4715
|
private validateActorFence(fence: ServiceActorRouteFence): ServiceActorState {
|
|
4698
|
-
if (
|
|
4716
|
+
if (
|
|
4717
|
+
fence.actor.nodeRid !== this.nodeRid ||
|
|
4718
|
+
fence.targetNodeGeneration !== this.nodeGeneration
|
|
4719
|
+
) {
|
|
4699
4720
|
throw new ServiceStaleGenerationError('actor', fence.actor.actorId);
|
|
4700
4721
|
}
|
|
4701
4722
|
const actor = this.registry.requireActor(fence.actor);
|
|
@@ -4706,7 +4727,10 @@ export class ServiceStatefulRuntime {
|
|
|
4706
4727
|
}
|
|
4707
4728
|
|
|
4708
4729
|
private validateActorMessageFence(fence: ServiceActorRouteFence): ServiceActorState {
|
|
4709
|
-
if (
|
|
4730
|
+
if (
|
|
4731
|
+
fence.actor.nodeRid !== this.nodeRid ||
|
|
4732
|
+
fence.targetNodeGeneration !== this.nodeGeneration
|
|
4733
|
+
) {
|
|
4710
4734
|
throw new ServiceStaleGenerationError('actor', fence.actor.actorId);
|
|
4711
4735
|
}
|
|
4712
4736
|
const actor = this.registry.actor(fence.actor.actorId);
|
|
@@ -4720,7 +4744,10 @@ export class ServiceStatefulRuntime {
|
|
|
4720
4744
|
}
|
|
4721
4745
|
|
|
4722
4746
|
private validateSpotFence(fence: ServiceSpotRouteFence): ServiceSpotState {
|
|
4723
|
-
if (
|
|
4747
|
+
if (
|
|
4748
|
+
fence.targetNodeRid !== this.nodeRid ||
|
|
4749
|
+
fence.targetNodeGeneration !== this.nodeGeneration
|
|
4750
|
+
) {
|
|
4724
4751
|
throw new ServiceStaleGenerationError('spot', fence.spot.spotId);
|
|
4725
4752
|
}
|
|
4726
4753
|
const spot = this.registry.requireSpot(fence.spot);
|
|
@@ -4744,7 +4771,10 @@ export class ServiceStatefulRuntime {
|
|
|
4744
4771
|
`Spot '${fence.spot.spotId}' is not Ready.`
|
|
4745
4772
|
);
|
|
4746
4773
|
}
|
|
4747
|
-
if (
|
|
4774
|
+
if (
|
|
4775
|
+
fence.targetNodeRid !== this.nodeRid ||
|
|
4776
|
+
fence.targetNodeGeneration !== this.nodeGeneration
|
|
4777
|
+
) {
|
|
4748
4778
|
throw createInternalFrameworkException(
|
|
4749
4779
|
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
4750
4780
|
`Spot '${fence.spot.spotId}' targets an unavailable owner route.`
|
|
@@ -4767,15 +4797,11 @@ export class ServiceStatefulRuntime {
|
|
|
4767
4797
|
);
|
|
4768
4798
|
}
|
|
4769
4799
|
if (
|
|
4770
|
-
current.targetNodeRid !== this.nodeRid
|
|
4771
|
-
|
|
4772
|
-
|
|
4773
|
-
|
|
4774
|
-
current.
|
|
4775
|
-
&& (
|
|
4776
|
-
current.authorityOwnerGeneration !== fence.authorityOwnerGeneration
|
|
4777
|
-
)
|
|
4778
|
-
)
|
|
4800
|
+
current.targetNodeRid !== this.nodeRid ||
|
|
4801
|
+
current.targetNodeGeneration !== this.nodeGeneration ||
|
|
4802
|
+
current.ownerLeaseGeneration !== fence.ownerLeaseGeneration ||
|
|
4803
|
+
(current.spot.generation === fence.spot.generation &&
|
|
4804
|
+
current.authorityOwnerGeneration !== fence.authorityOwnerGeneration)
|
|
4779
4805
|
) {
|
|
4780
4806
|
throw createInternalFrameworkException(
|
|
4781
4807
|
ZLinkFrameworkInternalErrorKind.SpotMoving,
|
|
@@ -4787,10 +4813,7 @@ export class ServiceStatefulRuntime {
|
|
|
4787
4813
|
|
|
4788
4814
|
private holdOrRelaySpotMessage(
|
|
4789
4815
|
ingress: RawServiceIngressRecord,
|
|
4790
|
-
wire: Extract<
|
|
4791
|
-
ServiceStatefulWireRecord,
|
|
4792
|
-
{ readonly kind: 'spotSend' | 'spotRequest' }
|
|
4793
|
-
>
|
|
4816
|
+
wire: Extract<ServiceStatefulWireRecord, { readonly kind: 'spotSend' | 'spotRequest' }>
|
|
4794
4817
|
): RawServicePumpResult | undefined {
|
|
4795
4818
|
const state = this.spotMessageFollow.get(spotKey(wire.target.spot));
|
|
4796
4819
|
if (state === undefined || !sameDirectSpotRoute(state.source, wire.target)) {
|
|
@@ -4855,7 +4878,7 @@ export class ServiceStatefulRuntime {
|
|
|
4855
4878
|
current.ingress.parts[1]!
|
|
4856
4879
|
];
|
|
4857
4880
|
if (current.wire.kind === 'spotSend') {
|
|
4858
|
-
if (!await this.raw.sendService(state.target.targetNodeRid, parts)) continue;
|
|
4881
|
+
if (!(await this.raw.sendService(state.target.targetNodeRid, parts))) continue;
|
|
4859
4882
|
} else {
|
|
4860
4883
|
try {
|
|
4861
4884
|
const remainingMs = Math.max(
|
|
@@ -4900,22 +4923,24 @@ export class ServiceStatefulRuntime {
|
|
|
4900
4923
|
if (target === undefined || suppressionFence === undefined) return;
|
|
4901
4924
|
const claim = this.spotMessageFollowSuppression.begin(suppressionFence);
|
|
4902
4925
|
if (claim === undefined) return;
|
|
4903
|
-
const operationLow =
|
|
4904
|
-
&& current.ingress.requestSequence !== 0n
|
|
4905
|
-
|
|
4906
|
-
|
|
4907
|
-
const accepted = await this.raw.sendService(current.ingress.sourceRoutingId, [
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
|
|
4912
|
-
|
|
4913
|
-
|
|
4914
|
-
|
|
4915
|
-
|
|
4916
|
-
|
|
4917
|
-
|
|
4918
|
-
|
|
4926
|
+
const operationLow =
|
|
4927
|
+
current.ingress.requestSequence !== undefined && current.ingress.requestSequence !== 0n
|
|
4928
|
+
? current.ingress.requestSequence
|
|
4929
|
+
: this.nextMessageFollowOperation++;
|
|
4930
|
+
const accepted = await this.raw.sendService(current.ingress.sourceRoutingId, [
|
|
4931
|
+
encodeMessageFollowHeader({
|
|
4932
|
+
source: messageFollowSpotRoute(state.source),
|
|
4933
|
+
target: messageFollowSpotRoute(target),
|
|
4934
|
+
hopCount: 1,
|
|
4935
|
+
queuedMessages: Math.min(state.queuedCount, 0xffff_ffff),
|
|
4936
|
+
queuedBytes: Math.min(state.queuedBytes, 0xffff_ffff),
|
|
4937
|
+
originalOperation: {
|
|
4938
|
+
high: this.nodeGeneration,
|
|
4939
|
+
low: operationLow
|
|
4940
|
+
},
|
|
4941
|
+
originalReplyRouteId: current.ingress.requestSequence ?? 0n
|
|
4942
|
+
})
|
|
4943
|
+
]);
|
|
4919
4944
|
if (accepted) {
|
|
4920
4945
|
this.spotMessageFollowSuppression.markSent(claim);
|
|
4921
4946
|
} else {
|
|
@@ -4994,18 +5019,18 @@ export class ServiceStatefulRuntime {
|
|
|
4994
5019
|
senderFence: ServiceBoundSessionSendFence | undefined
|
|
4995
5020
|
): ServiceActorRouteFence | undefined {
|
|
4996
5021
|
if (
|
|
4997
|
-
senderFence === undefined
|
|
4998
|
-
|
|
4999
|
-
|
|
5000
|
-
|
|
5022
|
+
senderFence === undefined ||
|
|
5023
|
+
senderFence.targetNodeGeneration !== this.nodeGeneration ||
|
|
5024
|
+
senderFence.authorityOwnerGeneration <= 0n ||
|
|
5025
|
+
senderFence.ownerLeaseGeneration <= 0n
|
|
5001
5026
|
) {
|
|
5002
5027
|
return undefined;
|
|
5003
5028
|
}
|
|
5004
5029
|
const local = actor.nodeRid === this.nodeRid ? this.registry.actor(actor.actorId) : undefined;
|
|
5005
5030
|
if (
|
|
5006
|
-
local === undefined
|
|
5007
|
-
|
|
5008
|
-
|
|
5031
|
+
local === undefined ||
|
|
5032
|
+
!sameActorRef(local.ref, actor) ||
|
|
5033
|
+
local.authorityOwnerGeneration !== senderFence.authorityOwnerGeneration
|
|
5009
5034
|
) {
|
|
5010
5035
|
return undefined;
|
|
5011
5036
|
}
|
|
@@ -5043,10 +5068,10 @@ export class ServiceStatefulRuntime {
|
|
|
5043
5068
|
// Object/authority/lease are bounded counters. Node lifecycle is a
|
|
5044
5069
|
// full-range opaque token, where only zero represents absence.
|
|
5045
5070
|
if (
|
|
5046
|
-
actor.generation <= 0n
|
|
5047
|
-
|
|
5048
|
-
|
|
5049
|
-
|
|
5071
|
+
actor.generation <= 0n ||
|
|
5072
|
+
fence.targetNodeGeneration === 0n ||
|
|
5073
|
+
fence.authorityOwnerGeneration <= 0n ||
|
|
5074
|
+
fence.ownerLeaseGeneration <= 0n
|
|
5050
5075
|
) {
|
|
5051
5076
|
return undefined;
|
|
5052
5077
|
}
|
|
@@ -5073,17 +5098,18 @@ export class ServiceStatefulRuntime {
|
|
|
5073
5098
|
};
|
|
5074
5099
|
}
|
|
5075
5100
|
const route = this.spotRoutes.get(spotKey(spot));
|
|
5076
|
-
return route !== undefined
|
|
5077
|
-
|
|
5078
|
-
|
|
5101
|
+
return route !== undefined &&
|
|
5102
|
+
route.targetNodeRid === targetNodeRid &&
|
|
5103
|
+
sameSpotRef(route.spot, spot)
|
|
5079
5104
|
? route
|
|
5080
5105
|
: undefined;
|
|
5081
5106
|
}
|
|
5082
5107
|
|
|
5083
5108
|
private tryEntrySpotFence(targetNodeRid: string): ServiceDirectSpotRouteFence | undefined {
|
|
5084
|
-
const targetNodeGeneration =
|
|
5085
|
-
|
|
5086
|
-
|
|
5109
|
+
const targetNodeGeneration =
|
|
5110
|
+
targetNodeRid === this.nodeRid
|
|
5111
|
+
? this.nodeGeneration
|
|
5112
|
+
: this.raw.topology.peer(targetNodeRid)?.descriptor.lifecycleGeneration;
|
|
5087
5113
|
if (targetNodeGeneration === undefined) return undefined;
|
|
5088
5114
|
return {
|
|
5089
5115
|
spot: {
|
|
@@ -5105,28 +5131,24 @@ export class ServiceStatefulRuntime {
|
|
|
5105
5131
|
if (targetNodeRid === this.nodeRid) {
|
|
5106
5132
|
const local = this.registry.spot(spot.spotId);
|
|
5107
5133
|
if (
|
|
5108
|
-
local?.kind === 'entry'
|
|
5109
|
-
|
|
5110
|
-
|
|
5111
|
-
|
|
5112
|
-
|
|
5134
|
+
local?.kind === 'entry' &&
|
|
5135
|
+
sameSpotRef(local.ref, spot) &&
|
|
5136
|
+
targetNodeGeneration === this.nodeGeneration &&
|
|
5137
|
+
requested.authorityOwnerGeneration === local.authorityOwnerGeneration &&
|
|
5138
|
+
isEntrySpotFence(requested)
|
|
5113
5139
|
) {
|
|
5114
5140
|
return requested;
|
|
5115
5141
|
}
|
|
5116
5142
|
const current = this.directSpotRoutes.get(directSpotKey(spot.spotId));
|
|
5117
|
-
return local !== undefined
|
|
5118
|
-
|
|
5119
|
-
|
|
5120
|
-
|
|
5121
|
-
|
|
5122
|
-
|
|
5123
|
-
|
|
5124
|
-
|
|
5125
|
-
current.
|
|
5126
|
-
|| (
|
|
5127
|
-
current.authorityOwnerGeneration === requested.authorityOwnerGeneration
|
|
5128
|
-
)
|
|
5129
|
-
)
|
|
5143
|
+
return local !== undefined &&
|
|
5144
|
+
local.state === 'ready' &&
|
|
5145
|
+
targetNodeGeneration === this.nodeGeneration &&
|
|
5146
|
+
current !== undefined &&
|
|
5147
|
+
current.targetNodeRid === this.nodeRid &&
|
|
5148
|
+
current.targetNodeGeneration === this.nodeGeneration &&
|
|
5149
|
+
current.ownerLeaseGeneration === requested.ownerLeaseGeneration &&
|
|
5150
|
+
(current.spot.generation !== requested.spot.generation ||
|
|
5151
|
+
current.authorityOwnerGeneration === requested.authorityOwnerGeneration)
|
|
5130
5152
|
? current
|
|
5131
5153
|
: undefined;
|
|
5132
5154
|
}
|
|
@@ -5160,9 +5182,8 @@ export class ServiceStatefulRuntime {
|
|
|
5160
5182
|
const current = this.registry.actor(actor.actorId);
|
|
5161
5183
|
const previousActor = current?.ref ?? actor;
|
|
5162
5184
|
const previousSpot = current?.spot;
|
|
5163
|
-
const previousSpotState =
|
|
5164
|
-
? undefined
|
|
5165
|
-
: this.registry.spot(previousSpot.spotId);
|
|
5185
|
+
const previousSpotState =
|
|
5186
|
+
previousSpot === undefined ? undefined : this.registry.spot(previousSpot.spotId);
|
|
5166
5187
|
const previousMembershipEpoch = current?.membershipEpoch ?? membershipEpoch - 1n;
|
|
5167
5188
|
if (current === undefined) {
|
|
5168
5189
|
this.registry.restoreActor(
|
|
@@ -5180,26 +5201,30 @@ export class ServiceStatefulRuntime {
|
|
|
5180
5201
|
});
|
|
5181
5202
|
const targetSpot = this.registry.spot(spot.spotId);
|
|
5182
5203
|
if (targetSpot?.kind !== 'entry') {
|
|
5183
|
-
this.enqueueActorControl(
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
5189
|
-
|
|
5190
|
-
|
|
5191
|
-
|
|
5192
|
-
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5204
|
+
this.enqueueActorControl(
|
|
5205
|
+
spot.spotId,
|
|
5206
|
+
{
|
|
5207
|
+
kind: 'actorControl',
|
|
5208
|
+
lifecycleKind: ActorLifecycleKind.Joined,
|
|
5209
|
+
previousActor,
|
|
5210
|
+
currentActor: committed.ref,
|
|
5211
|
+
previousSpotId: previousSpot?.spotId ?? null,
|
|
5212
|
+
currentSpotId: spot.spotId,
|
|
5213
|
+
previousSpotGeneration: previousSpot?.generation ?? 0n,
|
|
5214
|
+
currentSpotGeneration: spot.generation,
|
|
5215
|
+
previousMembershipEpoch,
|
|
5216
|
+
currentMembershipEpoch: committed.membershipEpoch,
|
|
5217
|
+
resultCode: 0
|
|
5218
|
+
},
|
|
5219
|
+
onTargetLifecycleCompleted
|
|
5220
|
+
);
|
|
5196
5221
|
}
|
|
5197
5222
|
if (
|
|
5198
|
-
actor.nodeRid === this.nodeRid
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5223
|
+
actor.nodeRid === this.nodeRid &&
|
|
5224
|
+
previousSpot !== undefined &&
|
|
5225
|
+
previousSpot.spotId !== spot.spotId &&
|
|
5226
|
+
previousSpotState?.ref.generation === previousSpot.generation &&
|
|
5227
|
+
(previousSpotState.kind === 'entry' || previousSpotState.kind === 'user')
|
|
5203
5228
|
) {
|
|
5204
5229
|
// A same-node Core join commits the target membership and publishes
|
|
5205
5230
|
// the source LEFT control separately. Remote joins use the formal
|
|
@@ -5221,18 +5246,19 @@ export class ServiceStatefulRuntime {
|
|
|
5221
5246
|
}
|
|
5222
5247
|
}
|
|
5223
5248
|
|
|
5224
|
-
|
|
5225
5249
|
private requireOpen(): void {
|
|
5226
5250
|
if (this.closed) throw new Error('Stateful runtime is closed.');
|
|
5227
5251
|
}
|
|
5228
5252
|
}
|
|
5229
5253
|
|
|
5230
5254
|
function isEntrySpotFence(fence: ServiceDirectSpotRouteFence): boolean {
|
|
5231
|
-
return
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
|
|
5235
|
-
|
|
5255
|
+
return (
|
|
5256
|
+
fence.spot.spotId === fence.targetNodeRid &&
|
|
5257
|
+
fence.spot.generation === fence.targetNodeGeneration &&
|
|
5258
|
+
fence.authorityOwnerGeneration === fence.targetNodeGeneration &&
|
|
5259
|
+
fence.ownerLeaseGeneration === fence.targetNodeGeneration &&
|
|
5260
|
+
fence.storeVersion === `entry:${fence.targetNodeGeneration}`
|
|
5261
|
+
);
|
|
5236
5262
|
}
|
|
5237
5263
|
|
|
5238
5264
|
function userSpotDeadline(deadlineUnixMs: bigint): {
|
|
@@ -5280,10 +5306,10 @@ function durableRequestCanReplay(error: unknown): boolean {
|
|
|
5280
5306
|
if (!isZLinkBackendResultError(error)) return false;
|
|
5281
5307
|
return error.operation === 'request'
|
|
5282
5308
|
? error.result === RequestResult.NotConnected || error.result === RequestResult.TimedOut
|
|
5283
|
-
: error.result === SubmitResult.NotConnected
|
|
5284
|
-
|
|
5285
|
-
|
|
5286
|
-
|
|
5309
|
+
: error.result === SubmitResult.NotConnected ||
|
|
5310
|
+
error.result === SubmitResult.NotFound ||
|
|
5311
|
+
error.result === SubmitResult.NotAdmitted ||
|
|
5312
|
+
error.result === SubmitResult.Backpressured;
|
|
5287
5313
|
}
|
|
5288
5314
|
|
|
5289
5315
|
function remainingDeadlineMs(deadlineUnixMs: bigint): number {
|
|
@@ -5302,9 +5328,8 @@ function userSpotOperationFingerprint(
|
|
|
5302
5328
|
record: ServiceUserSpotCreateRecord | ServiceUserSpotCloseRecord | ServiceActorCreateRecord
|
|
5303
5329
|
): string {
|
|
5304
5330
|
const { correlation: _correlation, ...semantic } = record;
|
|
5305
|
-
return JSON.stringify(
|
|
5306
|
-
|
|
5307
|
-
(_key, value: unknown) => typeof value === 'bigint' ? `${value}n` : value
|
|
5331
|
+
return JSON.stringify(semantic, (_key, value: unknown) =>
|
|
5332
|
+
typeof value === 'bigint' ? `${value}n` : value
|
|
5308
5333
|
);
|
|
5309
5334
|
}
|
|
5310
5335
|
|
|
@@ -5339,32 +5364,44 @@ function actorKey(actor: ServiceActorRef): string {
|
|
|
5339
5364
|
}
|
|
5340
5365
|
|
|
5341
5366
|
function sameSessionBinding(left: ServiceSessionBinding, right: ServiceSessionBinding): boolean {
|
|
5342
|
-
return
|
|
5343
|
-
|
|
5344
|
-
|
|
5345
|
-
|
|
5367
|
+
return (
|
|
5368
|
+
sameActorRef(left.actor, right.actor) &&
|
|
5369
|
+
left.sessionRid === right.sessionRid &&
|
|
5370
|
+
sameSessionOwnerIdentity(left, right) &&
|
|
5371
|
+
left.bindingGeneration === right.bindingGeneration
|
|
5372
|
+
);
|
|
5346
5373
|
}
|
|
5347
5374
|
|
|
5348
5375
|
function sameSessionIdentity(left: ServiceSessionBinding, right: ServiceSessionBinding): boolean {
|
|
5349
|
-
return
|
|
5350
|
-
|
|
5351
|
-
|
|
5376
|
+
return (
|
|
5377
|
+
sameActorRef(left.actor, right.actor) &&
|
|
5378
|
+
left.sessionRid === right.sessionRid &&
|
|
5379
|
+
sameSessionOwnerIdentity(left, right)
|
|
5380
|
+
);
|
|
5352
5381
|
}
|
|
5353
5382
|
|
|
5354
5383
|
function sameSessionOwnerIdentity(
|
|
5355
5384
|
left: Pick<
|
|
5356
5385
|
ServiceSessionBinding,
|
|
5357
|
-
|
|
5386
|
+
| 'sessionOwnerNodeRid'
|
|
5387
|
+
| 'sessionOwnerNodeGeneration'
|
|
5388
|
+
| 'sessionOwnerId'
|
|
5389
|
+
| 'sessionOwnerLeaseGeneration'
|
|
5358
5390
|
>,
|
|
5359
5391
|
right: Pick<
|
|
5360
5392
|
ServiceSessionBinding,
|
|
5361
|
-
|
|
5393
|
+
| 'sessionOwnerNodeRid'
|
|
5394
|
+
| 'sessionOwnerNodeGeneration'
|
|
5395
|
+
| 'sessionOwnerId'
|
|
5396
|
+
| 'sessionOwnerLeaseGeneration'
|
|
5362
5397
|
>
|
|
5363
5398
|
): boolean {
|
|
5364
|
-
return
|
|
5365
|
-
|
|
5366
|
-
|
|
5367
|
-
|
|
5399
|
+
return (
|
|
5400
|
+
left.sessionOwnerNodeRid === right.sessionOwnerNodeRid &&
|
|
5401
|
+
left.sessionOwnerNodeGeneration === right.sessionOwnerNodeGeneration &&
|
|
5402
|
+
left.sessionOwnerId === right.sessionOwnerId &&
|
|
5403
|
+
left.sessionOwnerLeaseGeneration === right.sessionOwnerLeaseGeneration
|
|
5404
|
+
);
|
|
5368
5405
|
}
|
|
5369
5406
|
|
|
5370
5407
|
function requireSessionOwnerIdentity(binding: ServiceSessionBinding): {
|
|
@@ -5373,9 +5410,9 @@ function requireSessionOwnerIdentity(binding: ServiceSessionBinding): {
|
|
|
5373
5410
|
readonly leaseGeneration: bigint;
|
|
5374
5411
|
} {
|
|
5375
5412
|
if (
|
|
5376
|
-
binding.sessionOwnerNodeGeneration === undefined
|
|
5377
|
-
|
|
5378
|
-
|
|
5413
|
+
binding.sessionOwnerNodeGeneration === undefined ||
|
|
5414
|
+
binding.sessionOwnerId === undefined ||
|
|
5415
|
+
binding.sessionOwnerLeaseGeneration === undefined
|
|
5379
5416
|
) {
|
|
5380
5417
|
throw new ServiceWireProtocolError(
|
|
5381
5418
|
`Session binding '${binding.sessionRid}' omits its owner lifecycle identity.`
|
|
@@ -5388,47 +5425,55 @@ function requireSessionOwnerIdentity(binding: ServiceSessionBinding): {
|
|
|
5388
5425
|
};
|
|
5389
5426
|
}
|
|
5390
5427
|
|
|
5391
|
-
function sessionDeliveryKey(
|
|
5392
|
-
|
|
5393
|
-
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
|
|
5399
|
-
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
|
|
5404
|
-
|
|
5405
|
-
|
|
5428
|
+
function sessionDeliveryKey(
|
|
5429
|
+
binding: Pick<
|
|
5430
|
+
ServiceSessionBinding,
|
|
5431
|
+
| 'actor'
|
|
5432
|
+
| 'sessionRid'
|
|
5433
|
+
| 'sessionOwnerNodeRid'
|
|
5434
|
+
| 'sessionOwnerNodeGeneration'
|
|
5435
|
+
| 'sessionOwnerId'
|
|
5436
|
+
| 'sessionOwnerLeaseGeneration'
|
|
5437
|
+
| 'bindingGeneration'
|
|
5438
|
+
>
|
|
5439
|
+
): string {
|
|
5440
|
+
return (
|
|
5441
|
+
`${actorKey(binding.actor)}\0${binding.sessionOwnerNodeRid}\0${binding.sessionRid}` +
|
|
5442
|
+
`\0${binding.sessionOwnerNodeGeneration ?? ''}` +
|
|
5443
|
+
`\0${binding.sessionOwnerId ?? ''}` +
|
|
5444
|
+
`\0${binding.sessionOwnerLeaseGeneration ?? ''}` +
|
|
5445
|
+
`\0${binding.bindingGeneration}`
|
|
5446
|
+
);
|
|
5406
5447
|
}
|
|
5407
5448
|
|
|
5408
5449
|
function exactRetiredDelivery(
|
|
5409
5450
|
binding: ServiceSessionBinding,
|
|
5410
5451
|
retired: ServiceRetiredBoundSessionRouteFence
|
|
5411
5452
|
): boolean {
|
|
5412
|
-
return
|
|
5413
|
-
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5453
|
+
return (
|
|
5454
|
+
binding.sessionRid === retired.sessionRid &&
|
|
5455
|
+
binding.sessionOwnerNodeRid === retired.sessionOwnerNodeRid &&
|
|
5456
|
+
binding.sessionOwnerNodeGeneration === retired.sessionOwnerNodeGeneration &&
|
|
5457
|
+
binding.sessionOwnerId === retired.sessionOwnerId &&
|
|
5458
|
+
binding.sessionOwnerLeaseGeneration === retired.sessionOwnerLeaseGeneration &&
|
|
5459
|
+
binding.bindingGeneration === retired.retiredBindingGeneration
|
|
5460
|
+
);
|
|
5418
5461
|
}
|
|
5419
5462
|
|
|
5420
5463
|
function replacementNoticeKey(
|
|
5421
5464
|
record: Extract<ServiceStatefulWireRecord, { readonly kind: 'boundSessionReplaced' }>
|
|
5422
5465
|
): string {
|
|
5423
|
-
return
|
|
5424
|
-
|
|
5425
|
-
|
|
5426
|
-
|
|
5427
|
-
|
|
5428
|
-
|
|
5429
|
-
|
|
5430
|
-
|
|
5431
|
-
|
|
5466
|
+
return (
|
|
5467
|
+
`${actorKey(record.actorAuthority.actor)}\0${record.retiredSession.sessionOwnerNodeRid}` +
|
|
5468
|
+
`\0${record.actorAuthority.targetNodeGeneration}` +
|
|
5469
|
+
`\0${record.actorAuthority.authorityOwnerGeneration}` +
|
|
5470
|
+
`\0${record.actorAuthority.ownerLeaseGeneration}` +
|
|
5471
|
+
`\0${record.retiredSession.sessionOwnerNodeGeneration}` +
|
|
5472
|
+
`\0${record.retiredSession.sessionOwnerId}` +
|
|
5473
|
+
`\0${record.retiredSession.sessionOwnerLeaseGeneration}` +
|
|
5474
|
+
`\0${record.retiredSession.sessionRid}` +
|
|
5475
|
+
`\0${record.retiredSession.retiredBindingGeneration}`
|
|
5476
|
+
);
|
|
5432
5477
|
}
|
|
5433
5478
|
|
|
5434
5479
|
function actorAuthorityFromRoute(route: ServiceActorRouteFence): ServiceBoundSessionActorAuthority {
|
|
@@ -5449,9 +5494,11 @@ function directSpotKey(spotId: string): string {
|
|
|
5449
5494
|
}
|
|
5450
5495
|
|
|
5451
5496
|
function sameActorRef(left: ServiceActorRef, right: ServiceActorRef): boolean {
|
|
5452
|
-
return
|
|
5453
|
-
|
|
5454
|
-
|
|
5497
|
+
return (
|
|
5498
|
+
left.nodeRid === right.nodeRid &&
|
|
5499
|
+
left.actorId === right.actorId &&
|
|
5500
|
+
left.generation === right.generation
|
|
5501
|
+
);
|
|
5455
5502
|
}
|
|
5456
5503
|
|
|
5457
5504
|
function sameSpotRef(left: ServiceSpotRef, right: ServiceSpotRef): boolean {
|
|
@@ -5468,9 +5515,7 @@ function freezeSpotRoute(route: ServiceSpotRouteFence): ServiceSpotRouteFence {
|
|
|
5468
5515
|
});
|
|
5469
5516
|
}
|
|
5470
5517
|
|
|
5471
|
-
function freezeDirectSpotRoute(
|
|
5472
|
-
route: ServiceDirectSpotRouteFence
|
|
5473
|
-
): ServiceDirectSpotRouteFence {
|
|
5518
|
+
function freezeDirectSpotRoute(route: ServiceDirectSpotRouteFence): ServiceDirectSpotRouteFence {
|
|
5474
5519
|
return Object.freeze({
|
|
5475
5520
|
...freezeSpotRoute(route),
|
|
5476
5521
|
ownerLeaseGeneration: route.ownerLeaseGeneration,
|
|
@@ -5511,39 +5556,48 @@ function matchesExpectedInstanceRoute(
|
|
|
5511
5556
|
: current !== undefined && sameInstanceRoute(current, expected);
|
|
5512
5557
|
}
|
|
5513
5558
|
|
|
5514
|
-
function sameInstanceRoute(
|
|
5515
|
-
|
|
5516
|
-
|
|
5517
|
-
|
|
5518
|
-
|
|
5519
|
-
|
|
5520
|
-
|
|
5521
|
-
|
|
5522
|
-
|
|
5559
|
+
function sameInstanceRoute(
|
|
5560
|
+
left: ServiceInstanceRouteFence,
|
|
5561
|
+
right: ServiceInstanceRouteFence
|
|
5562
|
+
): boolean {
|
|
5563
|
+
return (
|
|
5564
|
+
left.targetNodeRid === right.targetNodeRid &&
|
|
5565
|
+
left.targetNodeGeneration === right.targetNodeGeneration &&
|
|
5566
|
+
left.targetSpotId === right.targetSpotId &&
|
|
5567
|
+
left.objectGeneration === right.objectGeneration &&
|
|
5568
|
+
left.ownerId === right.ownerId &&
|
|
5569
|
+
left.authorityOwnerGeneration === right.authorityOwnerGeneration &&
|
|
5570
|
+
left.leaseGeneration === right.leaseGeneration &&
|
|
5571
|
+
left.storeVersion === right.storeVersion
|
|
5572
|
+
);
|
|
5523
5573
|
}
|
|
5524
5574
|
|
|
5525
5575
|
function sameInstanceOwnerIdentity(
|
|
5526
5576
|
left: ServiceInstanceRouteFence,
|
|
5527
5577
|
right: ServiceInstanceRouteFence
|
|
5528
5578
|
): boolean {
|
|
5529
|
-
return
|
|
5530
|
-
|
|
5531
|
-
|
|
5532
|
-
|
|
5533
|
-
|
|
5534
|
-
|
|
5535
|
-
|
|
5579
|
+
return (
|
|
5580
|
+
left.targetNodeRid === right.targetNodeRid &&
|
|
5581
|
+
left.targetNodeGeneration === right.targetNodeGeneration &&
|
|
5582
|
+
left.targetSpotId === right.targetSpotId &&
|
|
5583
|
+
left.objectGeneration === right.objectGeneration &&
|
|
5584
|
+
left.ownerId === right.ownerId &&
|
|
5585
|
+
left.authorityOwnerGeneration === right.authorityOwnerGeneration &&
|
|
5586
|
+
left.leaseGeneration === right.leaseGeneration
|
|
5587
|
+
);
|
|
5536
5588
|
}
|
|
5537
5589
|
|
|
5538
5590
|
function sameInstanceApplicationOwner(
|
|
5539
5591
|
left: ServiceInstanceRouteFence,
|
|
5540
5592
|
right: ServiceInstanceRouteFence
|
|
5541
5593
|
): boolean {
|
|
5542
|
-
return
|
|
5543
|
-
|
|
5544
|
-
|
|
5545
|
-
|
|
5546
|
-
|
|
5594
|
+
return (
|
|
5595
|
+
left.targetNodeRid === right.targetNodeRid &&
|
|
5596
|
+
left.targetNodeGeneration === right.targetNodeGeneration &&
|
|
5597
|
+
left.targetSpotId === right.targetSpotId &&
|
|
5598
|
+
left.ownerId === right.ownerId &&
|
|
5599
|
+
left.leaseGeneration === right.leaseGeneration
|
|
5600
|
+
);
|
|
5547
5601
|
}
|
|
5548
5602
|
|
|
5549
5603
|
function matchesExpectedDirectSpotRoute(
|
|
@@ -5573,8 +5627,7 @@ function sameDirectSpotRoute(
|
|
|
5573
5627
|
left: ServiceDirectSpotRouteFence,
|
|
5574
5628
|
right: ServiceDirectSpotRouteFence
|
|
5575
5629
|
): boolean {
|
|
5576
|
-
return sameDirectSpotRouteIdentity(left, right)
|
|
5577
|
-
&& left.storeVersion === right.storeVersion;
|
|
5630
|
+
return sameDirectSpotRouteIdentity(left, right) && left.storeVersion === right.storeVersion;
|
|
5578
5631
|
}
|
|
5579
5632
|
|
|
5580
5633
|
function messageFollowSpotRoute(
|
|
@@ -5614,31 +5667,34 @@ function sameMessageFollowSpotSource(
|
|
|
5614
5667
|
current: ServiceDirectSpotRouteFence,
|
|
5615
5668
|
source: Extract<ServiceMessageFollowRoute, { readonly kind: 'spot' }>
|
|
5616
5669
|
): boolean {
|
|
5617
|
-
return
|
|
5618
|
-
|
|
5619
|
-
|
|
5620
|
-
|
|
5621
|
-
|
|
5622
|
-
|
|
5670
|
+
return (
|
|
5671
|
+
current.spot.spotId === source.spot.spotId &&
|
|
5672
|
+
current.spot.generation === source.spot.generation &&
|
|
5673
|
+
current.targetNodeRid === source.targetNodeRid &&
|
|
5674
|
+
current.targetNodeGeneration === source.targetNodeGeneration &&
|
|
5675
|
+
current.authorityOwnerGeneration === source.authorityOwnerGeneration &&
|
|
5676
|
+
current.ownerLeaseGeneration === source.ownerLeaseGeneration
|
|
5677
|
+
);
|
|
5623
5678
|
}
|
|
5624
5679
|
|
|
5625
5680
|
function sameDirectSpotRouteIdentity(
|
|
5626
5681
|
left: ServiceDirectSpotRouteFence,
|
|
5627
5682
|
right: ServiceDirectSpotRouteFence
|
|
5628
5683
|
): boolean {
|
|
5629
|
-
return sameDirectSpotOwnerIdentity(left, right)
|
|
5630
|
-
&& left.spot.generation === right.spot.generation;
|
|
5684
|
+
return sameDirectSpotOwnerIdentity(left, right) && left.spot.generation === right.spot.generation;
|
|
5631
5685
|
}
|
|
5632
5686
|
|
|
5633
5687
|
function sameDirectSpotOwnerIdentity(
|
|
5634
5688
|
left: ServiceDirectSpotRouteFence,
|
|
5635
5689
|
right: ServiceDirectSpotRouteFence
|
|
5636
5690
|
): boolean {
|
|
5637
|
-
return
|
|
5638
|
-
|
|
5639
|
-
|
|
5640
|
-
|
|
5641
|
-
|
|
5691
|
+
return (
|
|
5692
|
+
left.spot.spotId === right.spot.spotId &&
|
|
5693
|
+
left.targetNodeRid === right.targetNodeRid &&
|
|
5694
|
+
left.targetNodeGeneration === right.targetNodeGeneration &&
|
|
5695
|
+
left.authorityOwnerGeneration === right.authorityOwnerGeneration &&
|
|
5696
|
+
left.ownerLeaseGeneration === right.ownerLeaseGeneration
|
|
5697
|
+
);
|
|
5642
5698
|
}
|
|
5643
5699
|
|
|
5644
5700
|
function routeMatchesLocal(
|
|
@@ -5647,25 +5703,34 @@ function routeMatchesLocal(
|
|
|
5647
5703
|
nodeRid: string,
|
|
5648
5704
|
nodeGeneration: bigint
|
|
5649
5705
|
): boolean {
|
|
5650
|
-
return
|
|
5651
|
-
|
|
5652
|
-
|
|
5653
|
-
|
|
5654
|
-
|
|
5706
|
+
return (
|
|
5707
|
+
route.targetNodeRid === nodeRid &&
|
|
5708
|
+
route.targetNodeGeneration === nodeGeneration &&
|
|
5709
|
+
route.targetSpotId === spot.ref.spotId &&
|
|
5710
|
+
route.objectGeneration === spot.ref.generation &&
|
|
5711
|
+
route.authorityOwnerGeneration === spot.authorityOwnerGeneration
|
|
5712
|
+
);
|
|
5655
5713
|
}
|
|
5656
5714
|
|
|
5657
5715
|
function instanceOperationKey(
|
|
5658
5716
|
record: Extract<ServiceStatefulWireRecord, { readonly kind: 'instanceSpot' }>
|
|
5659
5717
|
): string {
|
|
5660
|
-
return
|
|
5661
|
-
|
|
5718
|
+
return (
|
|
5719
|
+
`${record.sourceNodeRid}\0${record.sourceNodeGeneration}\0` +
|
|
5720
|
+
`${record.operation.high}\0${record.operation.low}`
|
|
5721
|
+
);
|
|
5662
5722
|
}
|
|
5663
5723
|
|
|
5664
5724
|
function instanceActivationKey(
|
|
5665
|
-
record: Extract<
|
|
5725
|
+
record: Extract<
|
|
5726
|
+
ServiceStatefulWireRecord,
|
|
5727
|
+
{ readonly kind: 'instanceSpot'; readonly activation: 'missing' }
|
|
5728
|
+
>
|
|
5666
5729
|
): string {
|
|
5667
|
-
return
|
|
5668
|
-
|
|
5730
|
+
return (
|
|
5731
|
+
`${record.target.targetNodeRid}\0${record.target.targetNodeGeneration}\0` +
|
|
5732
|
+
`${record.target.targetSpotId}\0${record.target.stableType}`
|
|
5733
|
+
);
|
|
5669
5734
|
}
|
|
5670
5735
|
|
|
5671
5736
|
function subscriptionKey(channelName: string, topicFilter: string): string {
|
|
@@ -5694,10 +5759,9 @@ function emptyPayload(): ServiceApplicationPayload {
|
|
|
5694
5759
|
* the stateful mailbox's internal record shape as an adapter only; it must
|
|
5695
5760
|
* not re-parse the canonical bytes with the legacy hand-written decoder.
|
|
5696
5761
|
*/
|
|
5697
|
-
function statefulActorJoinFromCanonical(
|
|
5698
|
-
|
|
5699
|
-
|
|
5700
|
-
> {
|
|
5762
|
+
function statefulActorJoinFromCanonical(
|
|
5763
|
+
value: ActorJoin28
|
|
5764
|
+
): Extract<ServiceStatefulWireRecord, { readonly kind: 'actorJoin' }> {
|
|
5701
5765
|
return {
|
|
5702
5766
|
kind: 'actorJoin',
|
|
5703
5767
|
correlation: value.correlation,
|
|
@@ -5737,13 +5801,11 @@ function instanceOperationParts(
|
|
|
5737
5801
|
throw new TypeError('Instance operation requires one header and one payload frame.');
|
|
5738
5802
|
}
|
|
5739
5803
|
if (metadataFrame === undefined) return headerAndPayload;
|
|
5740
|
-
return [
|
|
5741
|
-
headerAndPayload[0]!,
|
|
5742
|
-
validateServiceMetadataFrame(metadataFrame),
|
|
5743
|
-
headerAndPayload[1]!
|
|
5744
|
-
];
|
|
5804
|
+
return [headerAndPayload[0]!, validateServiceMetadataFrame(metadataFrame), headerAndPayload[1]!];
|
|
5745
5805
|
}
|
|
5746
5806
|
|
|
5747
|
-
export function statefulMailboxData(
|
|
5807
|
+
export function statefulMailboxData(
|
|
5808
|
+
record: ServiceMailboxRecord
|
|
5809
|
+
): ServiceStatefulMailboxData | undefined {
|
|
5748
5810
|
return record.stateful as ServiceStatefulMailboxData | undefined;
|
|
5749
5811
|
}
|