@zlink-systems/framework 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/contracts/Common/ZLinkMessage.js +5 -5
- package/dist/contracts/Configuration/CodecContentType.js +5 -7
- package/dist/contracts/Configuration/DiagnosticsValidation.js +1 -6
- package/dist/contracts/Configuration/EndpointNotation.js +4 -8
- package/dist/contracts/Configuration/Registration.js +14 -11
- package/dist/contracts/Configuration/RegistrationBuilders.js +32 -25
- package/dist/contracts/Configuration/RegistrationNormalizers.js +25 -26
- package/dist/contracts/Configuration/RegistrationValidators.js +36 -38
- package/dist/contracts/Configuration/SendTimeoutValidation.js +4 -4
- package/dist/contracts/Configuration/TimerRegistrationValidator.js +7 -7
- package/dist/contracts/Handlers/JsonContract.js +23 -11
- package/dist/contracts/Spots/SpotKind.js +16 -8
- package/dist/location-store-integration.js +16 -16
- package/dist/nest-integration.js +17 -11
- package/dist/runtime/abort.js +9 -5
- package/dist/runtime/actors/actor-authority-payload-codec.js +12 -16
- package/dist/runtime/actors/actor-authority-publication.js +18 -17
- package/dist/runtime/actors/actor-client.js +83 -63
- package/dist/runtime/actors/actor-context.js +24 -18
- package/dist/runtime/actors/actor-creation.js +4 -3
- package/dist/runtime/actors/actor-handoff.js +89 -92
- package/dist/runtime/actors/actor-local-native-join.js +18 -19
- package/dist/runtime/actors/actor-mailbox.js +1 -1
- package/dist/runtime/actors/actor-message-follow-context.js +24 -21
- package/dist/runtime/actors/actor-packet-relay-wire.js +1 -3
- package/dist/runtime/actors/actor-remote-joiner.js +20 -17
- package/dist/runtime/actors/actor-remote-wire.js +6 -7
- package/dist/runtime/actors/actor-runtime-state.js +87 -81
- package/dist/runtime/actors/actor-transfer-registry.js +3 -5
- package/dist/runtime/actors/bound-session-wire.js +14 -9
- package/dist/runtime/actors/deferred-join-accepted-journal.js +158 -124
- package/dist/runtime/actors/index.js +2 -5
- package/dist/runtime/actors/post-commit-actor-binder.js +3 -6
- package/dist/runtime/actors/post-commit-actor-location.js +1 -1
- package/dist/runtime/actors/spot-actor-dispatch.js +5 -5
- package/dist/runtime/actors/transferred-actor-rollback.js +4 -4
- package/dist/runtime/admission.js +2 -6
- package/dist/runtime/application-jobs/application-ingress-record-owner.js +2 -6
- package/dist/runtime/application-jobs/receive-flow-controller.js +11 -12
- package/dist/runtime/backend/mesh-completion-table.js +1 -3
- package/dist/runtime/backend/mesh-dispatch-pump.js +12 -16
- package/dist/runtime/backend/node/node-backend-adapter-factory.js +1 -3
- package/dist/runtime/backend/node/node-backend-adapter-support.js +26 -17
- package/dist/runtime/backend/node/node-mesh-backend-adapter.js +4 -3
- package/dist/runtime/backend/node/node-raw-binding-port.js +7 -6
- package/dist/runtime/backend/node/node-raw-mesh-backend.js +116 -113
- package/dist/runtime/backend/node/node-socket-backend-adapter.js +12 -13
- package/dist/runtime/backend/runtime-message.js +3 -3
- package/dist/runtime/backend/runtime-values.d.ts +2 -2
- package/dist/runtime/backend/runtime-values.js +4 -5
- package/dist/runtime/channels/channel-clients.js +27 -24
- package/dist/runtime/channels/channel-dispatch-pipeline.js +10 -18
- package/dist/runtime/channels/channel-dispatch-services.js +7 -4
- package/dist/runtime/channels/channel-dispatchers.js +9 -9
- package/dist/runtime/channels/channel-envelope-inspection.js +16 -13
- package/dist/runtime/channels/channel-envelope.js +83 -47
- package/dist/runtime/channels/channel-outbound-operations.js +8 -8
- package/dist/runtime/channels/channel-receive-loops.js +20 -22
- package/dist/runtime/channels/channel-receive-task-tracker.js +1 -1
- package/dist/runtime/channels/channel-runtime-lifecycle.js +61 -56
- package/dist/runtime/channels/channel-runtime-manager.js +18 -9
- package/dist/runtime/channels/channel-socket-registry.js +126 -123
- package/dist/runtime/channels/channel-transports.js +43 -33
- package/dist/runtime/channels/client-server-location-runtime.js +32 -40
- package/dist/runtime/channels/client-server-service-wire.js +29 -22
- package/dist/runtime/channels/dispatch-error-reporter.js +9 -7
- package/dist/runtime/channels/fanout-location-runtime.js +34 -47
- package/dist/runtime/channels/fanout-service-wire.js +3 -5
- package/dist/runtime/channels/local-spot-route-transport.js +7 -1
- package/dist/runtime/channels/route-member-snapshot.js +3 -3
- package/dist/runtime/channels/source-spot-router.js +6 -2
- package/dist/runtime/channels/spot-route-bridge-transport.js +5 -2
- package/dist/runtime/channels/spot-route-dispatch-strategy.js +2 -2
- package/dist/runtime/channels/spot-route-target-resolver.js +6 -6
- package/dist/runtime/diagnostics/index.js +38 -10
- package/dist/runtime/diagnostics/message-flow.js +47 -39
- package/dist/runtime/diagnostics/runtime-metrics.js +18 -9
- package/dist/runtime/diagnostics/runtime-observation-queue.js +8 -10
- package/dist/runtime/diagnostics/topology-runtime-projections.js +17 -10
- package/dist/runtime/execution/index.js +7 -6
- package/dist/runtime/execution/serial-execution-queue.js +12 -18
- package/dist/runtime/execution/state-lane.js +2 -2
- package/dist/runtime/foundation/actor-join-recovery-codec.js +31 -23
- package/dist/runtime/foundation/event-loop-resources.js +4 -4
- package/dist/runtime/foundation/message-follow-suppression-registry.js +4 -4
- package/dist/runtime/foundation/raw-service-mesh-runtime.js +136 -136
- package/dist/runtime/foundation/route-mesh-connection-policy.js +4 -4
- package/dist/runtime/foundation/runtime-state-projections.js +22 -11
- package/dist/runtime/foundation/service-authority-payload-codec.js +13 -15
- package/dist/runtime/foundation/service-discovery-registry.js +35 -33
- package/dist/runtime/foundation/service-instance-activation-recovery-codec.js +20 -22
- package/dist/runtime/foundation/service-liveness-registry.js +16 -12
- package/dist/runtime/foundation/service-maintenance-runtime.js +2 -4
- package/dist/runtime/foundation/service-relocation-aggregate-committer.js +24 -24
- package/dist/runtime/foundation/service-relocation-object-owner.js +6 -6
- package/dist/runtime/foundation/service-relocation-runtime.js +137 -94
- package/dist/runtime/foundation/service-runtime-contracts.js +5 -5
- package/dist/runtime/foundation/service-stateful-registry.js +24 -23
- package/dist/runtime/foundation/service-stateful-runtime.js +416 -438
- package/dist/runtime/foundation/service-stateful-wire-codec.js +95 -98
- package/dist/runtime/foundation/service-topology-registry.js +96 -95
- package/dist/runtime/foundation/service-weighted-selection.js +6 -11
- package/dist/runtime/foundation/service-wire-binary-primitives.js +5 -6
- package/dist/runtime/foundation/service-wire-m6a-codec.js +46 -35
- package/dist/runtime/framework-errors-internal.js +7 -10
- package/dist/runtime/handlers/handler-instance-scope.js +4 -5
- package/dist/runtime/handlers/index.js +3 -1
- package/dist/runtime/host/actor-packet-relay.js +74 -70
- package/dist/runtime/host/actor-placement-coordinator.js +29 -31
- package/dist/runtime/host/actor-runtime-options-factory.js +12 -14
- package/dist/runtime/host/actor-transfer-authority-runtime.js +7 -9
- package/dist/runtime/host/actor-transfer-runtime.js +176 -153
- package/dist/runtime/host/application-job-queue.js +18 -22
- package/dist/runtime/host/bound-session-relay.js +3 -1
- package/dist/runtime/host/channel-runtime-options-factory.js +31 -14
- package/dist/runtime/host/host-capacity-status.js +8 -4
- package/dist/runtime/host/index.js +244 -227
- package/dist/runtime/host/instance-activation-authority.js +93 -96
- package/dist/runtime/host/location-runtime-owner.js +16 -14
- package/dist/runtime/host/mesh-router-resolver.js +11 -13
- package/dist/runtime/host/relocation-direct-transfer.js +10 -9
- package/dist/runtime/host/remote-actor-join-receiver.js +20 -15
- package/dist/runtime/host/remote-actor-packet-target-store.js +18 -22
- package/dist/runtime/host/remote-bound-session-relay.js +97 -73
- package/dist/runtime/host/route-mesh-runtime.js +64 -69
- package/dist/runtime/host/runtime-shutdown.js +1 -2
- package/dist/runtime/host/service-relocation-host-runtime.js +586 -511
- package/dist/runtime/host/spot-address-transport.js +69 -71
- package/dist/runtime/host/spot-runtime-options-factory.js +30 -29
- package/dist/runtime/host/stateful-authority-route-runtime.js +65 -68
- package/dist/runtime/host/user-spot-creation-coordinator.js +61 -67
- package/dist/runtime/locations/actor-location-claims.js +26 -20
- package/dist/runtime/locations/aggregate-inventory-store.js +72 -75
- package/dist/runtime/locations/authority-key-codec.js +7 -7
- package/dist/runtime/locations/auto-connect-loop.js +3 -3
- package/dist/runtime/locations/auto-connect-planner.js +25 -19
- package/dist/runtime/locations/auto-connect-reconciler.js +51 -31
- package/dist/runtime/locations/in-memory-authority-store.js +128 -118
- package/dist/runtime/locations/in-memory-location-store.js +195 -168
- package/dist/runtime/locations/in-memory-provider-location-store.js +8 -10
- package/dist/runtime/locations/key-codec.js +1 -1
- package/dist/runtime/locations/lease-tracker.js +23 -21
- package/dist/runtime/locations/lifecycle.js +4 -5
- package/dist/runtime/locations/location-store-repository.js +368 -324
- package/dist/runtime/locations/opaque-record-key.js +2 -2
- package/dist/runtime/locations/resolvers.js +73 -82
- package/dist/runtime/locations/runtime.js +68 -63
- package/dist/runtime/locations/spot-location-claims.js +39 -35
- package/dist/runtime/messaging/framework-json-v1.js +7 -7
- package/dist/runtime/messaging/packet-name.js +7 -6
- package/dist/runtime/messaging/payload-codec.js +21 -18
- package/dist/runtime/protocol/service_wire_codec.generated.d.ts +30 -2
- package/dist/runtime/protocol/service_wire_codec.generated.js +1671 -234
- package/dist/runtime/routing-id.js +3 -5
- package/dist/runtime/spots/formal-remote-actor-admission-registry.js +35 -31
- package/dist/runtime/spots/formal-remote-actor-transfer-registry.js +1 -2
- package/dist/runtime/spots/index.js +208 -219
- package/dist/runtime/spots/spot-activation-registry.js +15 -14
- package/dist/runtime/spots/spot-activation-state.js +17 -15
- package/dist/runtime/spots/spot-activation.js +33 -16
- package/dist/runtime/spots/spot-actor-admission-coordinator.js +18 -15
- package/dist/runtime/spots/spot-actor-membership.js +1 -3
- package/dist/runtime/spots/spot-actor-packet-dispatch.js +23 -20
- package/dist/runtime/spots/spot-actor-packet-drain.js +12 -10
- package/dist/runtime/spots/spot-actor-packet-relay-dispatch.js +10 -6
- package/dist/runtime/spots/spot-closing.js +3 -7
- package/dist/runtime/spots/spot-context.js +2 -2
- package/dist/runtime/spots/spot-entry-activation.js +13 -14
- package/dist/runtime/spots/spot-handle.js +6 -3
- package/dist/runtime/spots/spot-handler-registry.js +12 -12
- package/dist/runtime/spots/spot-manager-public.js +5 -6
- package/dist/runtime/spots/spot-native-actor-join-admission.js +4 -3
- package/dist/runtime/spots/spot-node-autoconnect.js +15 -18
- package/dist/runtime/spots/spot-node-runtime-manager.js +74 -85
- package/dist/runtime/spots/spot-outbound.js +10 -14
- package/dist/runtime/spots/spot-remote-codec.js +22 -26
- package/dist/runtime/spots/spot-remote-route-codec.js +5 -7
- package/dist/runtime/spots/spot-route-packet-dispatch.js +16 -14
- package/dist/runtime/spots/spot-route-replies.js +4 -6
- package/dist/runtime/spots/spot-routed-actor-admission.js +9 -7
- package/dist/runtime/spots/spot-routed-bound-session-dispatch.js +1 -2
- package/dist/runtime/spots/spot-routed-frame-dispatch.js +5 -5
- package/dist/runtime/spots/spot-routed-spot-packet-dispatch.js +14 -10
- package/dist/runtime/spots/spot-serial-executor.js +5 -8
- package/dist/runtime/spots/spot-serial-turn-executor.js +4 -6
- package/dist/runtime/spots/spot-subscription-dispatch.js +5 -5
- package/dist/runtime/spots/spot-timer.js +25 -29
- package/dist/runtime/streams/actor-session-binding-registry.js +98 -107
- package/dist/runtime/streams/actor-session-lifecycle-coordinator.js +3 -1
- package/dist/runtime/streams/bound-actor-relay-sender.js +18 -16
- package/dist/runtime/streams/bound-session-service.js +8 -9
- package/dist/runtime/streams/index.js +20 -19
- package/dist/runtime/streams/managed-stream.js +32 -30
- package/dist/runtime/streams/native-fallback-bound-session.js +21 -17
- package/dist/runtime/streams/protocol.js +29 -18
- package/dist/runtime/streams/session-actor-coordinator.js +34 -37
- package/dist/runtime/streams/session-context.js +30 -28
- package/dist/runtime/streams/session-provider.js +2 -2
- package/dist/runtime/streams/session-serial-executor.js +1 -2
- package/dist/runtime/streams/stream-frame-factory.js +11 -5
- package/dist/runtime/streams/stream-session-runtime.js +70 -66
- package/dist/runtime/workers/index.js +15 -4
- package/package.json +2 -2
- package/src/contracts/Channels/RouteCalls.ts +6 -1
- package/src/contracts/Common/ZLinkMessage.ts +5 -5
- package/src/contracts/Common/ZLinkMessageMetadata.ts +4 -3
- package/src/contracts/Configuration/Builders.ts +14 -14
- package/src/contracts/Configuration/CodecContentType.ts +8 -10
- package/src/contracts/Configuration/DiagnosticsValidation.ts +1 -6
- package/src/contracts/Configuration/EndpointNotation.ts +11 -11
- package/src/contracts/Configuration/ObjectRoles.ts +1 -3
- package/src/contracts/Configuration/Registration.ts +46 -17
- package/src/contracts/Configuration/RegistrationBuilderPolicy.ts +10 -8
- package/src/contracts/Configuration/RegistrationBuilders.ts +107 -96
- package/src/contracts/Configuration/RegistrationCodecRegistry.ts +9 -3
- package/src/contracts/Configuration/RegistrationNormalizers.ts +116 -84
- package/src/contracts/Configuration/RegistrationTypes.ts +21 -25
- package/src/contracts/Configuration/RegistrationValidators.ts +122 -73
- package/src/contracts/Configuration/RouteChannelInternalState.ts +1 -4
- package/src/contracts/Configuration/RuntimeEndpointConnections.ts +4 -1
- package/src/contracts/Configuration/SendTimeoutValidation.ts +6 -4
- package/src/contracts/Configuration/TimerRegistrationValidator.ts +8 -13
- package/src/contracts/Dispatch/ZLinkDispatchOptions.ts +1 -6
- package/src/contracts/Errors/index.ts +1 -4
- package/src/contracts/Handlers/Attributes.ts +15 -9
- package/src/contracts/Handlers/IZLinkChannelHandlers.ts +24 -4
- package/src/contracts/Handlers/JsonContract.ts +53 -19
- package/src/contracts/Locations/ActorTransfer.ts +1 -5
- package/src/contracts/Locations/Authority.ts +4 -11
- package/src/contracts/Locations/Keys.ts +6 -1
- package/src/contracts/Locations/Options.ts +3 -2
- package/src/contracts/Locations/RelocationStore.ts +1 -4
- package/src/contracts/Locations/Resolvers.ts +5 -5
- package/src/contracts/Locations/RuntimeQuery.ts +6 -2
- package/src/contracts/Locations/Stores.ts +1 -2
- package/src/contracts/Locations/index.ts +2 -8
- package/src/contracts/RouteMesh/Contracts.ts +1 -7
- package/src/contracts/RouteMesh/RuntimeTopology.ts +1 -4
- package/src/contracts/Spots/Contracts.ts +2 -8
- package/src/contracts/Spots/SpotKind.ts +16 -8
- package/src/contracts/Spots/ZLinkSpot.ts +8 -11
- package/src/contracts/Streams/IZLinkSession.ts +0 -1
- package/src/contracts/Streams/IZLinkSessionActor.ts +5 -1
- package/src/contracts/Streams/IZLinkSessionPacketHandler.ts +5 -1
- package/src/internal.ts +5 -4
- package/src/location-store-integration.ts +24 -16
- package/src/nest-integration.ts +40 -25
- package/src/runtime/abort.ts +10 -4
- package/src/runtime/actors/actor-authority-payload-codec.ts +20 -29
- package/src/runtime/actors/actor-authority-publication.ts +92 -77
- package/src/runtime/actors/actor-client.ts +162 -133
- package/src/runtime/actors/actor-context.ts +138 -130
- package/src/runtime/actors/actor-creation.ts +47 -44
- package/src/runtime/actors/actor-execution-context.ts +6 -4
- package/src/runtime/actors/actor-handoff.ts +269 -293
- package/src/runtime/actors/actor-join-deferred-scope.ts +5 -4
- package/src/runtime/actors/actor-lifecycle-snapshot.ts +5 -8
- package/src/runtime/actors/actor-local-native-join.ts +131 -159
- package/src/runtime/actors/actor-mailbox.ts +16 -19
- package/src/runtime/actors/actor-message-follow-context.ts +46 -36
- package/src/runtime/actors/actor-packet-relay-wire.ts +32 -21
- package/src/runtime/actors/actor-remote-joiner.ts +50 -54
- package/src/runtime/actors/actor-remote-wire.ts +9 -13
- package/src/runtime/actors/actor-routed-join-transport.ts +10 -2
- package/src/runtime/actors/actor-runtime-contracts.ts +2 -5
- package/src/runtime/actors/actor-runtime-state.ts +127 -124
- package/src/runtime/actors/actor-transfer-registry.ts +9 -14
- package/src/runtime/actors/actor-transport-delivery-gate.ts +4 -6
- package/src/runtime/actors/bound-session-wire.ts +37 -17
- package/src/runtime/actors/deferred-join-accepted-journal.ts +351 -320
- package/src/runtime/actors/index.ts +43 -86
- package/src/runtime/actors/post-commit-actor-binder.ts +10 -10
- package/src/runtime/actors/post-commit-actor-location.ts +30 -23
- package/src/runtime/actors/spot-actor-dispatch.ts +78 -53
- package/src/runtime/actors/transferred-actor-rollback.ts +18 -13
- package/src/runtime/admission.ts +9 -9
- package/src/runtime/application-jobs/application-ingress-record-owner.ts +5 -15
- package/src/runtime/application-jobs/application-job-queue-handler-start-gate.ts +3 -2
- package/src/runtime/application-jobs/receive-flow-controller.ts +17 -14
- package/src/runtime/backend/contracts/index.ts +73 -52
- package/src/runtime/backend/mesh-actor-session-node-adapter.ts +4 -14
- package/src/runtime/backend/mesh-completion-table.ts +6 -9
- package/src/runtime/backend/mesh-dispatch-pump.ts +38 -30
- package/src/runtime/backend/node/node-backend-adapter-factory.ts +23 -21
- package/src/runtime/backend/node/node-backend-adapter-support.ts +36 -22
- package/src/runtime/backend/node/node-dealer-channel-client-transport.ts +59 -58
- package/src/runtime/backend/node/node-event-loop-poller.ts +2 -10
- package/src/runtime/backend/node/node-mesh-backend-adapter.ts +12 -5
- package/src/runtime/backend/node/node-monitor-backend-adapter.ts +4 -3
- package/src/runtime/backend/node/node-raw-binding-port.ts +29 -29
- package/src/runtime/backend/node/node-raw-mesh-backend.ts +335 -383
- package/src/runtime/backend/node/node-socket-backend-adapter.ts +101 -54
- package/src/runtime/backend/runtime-message.ts +5 -3
- package/src/runtime/backend/runtime-values.ts +10 -7
- package/src/runtime/channels/channel-autoconnect.ts +1 -4
- package/src/runtime/channels/channel-clients.ts +93 -73
- package/src/runtime/channels/channel-dispatch-pipeline.ts +40 -42
- package/src/runtime/channels/channel-dispatch-services.ts +51 -49
- package/src/runtime/channels/channel-dispatchers.ts +152 -96
- package/src/runtime/channels/channel-envelope-inspection.ts +20 -16
- package/src/runtime/channels/channel-envelope.ts +256 -150
- package/src/runtime/channels/channel-multipart.ts +13 -5
- package/src/runtime/channels/channel-outbound-operations.ts +37 -41
- package/src/runtime/channels/channel-receive-loops.ts +102 -98
- package/src/runtime/channels/channel-receive-task-tracker.ts +1 -1
- package/src/runtime/channels/channel-runtime-lifecycle.ts +248 -190
- package/src/runtime/channels/channel-runtime-manager.ts +63 -37
- package/src/runtime/channels/channel-socket-options.ts +3 -10
- package/src/runtime/channels/channel-socket-registry.ts +331 -273
- package/src/runtime/channels/channel-transports.ts +215 -197
- package/src/runtime/channels/client-server-location-runtime.ts +116 -100
- package/src/runtime/channels/client-server-service-wire.ts +37 -36
- package/src/runtime/channels/dispatch-error-reporter.ts +11 -8
- package/src/runtime/channels/fanout-location-runtime.ts +91 -94
- package/src/runtime/channels/fanout-service-wire.ts +5 -12
- package/src/runtime/channels/handler-filter-scope.ts +1 -4
- package/src/runtime/channels/index.ts +1 -4
- package/src/runtime/channels/local-spot-route-transport.ts +20 -5
- package/src/runtime/channels/route-member-snapshot.ts +11 -5
- package/src/runtime/channels/router-socket-spot-route-transport.ts +1 -7
- package/src/runtime/channels/source-spot-router.ts +26 -22
- package/src/runtime/channels/spot-direct-envelope.ts +10 -8
- package/src/runtime/channels/spot-node-route-transport.ts +13 -16
- package/src/runtime/channels/spot-route-bridge-transport.ts +26 -9
- package/src/runtime/channels/spot-route-dispatch-strategy.ts +40 -13
- package/src/runtime/channels/spot-route-target-resolver.ts +18 -9
- package/src/runtime/codecs/index.ts +3 -2
- package/src/runtime/diagnostics/flow-context.ts +12 -3
- package/src/runtime/diagnostics/index.ts +96 -61
- package/src/runtime/diagnostics/internal-event-contracts.ts +87 -33
- package/src/runtime/diagnostics/message-flow.ts +95 -76
- package/src/runtime/diagnostics/runtime-metrics.ts +46 -24
- package/src/runtime/diagnostics/runtime-observation-queue.ts +17 -28
- package/src/runtime/diagnostics/topology-runtime-projections.ts +62 -43
- package/src/runtime/execution/index.ts +18 -26
- package/src/runtime/execution/serial-execution-queue.ts +31 -40
- package/src/runtime/execution/state-lane.ts +2 -2
- package/src/runtime/foundation/actor-join-recovery-codec.ts +69 -51
- package/src/runtime/foundation/event-loop-resources.ts +8 -4
- package/src/runtime/foundation/message-follow-suppression-registry.ts +7 -14
- package/src/runtime/foundation/operation-identity.ts +1 -2
- package/src/runtime/foundation/operation-registry.ts +8 -2
- package/src/runtime/foundation/raw-service-mesh-runtime.ts +312 -385
- package/src/runtime/foundation/route-mesh-connection-policy.ts +7 -8
- package/src/runtime/foundation/runtime-state-projections.ts +24 -23
- package/src/runtime/foundation/service-authority-payload-codec.ts +54 -57
- package/src/runtime/foundation/service-discovery-registry.ts +67 -59
- package/src/runtime/foundation/service-instance-activation-recovery-codec.ts +28 -29
- package/src/runtime/foundation/service-liveness-registry.ts +18 -13
- package/src/runtime/foundation/service-mailbox.ts +14 -14
- package/src/runtime/foundation/service-maintenance-runtime.ts +22 -21
- package/src/runtime/foundation/service-metadata-codec.ts +2 -9
- package/src/runtime/foundation/service-relocation-aggregate-committer.ts +39 -36
- package/src/runtime/foundation/service-relocation-coordinator.ts +1 -4
- package/src/runtime/foundation/service-relocation-object-owner.ts +40 -58
- package/src/runtime/foundation/service-relocation-runtime.ts +292 -216
- package/src/runtime/foundation/service-runtime-contracts.ts +8 -8
- package/src/runtime/foundation/service-session-binding-ingress-port.ts +1 -4
- package/src/runtime/foundation/service-stateful-registry.ts +69 -59
- package/src/runtime/foundation/service-stateful-runtime.ts +1174 -1112
- package/src/runtime/foundation/service-stateful-wire-codec.ts +318 -270
- package/src/runtime/foundation/service-topology-registry.ts +157 -159
- package/src/runtime/foundation/service-weighted-selection.ts +9 -15
- package/src/runtime/foundation/service-wire-binary-primitives.ts +6 -12
- package/src/runtime/foundation/service-wire-m6a-codec.ts +69 -47
- package/src/runtime/framework-errors-internal.ts +31 -30
- package/src/runtime/handlers/handler-instance-scope.ts +18 -29
- package/src/runtime/handlers/index.ts +18 -8
- package/src/runtime/host/actor-packet-relay.ts +290 -261
- package/src/runtime/host/actor-placement-coordinator.ts +165 -150
- package/src/runtime/host/actor-runtime-options-factory.ts +89 -87
- package/src/runtime/host/actor-transfer-authority-runtime.ts +46 -38
- package/src/runtime/host/actor-transfer-runtime.ts +391 -395
- package/src/runtime/host/application-job-queue.ts +32 -48
- package/src/runtime/host/bound-session-relay.ts +7 -9
- package/src/runtime/host/channel-runtime-options-factory.ts +75 -35
- package/src/runtime/host/host-capacity-status.ts +10 -9
- package/src/runtime/host/index.ts +861 -789
- package/src/runtime/host/instance-activation-authority.ts +125 -143
- package/src/runtime/host/location-runtime-owner.ts +38 -26
- package/src/runtime/host/mesh-router-resolver.ts +30 -23
- package/src/runtime/host/relocation-direct-transfer.ts +15 -15
- package/src/runtime/host/relocation-integrity-fault-gate.ts +3 -2
- package/src/runtime/host/remote-actor-join-receiver.ts +43 -34
- package/src/runtime/host/remote-actor-packet-target-store.ts +54 -52
- package/src/runtime/host/remote-bound-session-relay.ts +274 -247
- package/src/runtime/host/route-mesh-runtime-options.ts +1 -3
- package/src/runtime/host/route-mesh-runtime.ts +227 -181
- package/src/runtime/host/runtime-shutdown.ts +4 -3
- package/src/runtime/host/service-relocation-control.ts +14 -5
- package/src/runtime/host/service-relocation-host-runtime.ts +1388 -1165
- package/src/runtime/host/spot-address-transport.ts +284 -237
- package/src/runtime/host/spot-node-runtime-options-factory.ts +14 -8
- package/src/runtime/host/spot-runtime-options-factory.ts +71 -62
- package/src/runtime/host/stateful-authority-route-runtime.ts +117 -116
- package/src/runtime/host/user-spot-creation-coordinator.ts +225 -220
- package/src/runtime/locations/actor-location-claims.ts +61 -49
- package/src/runtime/locations/actor-session-route-claims.ts +1 -4
- package/src/runtime/locations/aggregate-inventory-store.ts +105 -134
- package/src/runtime/locations/authority-key-codec.ts +13 -10
- package/src/runtime/locations/auto-connect-loop.ts +14 -11
- package/src/runtime/locations/auto-connect-planner.ts +42 -31
- package/src/runtime/locations/auto-connect-reconciler.ts +106 -63
- package/src/runtime/locations/canonical-codec.ts +3 -1
- package/src/runtime/locations/creation-operation-id.ts +4 -3
- package/src/runtime/locations/domain-store-contract.ts +13 -7
- package/src/runtime/locations/in-memory-authority-store.ts +232 -199
- package/src/runtime/locations/in-memory-location-store.ts +432 -250
- package/src/runtime/locations/in-memory-provider-location-store.ts +16 -17
- package/src/runtime/locations/internal-store-contracts.ts +95 -38
- package/src/runtime/locations/key-codec.ts +3 -4
- package/src/runtime/locations/lease-tracker.ts +36 -27
- package/src/runtime/locations/lifecycle-runtime.ts +15 -3
- package/src/runtime/locations/lifecycle.ts +33 -28
- package/src/runtime/locations/location-store-repository.ts +1329 -1185
- package/src/runtime/locations/opaque-record-key.ts +4 -9
- package/src/runtime/locations/resolvers.ts +201 -178
- package/src/runtime/locations/runtime.ts +251 -159
- package/src/runtime/locations/spot-location-claims.ts +110 -112
- package/src/runtime/messaging/creation-payload-codec.ts +4 -1
- package/src/runtime/messaging/framework-json-v1.ts +34 -15
- package/src/runtime/messaging/packet-name.ts +8 -6
- package/src/runtime/messaging/payload-codec.ts +70 -50
- package/src/runtime/messaging/submission-result.ts +6 -9
- package/src/runtime/protocol/service_wire_codec.generated.ts +1376 -159
- package/src/runtime/protocol/service_wire_pilot_codec.generated.ts +1 -1
- package/src/runtime/routing-id.ts +10 -7
- package/src/runtime/spots/formal-remote-actor-admission-registry.ts +56 -62
- package/src/runtime/spots/formal-remote-actor-transfer-registry.ts +23 -20
- package/src/runtime/spots/index.ts +593 -563
- package/src/runtime/spots/spot-activation-registry.ts +38 -34
- package/src/runtime/spots/spot-activation-state.ts +43 -51
- package/src/runtime/spots/spot-activation.ts +179 -116
- package/src/runtime/spots/spot-actor-admission-coordinator.ts +110 -78
- package/src/runtime/spots/spot-actor-join-dispatch.ts +57 -41
- package/src/runtime/spots/spot-actor-lifecycle-drain.ts +5 -7
- package/src/runtime/spots/spot-actor-membership.ts +25 -27
- package/src/runtime/spots/spot-actor-packet-dispatch.ts +184 -153
- package/src/runtime/spots/spot-actor-packet-drain.ts +54 -42
- package/src/runtime/spots/spot-actor-packet-relay-dispatch.ts +37 -39
- package/src/runtime/spots/spot-closing.ts +11 -12
- package/src/runtime/spots/spot-context.ts +46 -26
- package/src/runtime/spots/spot-entry-activation.ts +84 -65
- package/src/runtime/spots/spot-handle.ts +17 -15
- package/src/runtime/spots/spot-handler-registry.ts +44 -31
- package/src/runtime/spots/spot-location-claim.ts +5 -4
- package/src/runtime/spots/spot-manager-public.ts +62 -82
- package/src/runtime/spots/spot-native-actor-join-admission.ts +20 -16
- package/src/runtime/spots/spot-node-autoconnect.ts +54 -61
- package/src/runtime/spots/spot-node-runtime-manager.ts +230 -221
- package/src/runtime/spots/spot-outbound.ts +47 -59
- package/src/runtime/spots/spot-publisher-transport.ts +5 -4
- package/src/runtime/spots/spot-remote-codec.ts +66 -65
- package/src/runtime/spots/spot-remote-route-codec.ts +61 -41
- package/src/runtime/spots/spot-route-packet-dispatch.ts +128 -95
- package/src/runtime/spots/spot-route-replies.ts +23 -24
- package/src/runtime/spots/spot-route-reply-wire.ts +3 -2
- package/src/runtime/spots/spot-routed-actor-admission.ts +37 -27
- package/src/runtime/spots/spot-routed-bound-session-dispatch.ts +11 -17
- package/src/runtime/spots/spot-routed-frame-dispatch.ts +18 -15
- package/src/runtime/spots/spot-routed-spot-packet-dispatch.ts +35 -26
- package/src/runtime/spots/spot-routing-internal.ts +1 -4
- package/src/runtime/spots/spot-runtime-ports.ts +5 -5
- package/src/runtime/spots/spot-serial-executor.ts +5 -9
- package/src/runtime/spots/spot-serial-turn-executor.ts +19 -26
- package/src/runtime/spots/spot-subscription-dispatch.ts +56 -46
- package/src/runtime/spots/spot-timer.ts +98 -91
- package/src/runtime/streams/actor-session-binding-registry.ts +277 -235
- package/src/runtime/streams/actor-session-binding-runtime-owner.ts +7 -4
- package/src/runtime/streams/actor-session-lifecycle-coordinator.ts +3 -1
- package/src/runtime/streams/bound-actor-relay-sender.ts +53 -51
- package/src/runtime/streams/bound-session-response-target.ts +29 -8
- package/src/runtime/streams/bound-session-service.ts +39 -39
- package/src/runtime/streams/index.ts +176 -100
- package/src/runtime/streams/managed-stream.ts +98 -87
- package/src/runtime/streams/native-fallback-bound-session.ts +57 -52
- package/src/runtime/streams/protocol.ts +73 -43
- package/src/runtime/streams/session-actor-coordinator.ts +97 -89
- package/src/runtime/streams/session-calls.ts +2 -8
- package/src/runtime/streams/session-context.ts +66 -48
- package/src/runtime/streams/session-provider.ts +5 -7
- package/src/runtime/streams/session-serial-executor.ts +7 -6
- package/src/runtime/streams/stream-binding-runtime-ports.ts +79 -29
- package/src/runtime/streams/stream-frame-factory.ts +55 -38
- package/src/runtime/streams/stream-session-runtime.ts +268 -210
- package/src/runtime/workers/index.ts +68 -29
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
ZLinkFrameworkInternalErrorKind,
|
|
3
|
+
createInternalFrameworkException
|
|
4
|
+
} from '../framework-errors-internal';
|
|
2
5
|
import type { RoutingId } from '../../contracts';
|
|
3
6
|
import {
|
|
4
7
|
ZLinkFrameworkException,
|
|
@@ -21,10 +24,7 @@ import {
|
|
|
21
24
|
} from '../foundation/runtime-state-projections';
|
|
22
25
|
|
|
23
26
|
type ZLinkDrainForceReason =
|
|
24
|
-
| '
|
|
25
|
-
| 'drain_state_publish_failed'
|
|
26
|
-
| 'owner_cleanup_failed'
|
|
27
|
-
| 'teardown_failed';
|
|
27
|
+
'deadline_exceeded' | 'drain_state_publish_failed' | 'owner_cleanup_failed' | 'teardown_failed';
|
|
28
28
|
|
|
29
29
|
type ZLinkMeshDrainResult =
|
|
30
30
|
| { readonly kind: 'drained' }
|
|
@@ -36,21 +36,14 @@ type ZLinkMeshDrainResult =
|
|
|
36
36
|
import type { ZLinkBackendMeshNode } from '../backend';
|
|
37
37
|
import type { ZLinkRuntimeAdmissionGate } from '../admission';
|
|
38
38
|
import type { ZLinkSpotNodeOptions } from '../configuration';
|
|
39
|
-
import {
|
|
40
|
-
ZLinkObjectRole,
|
|
41
|
-
type ZLinkMeshNodeDescriptor
|
|
42
|
-
} from '../../contracts';
|
|
39
|
+
import { ZLinkObjectRole, type ZLinkMeshNodeDescriptor } from '../../contracts';
|
|
43
40
|
|
|
44
41
|
export interface ZLinkRouteMeshRuntimeCoordinatorOptions {
|
|
45
42
|
readonly meshNames: readonly string[];
|
|
46
43
|
readonly meshOptions: ReadonlyMap<string, ZLinkSpotNodeOptions>;
|
|
47
44
|
readonly meshNode: (meshName: string) => ZLinkBackendMeshNode | undefined;
|
|
48
|
-
readonly meshNodeDescriptor?: (
|
|
49
|
-
|
|
50
|
-
) => ZLinkMeshNodeDescriptor | undefined;
|
|
51
|
-
readonly localPlacementCounts?: (
|
|
52
|
-
meshName: string
|
|
53
|
-
) => ZLinkLocalPlacementCounts | undefined;
|
|
45
|
+
readonly meshNodeDescriptor?: (meshName: string) => ZLinkMeshNodeDescriptor | undefined;
|
|
46
|
+
readonly localPlacementCounts?: (meshName: string) => ZLinkLocalPlacementCounts | undefined;
|
|
54
47
|
readonly isLocationStoreHealthy?: () => boolean;
|
|
55
48
|
readonly hostState?: () => ZLinkFrameworkRuntimeState;
|
|
56
49
|
readonly admission: ZLinkRuntimeAdmissionGate;
|
|
@@ -120,23 +113,26 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
120
113
|
const status = node.status();
|
|
121
114
|
const descriptor = this.options.meshNodeDescriptor?.(meshName);
|
|
122
115
|
const localPlacementCounts = this.options.localPlacementCounts?.(meshName);
|
|
123
|
-
const populationCapacity =
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
116
|
+
const populationCapacity =
|
|
117
|
+
descriptor === undefined || localPlacementCounts === undefined
|
|
118
|
+
? descriptor?.populationCapacity
|
|
119
|
+
: {
|
|
120
|
+
...descriptor.populationCapacity,
|
|
121
|
+
actors: {
|
|
122
|
+
...descriptor.populationCapacity.actors,
|
|
123
|
+
active: localPlacementCounts.activeActorCount
|
|
124
|
+
},
|
|
125
|
+
spots: {
|
|
126
|
+
...descriptor.populationCapacity.spots,
|
|
127
|
+
active: localPlacementCounts.activeSpotCount
|
|
128
|
+
}
|
|
129
|
+
};
|
|
136
130
|
const backendPeers = node.peers();
|
|
137
|
-
const peerChannels = backendPeers.map((peer) =>
|
|
138
|
-
|
|
139
|
-
|
|
131
|
+
const peerChannels = backendPeers.map((peer) =>
|
|
132
|
+
peer.routingId === null
|
|
133
|
+
? { names: [] as readonly string[], weights: [] as readonly number[] }
|
|
134
|
+
: node.peerChannels(peer.routingId, peer.lifecycleGeneration)
|
|
135
|
+
);
|
|
140
136
|
const peers = backendPeers.map((peer) => {
|
|
141
137
|
const state = peerState(peer.state);
|
|
142
138
|
return {
|
|
@@ -145,60 +141,61 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
145
141
|
unavailableReason: peerUnavailableReason(state)
|
|
146
142
|
};
|
|
147
143
|
});
|
|
148
|
-
const channels = Object.entries(this.options.meshOptions.get(meshName)?.meshChannels ?? {})
|
|
149
|
-
|
|
150
|
-
const readyMemberCount = BigInt(
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
144
|
+
const channels = Object.entries(this.options.meshOptions.get(meshName)?.meshChannels ?? {}).map(
|
|
145
|
+
([channelName, channel]) => {
|
|
146
|
+
const readyMemberCount = BigInt(
|
|
147
|
+
peerChannels.filter(
|
|
148
|
+
(entry, index) =>
|
|
149
|
+
peers[index]?.state === ZLinkPeerState.Ready &&
|
|
150
|
+
entry.names.some(
|
|
151
|
+
(name, channelIndex) =>
|
|
152
|
+
name === channelName && (entry.weights[channelIndex] ?? 0) > 0
|
|
153
|
+
)
|
|
154
|
+
).length
|
|
155
|
+
);
|
|
155
156
|
const localWeight = descriptor?.channelWeights[channelName] ?? channel.weight ?? 100;
|
|
156
|
-
const readyTargetCount =
|
|
157
|
-
+ (channel.server === true && localWeight > 0 ? 1 : 0);
|
|
157
|
+
const readyTargetCount =
|
|
158
|
+
Number(readyMemberCount) + (channel.server === true && localWeight > 0 ? 1 : 0);
|
|
158
159
|
return { channelName, isReady: readyTargetCount > 0, readyTargetCount };
|
|
159
|
-
}
|
|
160
|
+
}
|
|
161
|
+
);
|
|
160
162
|
const backendTopologyState = backendState(status.state);
|
|
161
|
-
const hasUnavailableRequiredPeer = peers.some(
|
|
162
|
-
peer
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
: drain.state;
|
|
167
|
-
const hostState = this.options.hostState?.()
|
|
168
|
-
?? ZLinkFrameworkRuntimeState.Serving;
|
|
163
|
+
const hasUnavailableRequiredPeer = peers.some(
|
|
164
|
+
(peer) =>
|
|
165
|
+
peer.state === ZLinkPeerState.Connecting || peer.state === ZLinkPeerState.NotConnected
|
|
166
|
+
);
|
|
167
|
+
const localTopologyState =
|
|
168
|
+
drain.state === ZLinkTopologyState.Ready ? backendTopologyState : drain.state;
|
|
169
|
+
const hostState = this.options.hostState?.() ?? ZLinkFrameworkRuntimeState.Serving;
|
|
169
170
|
const hostTopologyState = topologyStateForHost(hostState, localTopologyState);
|
|
170
171
|
const locationStoreHealthy = this.options.isLocationStoreHealthy?.() ?? true;
|
|
171
|
-
const state =
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
172
|
+
const state =
|
|
173
|
+
hostTopologyState === ZLinkTopologyState.Ready &&
|
|
174
|
+
(hasUnavailableRequiredPeer || !locationStoreHealthy)
|
|
175
|
+
? ZLinkTopologyState.Degraded
|
|
176
|
+
: hostTopologyState;
|
|
175
177
|
const hostReady = runtimeStateIsReady(hostState);
|
|
176
178
|
const objectRole = descriptor?.objectRole ?? ZLinkObjectRole.None;
|
|
177
179
|
const placementWeight = descriptor?.placementWeight ?? 0;
|
|
178
|
-
const capacityAvailable =
|
|
179
|
-
|
|
180
|
-
&&
|
|
181
|
-
&&
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
&&
|
|
188
|
-
&&
|
|
189
|
-
&&
|
|
190
|
-
|
|
180
|
+
const capacityAvailable =
|
|
181
|
+
descriptor !== undefined &&
|
|
182
|
+
populationCapacity !== undefined &&
|
|
183
|
+
hasRemainingCapacity(descriptor.activationConcurrency) &&
|
|
184
|
+
(hasRemainingCapacity(populationCapacity.actors) ||
|
|
185
|
+
hasRemainingCapacity(populationCapacity.spots));
|
|
186
|
+
const placementAvailable =
|
|
187
|
+
objectRole === ZLinkObjectRole.Server &&
|
|
188
|
+
placementWeight > 0 &&
|
|
189
|
+
capacityAvailable &&
|
|
190
|
+
locationStoreHealthy &&
|
|
191
|
+
hostReady &&
|
|
192
|
+
localTopologyState === ZLinkTopologyState.Ready;
|
|
191
193
|
const snapshot: ZLinkRouteMeshStatus = {
|
|
192
194
|
meshName,
|
|
193
195
|
state,
|
|
194
|
-
isReady: topologyRuntimeIsReady(
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
),
|
|
198
|
-
readyPeerCount: peers.filter(peer => peer.state === ZLinkPeerState.Ready).length,
|
|
199
|
-
channels: hostReady
|
|
200
|
-
? channels
|
|
201
|
-
: channels.map(channel => ({ ...channel, isReady: false })),
|
|
196
|
+
isReady: topologyRuntimeIsReady(hostState, state === ZLinkTopologyState.Ready ? 1 : 0),
|
|
197
|
+
readyPeerCount: peers.filter((peer) => peer.state === ZLinkPeerState.Ready).length,
|
|
198
|
+
channels: hostReady ? channels : channels.map((channel) => ({ ...channel, isReady: false })),
|
|
202
199
|
peers,
|
|
203
200
|
placement: {
|
|
204
201
|
isAvailable: placementAvailable,
|
|
@@ -210,9 +207,9 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
210
207
|
? ZLinkTopologyReason.RuntimeNotReady
|
|
211
208
|
: !locationStoreHealthy
|
|
212
209
|
? ZLinkTopologyReason.LocationUnavailable
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
210
|
+
: placementWeight <= 0
|
|
211
|
+
? ZLinkTopologyReason.NoReadyTarget
|
|
212
|
+
: ZLinkTopologyReason.CapacityExceeded
|
|
216
213
|
},
|
|
217
214
|
sequence: drain.sequence > status.lastChangedMs ? drain.sequence : status.lastChangedMs,
|
|
218
215
|
observedAt: new Date()
|
|
@@ -227,7 +224,8 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
227
224
|
signal?: AbortSignal
|
|
228
225
|
): AsyncIterable<ZLinkObservedStatus<ZLinkRouteMeshStatus>> {
|
|
229
226
|
const state = this.requireState(meshName);
|
|
230
|
-
if (!Number.isInteger(capacity) || capacity <= 0)
|
|
227
|
+
if (!Number.isInteger(capacity) || capacity <= 0)
|
|
228
|
+
throw new RangeError('Observer capacity must be positive.');
|
|
231
229
|
if (state.observers.size === 0) this.seedPlacementFingerprint(meshName);
|
|
232
230
|
const queue = new RuntimeEventQueue<ZLinkRouteMeshStatus>(capacity, signal);
|
|
233
231
|
queue.onClose(() => {
|
|
@@ -263,15 +261,14 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
263
261
|
let terminal: ZLinkRouteMeshStatus | undefined;
|
|
264
262
|
try {
|
|
265
263
|
const current = this.snapshot(meshName);
|
|
266
|
-
const terminalSequence =
|
|
267
|
-
? current.sequence + 1n
|
|
268
|
-
: state.sequence;
|
|
264
|
+
const terminalSequence =
|
|
265
|
+
current.sequence >= state.sequence ? current.sequence + 1n : state.sequence;
|
|
269
266
|
state.sequence = terminalSequence;
|
|
270
267
|
terminal = {
|
|
271
268
|
...current,
|
|
272
269
|
state: ZLinkTopologyState.Stopped,
|
|
273
270
|
isReady: false,
|
|
274
|
-
channels: current.channels.map(channel => ({
|
|
271
|
+
channels: current.channels.map((channel) => ({
|
|
275
272
|
...channel,
|
|
276
273
|
isReady: false
|
|
277
274
|
})),
|
|
@@ -286,15 +283,14 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
286
283
|
} catch {
|
|
287
284
|
const current = state.lastSnapshot;
|
|
288
285
|
if (current !== undefined) {
|
|
289
|
-
const terminalSequence =
|
|
290
|
-
? current.sequence + 1n
|
|
291
|
-
: state.sequence;
|
|
286
|
+
const terminalSequence =
|
|
287
|
+
current.sequence >= state.sequence ? current.sequence + 1n : state.sequence;
|
|
292
288
|
state.sequence = terminalSequence;
|
|
293
289
|
terminal = {
|
|
294
290
|
...current,
|
|
295
291
|
state: ZLinkTopologyState.Stopped,
|
|
296
292
|
isReady: false,
|
|
297
|
-
channels: current.channels.map(channel => ({
|
|
293
|
+
channels: current.channels.map((channel) => ({
|
|
298
294
|
...channel,
|
|
299
295
|
isReady: false
|
|
300
296
|
})),
|
|
@@ -316,7 +312,11 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
316
312
|
}
|
|
317
313
|
}
|
|
318
314
|
|
|
319
|
-
drain(
|
|
315
|
+
drain(
|
|
316
|
+
meshName: string,
|
|
317
|
+
deadlineMs = 30_000,
|
|
318
|
+
signal?: AbortSignal
|
|
319
|
+
): Promise<ZLinkMeshDrainResult> {
|
|
320
320
|
const state = this.requireState(meshName);
|
|
321
321
|
if (this.states.size > 1) return Promise.reject(multiMeshDrainError());
|
|
322
322
|
if (!Number.isFinite(deadlineMs) || deadlineMs <= 0) {
|
|
@@ -329,7 +329,9 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
329
329
|
awaitDrained(meshName: string, signal?: AbortSignal): Promise<ZLinkMeshDrainResult> {
|
|
330
330
|
const state = this.requireState(meshName);
|
|
331
331
|
if (this.states.size > 1) return Promise.reject(multiMeshDrainError());
|
|
332
|
-
const operation =
|
|
332
|
+
const operation =
|
|
333
|
+
state.operation ??
|
|
334
|
+
new Promise<ZLinkMeshDrainResult>((resolve) => state.waiters.push(resolve));
|
|
333
335
|
return waitForOperation(operation, signal);
|
|
334
336
|
}
|
|
335
337
|
|
|
@@ -368,7 +370,10 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
368
370
|
if (this.hostRetiringPrepared) return 'prepared';
|
|
369
371
|
const deadline = new AbortController();
|
|
370
372
|
const timer = setTimeout(
|
|
371
|
-
() =>
|
|
373
|
+
() =>
|
|
374
|
+
deadline.abort(
|
|
375
|
+
createDeadlineExceededError('Retire descriptor publication deadline exceeded.')
|
|
376
|
+
),
|
|
372
377
|
deadlineMs
|
|
373
378
|
);
|
|
374
379
|
const attempted: string[] = [];
|
|
@@ -384,7 +389,10 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
384
389
|
// every attempted descriptor before the host reports a reversible block.
|
|
385
390
|
const rollback = new AbortController();
|
|
386
391
|
const rollbackTimer = setTimeout(
|
|
387
|
-
() =>
|
|
392
|
+
() =>
|
|
393
|
+
rollback.abort(
|
|
394
|
+
createDeadlineExceededError('Retire descriptor rollback deadline exceeded.')
|
|
395
|
+
),
|
|
388
396
|
Math.min(deadlineMs, 1000)
|
|
389
397
|
);
|
|
390
398
|
let rollbackFailed = false;
|
|
@@ -423,8 +431,11 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
423
431
|
deadlineMs
|
|
424
432
|
);
|
|
425
433
|
try {
|
|
426
|
-
await Promise.all(
|
|
427
|
-
|
|
434
|
+
await Promise.all(
|
|
435
|
+
entries.map(([meshName]) =>
|
|
436
|
+
this.options.drainResources(meshName, deadline.signal, stopStartingSignal)
|
|
437
|
+
)
|
|
438
|
+
);
|
|
428
439
|
this.hostRetiringPrepared = false;
|
|
429
440
|
return { kind: 'drained' };
|
|
430
441
|
} catch (error) {
|
|
@@ -437,12 +448,16 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
437
448
|
: classified;
|
|
438
449
|
const rollback = new AbortController();
|
|
439
450
|
const rollbackTimer = setTimeout(
|
|
440
|
-
() =>
|
|
451
|
+
() =>
|
|
452
|
+
rollback.abort(
|
|
453
|
+
createDeadlineExceededError('Relocation descriptor rollback deadline exceeded.')
|
|
454
|
+
),
|
|
441
455
|
Math.min(Math.max(1, deadlineMs), 1000)
|
|
442
456
|
);
|
|
443
457
|
try {
|
|
444
|
-
await Promise.all(
|
|
445
|
-
this.options.rollbackRetiring(meshName, rollback.signal))
|
|
458
|
+
await Promise.all(
|
|
459
|
+
entries.map(([meshName]) => this.options.rollbackRetiring(meshName, rollback.signal))
|
|
460
|
+
);
|
|
446
461
|
} catch {
|
|
447
462
|
throw new ZLinkRetiringRollbackError();
|
|
448
463
|
} finally {
|
|
@@ -464,7 +479,10 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
464
479
|
this.options.admission.seal(meshName);
|
|
465
480
|
this.transition(meshName, state, ZLinkTopologyState.Stopping);
|
|
466
481
|
const deadline = new AbortController();
|
|
467
|
-
const timer = setTimeout(
|
|
482
|
+
const timer = setTimeout(
|
|
483
|
+
() => deadline.abort(createDeadlineExceededError('Drain deadline exceeded.')),
|
|
484
|
+
deadlineMs
|
|
485
|
+
);
|
|
468
486
|
let result: ZLinkMeshDrainResult;
|
|
469
487
|
try {
|
|
470
488
|
await this.options.publishDraining(meshName, deadline.signal);
|
|
@@ -476,9 +494,12 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
476
494
|
this.transition(meshName, state, ZLinkTopologyState.Stopped);
|
|
477
495
|
} catch (error) {
|
|
478
496
|
const classified = drainFailureReason(error);
|
|
479
|
-
const reason: ZLinkDrainForceReason =
|
|
480
|
-
|
|
481
|
-
|
|
497
|
+
const reason: ZLinkDrainForceReason =
|
|
498
|
+
classified !== 'teardown_failed'
|
|
499
|
+
? classified
|
|
500
|
+
: deadline.signal.aborted
|
|
501
|
+
? 'deadline_exceeded'
|
|
502
|
+
: classified;
|
|
482
503
|
await this.options.forceStopResources(meshName).catch(() => undefined);
|
|
483
504
|
result = { kind: 'forceStopped', reason };
|
|
484
505
|
this.transition(meshName, state, ZLinkTopologyState.Failed);
|
|
@@ -499,25 +520,32 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
499
520
|
state.deadline = deadlineAt;
|
|
500
521
|
}
|
|
501
522
|
const deadline = new AbortController();
|
|
502
|
-
const timer = setTimeout(
|
|
523
|
+
const timer = setTimeout(
|
|
524
|
+
() => deadline.abort(createDeadlineExceededError('Drain deadline exceeded.')),
|
|
525
|
+
deadlineMs
|
|
526
|
+
);
|
|
503
527
|
let result: ZLinkMeshDrainResult;
|
|
504
528
|
try {
|
|
505
529
|
if (!this.hostRetiringPrepared) {
|
|
506
|
-
await Promise.all(
|
|
507
|
-
this.options.publishRetiring(meshName, deadline.signal))
|
|
530
|
+
await Promise.all(
|
|
531
|
+
entries.map(([meshName]) => this.options.publishRetiring(meshName, deadline.signal))
|
|
532
|
+
);
|
|
508
533
|
}
|
|
509
534
|
this.hostRetiringPrepared = false;
|
|
510
|
-
await Promise.all(
|
|
511
|
-
this.options.drainResources(meshName, deadline.signal))
|
|
535
|
+
await Promise.all(
|
|
536
|
+
entries.map(([meshName]) => this.options.drainResources(meshName, deadline.signal))
|
|
537
|
+
);
|
|
512
538
|
for (const [meshName, state] of entries) {
|
|
513
539
|
this.options.admission.seal(meshName);
|
|
514
540
|
this.transition(meshName, state, ZLinkTopologyState.Stopping);
|
|
515
541
|
}
|
|
516
|
-
await Promise.all(
|
|
517
|
-
this.options.publishDraining(meshName, deadline.signal))
|
|
542
|
+
await Promise.all(
|
|
543
|
+
entries.map(([meshName]) => this.options.publishDraining(meshName, deadline.signal))
|
|
544
|
+
);
|
|
518
545
|
await this.options.publishHostDraining(deadline.signal);
|
|
519
|
-
await Promise.all(
|
|
520
|
-
this.options.admission.awaitZero(meshName, deadline.signal))
|
|
546
|
+
await Promise.all(
|
|
547
|
+
entries.map(([meshName]) => this.options.admission.awaitZero(meshName, deadline.signal))
|
|
548
|
+
);
|
|
521
549
|
await this.options.cleanupHostResources(deadline.signal);
|
|
522
550
|
result = { kind: 'drained' };
|
|
523
551
|
for (const [meshName, state] of entries) {
|
|
@@ -525,11 +553,17 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
525
553
|
}
|
|
526
554
|
} catch (error) {
|
|
527
555
|
const classified = drainFailureReason(error);
|
|
528
|
-
const reason: ZLinkDrainForceReason =
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
556
|
+
const reason: ZLinkDrainForceReason =
|
|
557
|
+
classified !== 'teardown_failed'
|
|
558
|
+
? classified
|
|
559
|
+
: deadline.signal.aborted
|
|
560
|
+
? 'deadline_exceeded'
|
|
561
|
+
: classified;
|
|
562
|
+
await Promise.all(
|
|
563
|
+
entries.map(([meshName]) =>
|
|
564
|
+
this.options.forceStopResources(meshName).catch(() => undefined)
|
|
565
|
+
)
|
|
566
|
+
);
|
|
533
567
|
result = { kind: 'forceStopped', reason };
|
|
534
568
|
for (const [meshName, state] of entries) {
|
|
535
569
|
this.transition(meshName, state, ZLinkTopologyState.Failed);
|
|
@@ -549,12 +583,13 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
549
583
|
cleanupDeadline: Date | number,
|
|
550
584
|
remainingTimeoutMs?: number
|
|
551
585
|
): Promise<ZLinkMeshDrainResult> {
|
|
552
|
-
const timeoutMs = Math.max(
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
586
|
+
const timeoutMs = Math.max(
|
|
587
|
+
0,
|
|
588
|
+
remainingTimeoutMs ??
|
|
589
|
+
(cleanupDeadline instanceof Date ? cleanupDeadline.getTime() - Date.now() : cleanupDeadline)
|
|
590
|
+
);
|
|
591
|
+
const deadlineAt =
|
|
592
|
+
cleanupDeadline instanceof Date ? cleanupDeadline : new Date(Date.now() + cleanupDeadline);
|
|
558
593
|
const entries = [...this.states.entries()];
|
|
559
594
|
if (entries.length === 0) return { kind: 'drained' };
|
|
560
595
|
for (const [, state] of entries) {
|
|
@@ -568,19 +603,19 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
568
603
|
let result: ZLinkMeshDrainResult;
|
|
569
604
|
try {
|
|
570
605
|
for (const [meshName, state] of entries) {
|
|
571
|
-
this.options.admission.seal(
|
|
572
|
-
meshName,
|
|
573
|
-
ZLinkFrameworkInternalErrorKind.RuntimeShutdown
|
|
574
|
-
);
|
|
606
|
+
this.options.admission.seal(meshName, ZLinkFrameworkInternalErrorKind.RuntimeShutdown);
|
|
575
607
|
this.transition(meshName, state, ZLinkTopologyState.Stopping);
|
|
576
608
|
}
|
|
577
|
-
await Promise.all(
|
|
578
|
-
this.options.publishDraining(meshName, deadline.signal))
|
|
609
|
+
await Promise.all(
|
|
610
|
+
entries.map(([meshName]) => this.options.publishDraining(meshName, deadline.signal))
|
|
611
|
+
);
|
|
579
612
|
await this.options.publishHostDraining(deadline.signal);
|
|
580
|
-
await Promise.all(
|
|
581
|
-
this.options.admission.awaitZero(meshName, deadline.signal))
|
|
582
|
-
|
|
583
|
-
|
|
613
|
+
await Promise.all(
|
|
614
|
+
entries.map(([meshName]) => this.options.admission.awaitZero(meshName, deadline.signal))
|
|
615
|
+
);
|
|
616
|
+
await Promise.all(
|
|
617
|
+
entries.map(([meshName]) => this.options.shutdownResources?.(meshName, deadline.signal))
|
|
618
|
+
);
|
|
584
619
|
await this.options.cleanupHostResources(deadline.signal);
|
|
585
620
|
result = { kind: 'drained' };
|
|
586
621
|
for (const [meshName, state] of entries) {
|
|
@@ -588,11 +623,17 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
588
623
|
}
|
|
589
624
|
} catch (error) {
|
|
590
625
|
const classified = drainFailureReason(error);
|
|
591
|
-
const reason: ZLinkDrainForceReason =
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
626
|
+
const reason: ZLinkDrainForceReason =
|
|
627
|
+
classified !== 'teardown_failed'
|
|
628
|
+
? classified
|
|
629
|
+
: deadline.signal.aborted
|
|
630
|
+
? 'deadline_exceeded'
|
|
631
|
+
: classified;
|
|
632
|
+
await Promise.all(
|
|
633
|
+
entries.map(([meshName]) =>
|
|
634
|
+
this.options.forceStopResources(meshName).catch(() => undefined)
|
|
635
|
+
)
|
|
636
|
+
);
|
|
596
637
|
result = { kind: 'forceStopped', reason };
|
|
597
638
|
for (const [meshName, state] of entries) {
|
|
598
639
|
this.transition(meshName, state, ZLinkTopologyState.Failed);
|
|
@@ -608,11 +649,7 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
608
649
|
return result;
|
|
609
650
|
}
|
|
610
651
|
|
|
611
|
-
private transition(
|
|
612
|
-
meshName: string,
|
|
613
|
-
state: ZLinkMeshDrainState,
|
|
614
|
-
next: ZLinkTopologyState
|
|
615
|
-
): void {
|
|
652
|
+
private transition(meshName: string, state: ZLinkMeshDrainState, next: ZLinkTopologyState): void {
|
|
616
653
|
if (state.state === next) return;
|
|
617
654
|
state.state = next;
|
|
618
655
|
state.sequence += 1n;
|
|
@@ -627,15 +664,14 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
627
664
|
const terminal = next === ZLinkTopologyState.Stopped || next === ZLinkTopologyState.Failed;
|
|
628
665
|
if (current !== undefined) {
|
|
629
666
|
if (terminal) {
|
|
630
|
-
const terminalSequence =
|
|
631
|
-
? current.sequence + 1n
|
|
632
|
-
: state.sequence;
|
|
667
|
+
const terminalSequence =
|
|
668
|
+
current.sequence >= state.sequence ? current.sequence + 1n : state.sequence;
|
|
633
669
|
state.sequence = terminalSequence;
|
|
634
670
|
const terminalStatus = {
|
|
635
671
|
...current,
|
|
636
672
|
state: next,
|
|
637
673
|
isReady: false,
|
|
638
|
-
channels: current.channels.map(channel => ({
|
|
674
|
+
channels: current.channels.map((channel) => ({
|
|
639
675
|
...channel,
|
|
640
676
|
isReady: false
|
|
641
677
|
})),
|
|
@@ -670,9 +706,10 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
670
706
|
if (descriptor !== undefined) {
|
|
671
707
|
this.placementFingerprints.set(meshName, placementFingerprint(descriptor));
|
|
672
708
|
}
|
|
673
|
-
this.peerFingerprints.set(
|
|
674
|
-
|
|
675
|
-
|
|
709
|
+
this.peerFingerprints.set(
|
|
710
|
+
meshName,
|
|
711
|
+
peerFingerprintMap(this.options.meshNode(meshName)?.peers() ?? [])
|
|
712
|
+
);
|
|
676
713
|
}
|
|
677
714
|
|
|
678
715
|
private startPlacementObserver(): void {
|
|
@@ -690,8 +727,7 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
690
727
|
private observePlacementChanges(): void {
|
|
691
728
|
for (const [meshName, state] of this.states) {
|
|
692
729
|
if (state.observers.size === 0) continue;
|
|
693
|
-
const locationStoreHealthy =
|
|
694
|
-
this.options.isLocationStoreHealthy?.() ?? true;
|
|
730
|
+
const locationStoreHealthy = this.options.isLocationStoreHealthy?.() ?? true;
|
|
695
731
|
if (locationStoreHealthy !== this.locationStoreHealthFingerprints.get(meshName)) {
|
|
696
732
|
this.locationStoreHealthFingerprints.set(meshName, locationStoreHealthy);
|
|
697
733
|
state.sequence += 1n;
|
|
@@ -716,10 +752,7 @@ export class ZLinkRouteMeshRuntimeCoordinator implements ZLinkRouteMeshRuntime {
|
|
|
716
752
|
const peers = node?.peers() ?? [];
|
|
717
753
|
const nextPeers = peerFingerprintMap(peers);
|
|
718
754
|
this.peerFingerprints.set(meshName, nextPeers);
|
|
719
|
-
for (const peerRid of new Set([
|
|
720
|
-
...previousPeers.keys(),
|
|
721
|
-
...nextPeers.keys()
|
|
722
|
-
])) {
|
|
755
|
+
for (const peerRid of new Set([...previousPeers.keys(), ...nextPeers.keys()])) {
|
|
723
756
|
if (previousPeers.get(peerRid) === nextPeers.get(peerRid)) continue;
|
|
724
757
|
state.sequence += 1n;
|
|
725
758
|
for (const observer of state.observers) {
|
|
@@ -776,31 +809,31 @@ function placementFingerprint(descriptor: ZLinkMeshNodeDescriptor): string {
|
|
|
776
809
|
descriptor.populationCapacity.spots,
|
|
777
810
|
spotTypes,
|
|
778
811
|
descriptor.activationConcurrency,
|
|
779
|
-
Object.entries(descriptor.channelWeights).sort(([left], [right]) =>
|
|
780
|
-
left.localeCompare(right))
|
|
812
|
+
Object.entries(descriptor.channelWeights).sort(([left], [right]) => left.localeCompare(right))
|
|
781
813
|
]);
|
|
782
814
|
}
|
|
783
815
|
|
|
784
|
-
function hasRemainingCapacity(
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
816
|
+
function hasRemainingCapacity(capacity: {
|
|
817
|
+
readonly active: number;
|
|
818
|
+
readonly limit: number;
|
|
819
|
+
readonly reserved?: number;
|
|
820
|
+
}): boolean {
|
|
821
|
+
return capacity.limit === 0 || capacity.active + (capacity.reserved ?? 0) < capacity.limit;
|
|
789
822
|
}
|
|
790
823
|
|
|
791
|
-
function peerFingerprintMap(
|
|
792
|
-
|
|
793
|
-
)
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
824
|
+
function peerFingerprintMap(peers: ReturnType<ZLinkBackendMeshNode['peers']>): Map<string, string> {
|
|
825
|
+
return new Map(
|
|
826
|
+
peers.map((peer) => [
|
|
827
|
+
String(peer.routingId),
|
|
828
|
+
JSON.stringify([
|
|
829
|
+
peer.lifecycleGeneration.toString(),
|
|
830
|
+
peer.descriptorRevision.toString(),
|
|
831
|
+
peer.endpoint,
|
|
832
|
+
peer.state,
|
|
833
|
+
peer.lastError
|
|
834
|
+
])
|
|
802
835
|
])
|
|
803
|
-
|
|
836
|
+
);
|
|
804
837
|
}
|
|
805
838
|
|
|
806
839
|
export class ZLinkRetiringRollbackError extends Error {
|
|
@@ -826,21 +859,29 @@ function multiMeshDrainError(): ZLinkFrameworkException {
|
|
|
826
859
|
|
|
827
860
|
function backendState(state: number): ZLinkTopologyState {
|
|
828
861
|
switch (state) {
|
|
829
|
-
case 1:
|
|
862
|
+
case 1:
|
|
863
|
+
return ZLinkTopologyState.Starting;
|
|
830
864
|
case 2:
|
|
831
865
|
case 3:
|
|
832
|
-
case 4:
|
|
833
|
-
|
|
834
|
-
case
|
|
835
|
-
|
|
866
|
+
case 4:
|
|
867
|
+
return ZLinkTopologyState.Ready;
|
|
868
|
+
case 5:
|
|
869
|
+
return ZLinkTopologyState.Stopping;
|
|
870
|
+
case 6:
|
|
871
|
+
return ZLinkTopologyState.Stopped;
|
|
872
|
+
default:
|
|
873
|
+
return ZLinkTopologyState.Failed;
|
|
836
874
|
}
|
|
837
875
|
}
|
|
838
876
|
|
|
839
877
|
function peerState(state: number): ZLinkPeerState {
|
|
840
878
|
switch (state) {
|
|
841
|
-
case 3:
|
|
842
|
-
|
|
843
|
-
case
|
|
879
|
+
case 3:
|
|
880
|
+
return ZLinkPeerState.Ready;
|
|
881
|
+
case 4:
|
|
882
|
+
return ZLinkPeerState.Draining;
|
|
883
|
+
case 6:
|
|
884
|
+
return ZLinkPeerState.NotRequired;
|
|
844
885
|
case 1:
|
|
845
886
|
case 2:
|
|
846
887
|
return ZLinkPeerState.Connecting;
|
|
@@ -864,8 +905,13 @@ function peerUnavailableReason(state: ZLinkPeerState): ZLinkTopologyReason | und
|
|
|
864
905
|
function drainFailureReason(error: unknown): ZLinkDrainForceReason {
|
|
865
906
|
const name = error instanceof Error ? error.name : '';
|
|
866
907
|
if (name === 'ZLinkDrainingStatePublishError') return 'drain_state_publish_failed';
|
|
867
|
-
if (
|
|
868
|
-
|
|
908
|
+
if (
|
|
909
|
+
name === 'ZLinkOwnerCleanupError' ||
|
|
910
|
+
(error instanceof AggregateError &&
|
|
911
|
+
error.errors.some(
|
|
912
|
+
(nested) => nested instanceof Error && nested.name === 'ZLinkOwnerCleanupError'
|
|
913
|
+
))
|
|
914
|
+
) {
|
|
869
915
|
return 'owner_cleanup_failed';
|
|
870
916
|
}
|
|
871
917
|
return 'teardown_failed';
|