@cotal-ai/manager 0.91.0 → 0.92.1

package/dist/run-hosting.js

@@ -43,7 +43,7 @@ import { randomBytes } from "node:crypto";
 import { credsAuthenticator } from "@nats-io/transport-node";
 import { jetstream, jetstreamManager } from "@nats-io/jetstream";
 import { Kvm } from "@nats-io/kv";
-import { COTAL_LANG_RUN_HOST, DEFAULT_SERVER, EpEnvelopeError, LANG_PROBLEM_DETAIL_KIND, RUN_ACTIVATION_WAIT_MS, RUN_HOST_KIND, dialerFor, inspectCredHealth, mintCreds, newIdentity, newTakeoverId, admissionBucket, createRunAdmission, readRunAdmission, revokeRunAdmission, readRunRevocation, withIssuerSession, epRequestSubject, isIssuedCaller, EP_UNBOUND_CALLER_AUTHORITY, runDriverCaller, openRecordsBucket, readRunProgram, readRunRecord, registry, standaloneConnectOpts, walkKvEntries, } from "@cotal-ai/core";
+import { COTAL_LANG_RUN_HOST, DEFAULT_SERVER, EpEnvelopeError, LANG_PROBLEM_DETAIL_KIND, RUN_ACTIVATION_WAIT_MS, RUN_HOST_KIND, dialerFor, inspectCredHealth, mintCreds, newIdentity, newTakeoverId, admissionBucket, createRunAdmission, bindAndAdmitRunSubmission, readRunSubmission, submissionFingerprint, newSubmissionRunId, KEYED_SUBMISSION_MAX_DEADLINE_MS, canonicalJson, readRunAdmission, revokeRunAdmission, commitRunCancellation, readRunCancelInput, readCancellationAdmission, runCancelCoordinates, readRunCancellationStatus, readRunRevocation, withIssuerSession, epRequestSubject, isIssuedCaller, issuedPermitsSubject as importedIssuedPermitsSubject, EP_UNBOUND_CALLER_AUTHORITY, runDriverCaller, openRecordsBucket, readRunProgram, readRunRecord, registry, standaloneConnectOpts, walkKvEntries, } from "@cotal-ai/core";
 /** The signerless callback set. RunHosting is remote exactly when ALL four are supplied; a partial
  *  set refuses construction, and nothing falls back to a local signer or a bare connection. */
 const REMOTE_CALLBACKS = ["admitRun", "issueAttempt", "issueOperator", "renewRun"];
@@ -55,6 +55,7 @@ const MAX_LAUNCHING = 8;
 export class RunHosting {
     ctx;
     runs = new Map();
+    cancellations = new Map();
     launching = 0;
     reconciled = false;
     stopping = false;
@@ -118,6 +119,127 @@ export class RunHosting {
         });
         return { runId };
     }
+    /** `requestEndsAt` is on this process's monotonic clock, measured from the serve layer's own
+     *  receive sample; it never leaves this process. */
+    keyedCoordinates(ctx) {
+        const caller = ctx.subject.caller, bind = ctx.request.bind;
+        if (ctx.subject.rail !== "v1" || !isIssuedCaller(caller))
+            throw new EpEnvelopeError("permission-denied", `keyed runs bind to the caller's issued authority, and this request rode the legacy rail with none; re-mint the credential as an issuance and call on the versioned rail (SPEC 13.15, 14)`, [{ kind: EP_UNBOUND_CALLER_AUTHORITY, owner: caller.owner, actor: caller.actor, uid: caller.uid }]);
+        if (ctx.subject.route !== "inst" || ctx.subject.instanceId !== this.ctx.instanceId || !bind || bind.instanceId !== this.ctx.instanceId
+            || bind.epoch !== ctx.identity.epoch || !Number.isFinite(ctx.receivedAtMonotonicMs)
+            || !Number.isSafeInteger(ctx.request.deadlineMs) || ctx.request.deadlineMs <= 0)
+            throw new EpEnvelopeError("permission-denied", "keyed runs require fresh issued exact-instance/epoch contracts and original receive deadline");
+        if (ctx.request.deadlineMs > KEYED_SUBMISSION_MAX_DEADLINE_MS)
+            throw new EpEnvelopeError("bad-request", `keyed deadlineMs exceeds the ${KEYED_SUBMISSION_MAX_DEADLINE_MS}ms request budget`);
+        return { caller, requestEndsAt: ctx.receivedAtMonotonicMs + ctx.request.deadlineMs };
+    }
+    /** The binding winner alone launches. Duplicates observe, even if an earlier admission is
+     * absent. A lost reply never allocates a replacement run or repairs an incomplete binding. */
+    async startKeyed(ctx, args) {
+        this.assertReconciled();
+        const { caller, requestEndsAt } = this.keyedCoordinates(ctx), host = this.host();
+        const verdict = host.validate(args.source, args.file);
+        if (!verdict.ok)
+            throw new EpEnvelopeError("bad-request", "keyed program does not validate", verdict.errors.map(e => ({ kind: LANG_PROBLEM_DETAIL_KIND, ...withoutFrame(e) })));
+        const placements = verdict.placements ?? [];
+        if (placements.length > 1)
+            throw new EpEnvelopeError("unimplemented", "keyed run may place spawns on one manager instance only");
+        // A launch refusal after the binding would leave an admission that no duplicate or read ever
+        // launches, so the stopping and capacity refusals, and the slot, are taken before it.
+        const release = this.reserveLaunch();
+        let slot;
+        try {
+            let result;
+            if (this.remote) {
+                if (!this.ctx.submitRun)
+                    throw new EpEnvelopeError("unimplemented", "this issuer does not support keyed submissions");
+                const remote = await this.ctx.submitRun({ subject: this.servedSubject(ctx) });
+                result = remote;
+                if (remote.view.submissionId !== ctx.request.id || remote.view.instanceId !== this.ctx.instanceId
+                    || remote.view.processEpoch !== ctx.identity.epoch || remote.view.fingerprint !== submissionFingerprint(ctx.request, ctx.subject).fingerprint)
+                    throw new EpEnvelopeError("internal", "issuer keyed response names other submission coordinates");
+            }
+            else {
+                const auth = this.ctx.auth;
+                if (!auth || !isIssuedCaller(caller))
+                    throw new EpEnvelopeError("unimplemented", "keyed runs require issued static or signerless authority");
+                const ref = { space: this.ctx.space, owner: caller.owner, actor: caller.actor, uid: caller.uid, generation: caller.generation };
+                const resolved = await withIssuerSession({ servers: this.ctx.servers ?? DEFAULT_SERVER, space: this.ctx.space, auth, tls: false }, s => s.store.resolve(ref, s.sourceIsLive));
+                const subject = this.servedSubject(ctx);
+                if (!importedIssuedPermitsSubject(resolved.evidence.permissions.publish, subject))
+                    throw new EpEnvelopeError("permission-denied", "issued ceiling does not permit keyed start");
+                const runId = newSubmissionRunId();
+                slot = this.claim(runId, placements);
+                const admission = { version: 1, space: this.ctx.space, endpoint: this.ctx.endpoint,
+                    runId, instanceId: this.ctx.instanceId, caller, ceiling: resolved.evidence.permissions,
+                    provenance: { kind: "issued", ref, resolvedRevision: resolved.revision }, admittedAt: Date.now() };
+                result = await this.withAdmitter(runId, (kv, jsm) => bindAndAdmitRunSubmission({ kv, jsm, admission,
+                    submissionId: ctx.request.id, fingerprint: submissionFingerprint(ctx.request, ctx.subject).fingerprint,
+                    processEpoch: ctx.identity.epoch, deadline: { requestEndsAt,
+                        ...(resolved.evidence.expiresAt !== undefined ? { authorityExpiresAt: resolved.evidence.expiresAt * 1000 } : {}) },
+                    assertCurrent: async () => {
+                        const fresh = await withIssuerSession({ servers: this.ctx.servers ?? DEFAULT_SERVER, space: this.ctx.space, auth, tls: false }, s => s.store.resolve(ref, s.sourceIsLive));
+                        if (fresh.revision !== resolved.revision || canonicalJson(fresh.evidence) !== canonicalJson(resolved.evidence))
+                            throw new EpEnvelopeError("permission-denied", "original keyed authority changed; retain its binding");
+                    } }), { caller, id: ctx.request.id });
+            }
+            if (!result.winner || !result.admission)
+                return this.submissionActivation(result.view);
+            const admission = result.admission;
+            if (admission.runId !== result.view.runId || admission.instanceId !== this.ctx.instanceId
+                || admission.caller.owner !== caller.owner || admission.caller.actor !== caller.actor || admission.caller.uid !== caller.uid
+                || !isIssuedCaller(admission.caller) || !isIssuedCaller(caller) || admission.caller.generation !== caller.generation)
+                throw new EpEnvelopeError("internal", "keyed admission names another caller/run/instance");
+            if (performance.now() >= requestEndsAt)
+                return { ...result.view, reason: "reconcile-held; original callback deadline elapsed before activation" };
+            // The signerless issuer mints the run id, so that slot is claimed on the winning reply.
+            slot ??= this.claim(admission.runId, placements);
+            await this.launch(host, { mode: "new", runId: admission.runId, source: args.source,
+                ...(args.file !== undefined ? { file: args.file } : {}), epoch: 1, fencingToken: 1,
+                timeout: args.timeout ?? DEFAULT_CHECKPOINT_TIMEOUT, admission: { admission, revoked: undefined }, placements }, slot, release);
+            return { ...result.view, state: "activated" };
+        }
+        finally {
+            release();
+            if (slot !== undefined && slot.drive === undefined)
+                this.free(slot);
+        }
+    }
+    async submissionStatus(ctx, submissionId) {
+        const { caller } = this.keyedCoordinates(ctx);
+        if (this.remote) {
+            if (!this.ctx.submitRun)
+                throw new EpEnvelopeError("unimplemented", "this issuer does not support submission reads");
+            const r = await this.ctx.submitRun({ subject: this.servedSubject(ctx) });
+            if (r.winner || r.admission || r.view.submissionId !== submissionId || r.view.instanceId !== this.ctx.instanceId
+                || r.view.processEpoch !== ctx.identity.epoch)
+                throw new EpEnvelopeError("internal", "invalid issuer submission read");
+            return this.submissionActivation(r.view);
+        }
+        const auth = this.ctx.auth;
+        if (!auth || !isIssuedCaller(caller))
+            throw new EpEnvelopeError("unimplemented", "keyed read needs issued authority");
+        const ref = { space: this.ctx.space, owner: caller.owner, actor: caller.actor, uid: caller.uid, generation: caller.generation };
+        await withIssuerSession({ servers: this.ctx.servers ?? DEFAULT_SERVER, space: this.ctx.space, auth, tls: false }, async (s) => {
+            const resolved = await s.store.resolve(ref, s.sourceIsLive);
+            if (!importedIssuedPermitsSubject(resolved.evidence.permissions.publish, this.servedSubject(ctx)))
+                throw new EpEnvelopeError("permission-denied", "issued ceiling does not permit submission read");
+        });
+        const view = await this.withAdmitter(newSubmissionRunId(), (_kv, jsm) => readRunSubmission({ jsm,
+            space: this.ctx.space, endpoint: this.ctx.endpoint, caller, submissionId, instanceId: this.ctx.instanceId, processEpoch: ctx.identity.epoch }));
+        return this.submissionActivation(view);
+    }
+    async submissionActivation(view) {
+        if (view.state !== "admitted" || !view.runId)
+            return view;
+        try {
+            const record = await this.withOperator({ runId: view.runId }, (_planes, kv) => readRunRecord(kv, this.ctx.endpoint, view.runId));
+            return record?.status ? { ...view, state: "activated" } : view;
+        }
+        catch {
+            return { ...view, state: "indeterminate", reason: "bound admission exists; activation observation unavailable" };
+        }
+    }
     /** Resolve the caller's issued authority and write the run's admission record, create-only,
      *  BEFORE the driver is launched and before the start reply. The ceiling is the evidence's,
      *  verbatim; nothing is read from a mutable ledger or an agent file. */
@@ -185,20 +307,20 @@ export class RunHosting {
         return this.servedSubject(ctx);
     }
     /** One admission-store write over an ephemeral `run-admitter` credential pinned to this run. */
-    async withAdmitter(runId, fn) {
+    async withAdmitter(runId, fn, submission) {
         const auth = this.ctx.auth;
         if (auth === undefined)
             throw new EpEnvelopeError("unimplemented", "an open mesh admits no hosted run (SPEC 14.8)");
         const nc = await dialerFor(this.ctx.servers ?? DEFAULT_SERVER)({
             servers: this.ctx.servers ?? DEFAULT_SERVER,
             ...standaloneConnectOpts({
-                creds: await mintCreds(auth, newIdentity(), "run-admitter", { runAdmitter: { endpoint: this.ctx.endpoint, runId } }),
+                creds: await mintCreds(auth, newIdentity(), "run-admitter", { runAdmitter: { endpoint: this.ctx.endpoint, runId, ...(submission ? { submission } : {}) } }),
                 /* not yet wired to a recorded transport */ tls: false,
             }),
             maxReconnectAttempts: 0,
         });
         try {
-            return await fn(await new Kvm(nc).open(admissionBucket(this.ctx.space)));
+            return await fn(await new Kvm(nc).open(admissionBucket(this.ctx.space)), await jetstreamManager(nc));
         }
         finally {
             await nc.drain().catch(() => nc.close());
@@ -210,6 +332,220 @@ export class RunHosting {
     async admitted(runId) {
         return await this.withOperator({ runId }, (planes) => readRunAdmission(planes.jsm, this.ctx.space, this.ctx.endpoint, runId));
     }
+    /** Input precedes serving/reconciliation for cancellation, unlike the keyed commands. */
+    validateCancellation(ctx, raw) {
+        runCancelCoordinates(ctx, this.ctx.instanceId);
+        readRunCancelInput(raw);
+    }
+    cancelPending(runId) {
+        return [...this.cancellations.values()].some(entry => entry.runId === runId);
+    }
+    assertNoCancel(runId) {
+        if (this.cancelPending(runId))
+            throw new EpEnvelopeError("failed-precondition", "a local cancellation is pending for this run", [{ kind: "ai.cotal.run.cancel-pending", runId }]);
+    }
+    cancelEligible(slot, status) {
+        return this.runs.get(slot.runId) === slot && (!slot.drive || !slot.activated
+            || (status?.holder === `${this.ctx.holder.id}.${slot.takeoverId}` && status.epoch === slot.epoch && status.fencingToken === slot.fencingToken));
+    }
+    observeCancellation(runId) {
+        for (const entry of this.cancellations.values())
+            if (entry.runId === runId)
+                entry.observed = true;
+    }
+    clearObservedCancellation(runId) {
+        if (this.runs.has(runId))
+            return;
+        for (const [key, entry] of this.cancellations)
+            if (entry.runId === runId && entry.observed)
+                this.cancellations.delete(key);
+    }
+    /** A served cancellation reports local observations, never cross-host cessation or process cleanup. */
+    async cancel(ctx, raw) {
+        const endsAt = runCancelCoordinates(ctx, this.ctx.instanceId);
+        const input = readRunCancelInput(raw);
+        this.assertReconciled();
+        if (this.stopping)
+            throw new EpEnvelopeError("unavailable", "the manager is stopping");
+        const auth = this.ctx.auth;
+        const caller = ctx.subject.caller;
+        if ((!auth && !this.remote) || !isIssuedCaller(caller))
+            throw new EpEnvelopeError("permission-denied", "run-cancel request is not authorized");
+        if (this.remote && !this.ctx.cancelRun)
+            throw new EpEnvelopeError("unimplemented", "the issuing host run-cancel door is not connected");
+        const budget = () => {
+            if (performance.now() >= endsAt)
+                throw new EpEnvelopeError("deadline-exceeded", "run-cancel original request budget expired");
+        };
+        const subject = this.servedSubject(ctx);
+        let authority = "original";
+        const current = async () => {
+            budget();
+            if (this.remote) {
+                const inspected = await this.ctx.cancelRun({ stage: "inspect", subject, cancel: input });
+                if (inspected.stage !== "inspect" || inspected.runId !== input.runId)
+                    throw new EpEnvelopeError("unavailable", "invalid run-cancel inspection response");
+                authority = inspected.authority;
+                budget();
+                return;
+            }
+            await withIssuerSession({ servers: this.ctx.servers ?? DEFAULT_SERVER, space: this.ctx.space, auth: auth, tls: false }, async (session) => {
+                const resolved = await session.store.resolve({ space: this.ctx.space, owner: caller.owner, actor: caller.actor, uid: caller.uid, generation: caller.generation }, session.sourceIsLive);
+                if (!importedIssuedPermitsSubject(resolved.evidence.permissions.publish, subject))
+                    throw new EpEnvelopeError("permission-denied", "run-cancel request is not authorized");
+            });
+            budget();
+        };
+        await current();
+        // Cancellation reads records and admission after revocation, and also before a spec exists.
+        // Use the existing endpoint inventory reader: run-scoped issuance requires an unrevoked
+        // admission. This reader carries no answer writes and cancellation reads no run journal.
+        const read = (fn) => this.withOperator({}, fn);
+        const admission = await read(planes => readCancellationAdmission(planes.jsm, this.ctx.space, this.ctx.endpoint, input.runId));
+        budget();
+        const original = admission.admission.caller;
+        if (!this.remote && (original.owner !== caller.owner || original.actor !== caller.actor || original.uid !== caller.uid))
+            throw new EpEnvelopeError("permission-denied", "run-cancel request is not authorized");
+        const readStatus = async () => {
+            budget();
+            try {
+                const status = await read(planes => readRunCancellationStatus(planes.jsm, this.ctx.space, this.ctx.endpoint, input.runId));
+                budget();
+                return status;
+            }
+            catch (error) {
+                budget();
+                throw error;
+            }
+        };
+        let status = await readStatus();
+        let marker = admission.revoked;
+        let duplicate = marker !== undefined;
+        const reply = (phase, driveOutcome) => ({
+            runId: input.runId, authority, duplicate,
+            ...(marker ? { revocation: { by: marker.by, reason: marker.reason, revokedAt: marker.revokedAt } } : {}),
+            observed: { processEpoch: ctx.identity.epoch, phase,
+                ...(status ? { recordedState: status.state, holder: status.holder, runEpoch: status.epoch, fencingToken: status.fencingToken } : {}),
+                ...(driveOutcome ? { driveOutcome } : {}) },
+        });
+        if (status?.state === "completed" || status?.state === "failed") {
+            if (marker) {
+                this.observeCancellation(input.runId);
+                this.clearObservedCancellation(input.runId);
+            }
+            return reply("terminal");
+        }
+        const slot = this.runs.get(input.runId);
+        if ((!slot || !this.cancelEligible(slot, status)) && status?.state === "running") {
+            if (!marker)
+                throw new EpEnvelopeError("failed-precondition", "run is not driven by this manager's current attempt");
+            this.observeCancellation(input.runId);
+            this.clearObservedCancellation(input.runId);
+            return reply("not-driven-here");
+        }
+        let intent = this.cancellations.get(subject);
+        if (intent && (intent.runId !== input.runId || intent.reason !== input.reason))
+            throw new EpEnvelopeError("permission-denied", "run-cancel request identity changed");
+        if (intent && !marker)
+            throw new EpEnvelopeError("deadline-exceeded", "run-cancel request remains pending; no second dispatch", undefined, "unknown");
+        if (!intent) {
+            intent = { ...input, observed: false, mayHaveDispatched: false };
+            this.cancellations.set(subject, intent);
+        }
+        if (!marker) {
+            try {
+                if (this.remote) {
+                    budget();
+                    const olderUnknown = intent.mayHaveDispatched;
+                    intent.mayHaveDispatched = true;
+                    const committed = await this.ctx.cancelRun({ stage: "commit", subject, cancel: input });
+                    if (committed.stage !== "commit" || committed.runId !== input.runId)
+                        throw new EpEnvelopeError("unavailable", "invalid run-cancel commit response");
+                    if (committed.result.kind !== "committed") {
+                        // A no-dispatch response cannot erase ambiguity from an older call or issuer lifetime.
+                        if (committed.result.kind === "refused-before-dispatch" && !olderUnknown)
+                            intent.mayHaveDispatched = false;
+                        throw new EpEnvelopeError(committed.result.code, "run-cancel issuer did not establish a marker", undefined, committed.result.kind === "outcome-unknown" ? "unknown" : "not-executed");
+                    }
+                    duplicate = committed.result.outcome === "existed";
+                    marker = committed.result.revocation;
+                }
+                else {
+                    let created;
+                    const outcome = await this.withAdmitter(input.runId, async (kv) => {
+                        await current();
+                        created = { version: 1, runId: input.runId, by: `${caller.owner}.${caller.actor}`, reason: input.reason, revokedAt: Date.now() };
+                        return commitRunCancellation(kv, this.ctx.endpoint, created, () => {
+                            budget();
+                            intent.mayHaveDispatched = true;
+                        });
+                    });
+                    duplicate = outcome === "existed";
+                    marker = outcome === "created" ? created
+                        : await read(planes => readRunRevocation(planes.jsm, this.ctx.space, this.ctx.endpoint, input.runId));
+                    if (!marker)
+                        throw new EpEnvelopeError("unavailable", "run-cancel marker is not yet observed");
+                }
+            }
+            catch (error) {
+                if (performance.now() < endsAt) {
+                    try {
+                        marker = await read(planes => readRunRevocation(planes.jsm, this.ctx.space, this.ctx.endpoint, input.runId));
+                    }
+                    catch { /* Unknown reads preserve the pending intent. */ }
+                }
+                if (marker)
+                    duplicate = null;
+                else {
+                    // A prior possible dispatch is sticky, including across another invocation of this key.
+                    if (!intent.mayHaveDispatched && performance.now() < endsAt) {
+                        try {
+                            const absent = await read(planes => readRunRevocation(planes.jsm, this.ctx.space, this.ctx.endpoint, input.runId));
+                            if (!absent && this.cancellations.get(subject) === intent)
+                                this.cancellations.delete(subject);
+                        }
+                        catch { /* No absence proof. */ }
+                    }
+                    if (intent.mayHaveDispatched)
+                        throw new EpEnvelopeError("deadline-exceeded", "run-cancel write outcome is unknown", undefined, "unknown");
+                    throw error;
+                }
+            }
+        }
+        this.observeCancellation(input.runId);
+        if (slot?.outcome) {
+            this.clearObservedCancellation(input.runId);
+            return reply("stopped", slot.outcome);
+        }
+        if (!slot || this.runs.get(input.runId) !== slot) {
+            this.clearObservedCancellation(input.runId);
+            return reply(slot && !slot.drive ? "launch-abandoned" : "not-driven-here");
+        }
+        // A preactivation slot may have activated while the marker was being written. Re-observe
+        // before selecting its handle; an old holder observation cannot stop a superseded attempt.
+        if (slot.activated)
+            status = await readStatus();
+        if (slot.outcome) {
+            this.clearObservedCancellation(input.runId);
+            return reply("stopped", slot.outcome);
+        }
+        if (!this.cancelEligible(slot, status))
+            return reply("not-driven-here");
+        if (slot.drive)
+            slot.drive.cancel(input.reason);
+        while (performance.now() < endsAt) {
+            if (slot.outcome) {
+                this.clearObservedCancellation(input.runId);
+                return reply("stopped", slot.outcome);
+            }
+            if (this.runs.get(input.runId) !== slot && !slot.drive) {
+                this.clearObservedCancellation(input.runId);
+                return reply("launch-abandoned");
+            }
+            await new Promise(resolve => setTimeout(resolve, Math.min(25, Math.max(1, endsAt - performance.now()))));
+        }
+        return reply("stop-requested");
+    }
     /** Revoke a hosted run (SPEC 14.8): the marker is create-only and independent of the immutable
      *  admission; every later channel effect of the run refuses on reading it, and a drive parked in
      *  a wait ends within one poll. Idempotent. */
@@ -551,6 +887,7 @@ export class RunHosting {
      *  between the check and the set. A held slot is a conflict for every later claimant until the
      *  attempt that holds it ends. */
     claim(runId, placements = []) {
+        this.assertNoCancel(runId);
         if (this.runs.has(runId))
             throw new EpEnvelopeError("conflict", `run ${runId} is being driven by this manager already`);
         const takeoverId = newTakeoverId();
@@ -563,24 +900,19 @@ export class RunHosting {
     free(slot) {
         if (this.runs.get(slot.runId) === slot)
             this.runs.delete(slot.runId);
+        this.clearObservedCancellation(slot.runId);
     }
     /** Launch one attempt on a slot. The slot is claimed here when the caller has not already
      *  (`start`), and is this launch's to give back from the first line: every refusal below,
      *  including the two before a drive is attempted, frees a slot whose drive never started. */
-    async launch(host, req, claimed) {
+    async launch(host, req, claimed, reserved) {
         const slot = claimed ?? this.claim(req.runId, req.placements ?? []);
+        let release = reserved;
         try {
             if (this.stopping)
                 throw new EpEnvelopeError("unavailable", "the manager is stopping and hosts no new drives");
-            if (this.launching >= MAX_LAUNCHING)
-                throw new EpEnvelopeError("resource-exhausted", `${MAX_LAUNCHING} workflow runs are launching on this manager already; retry once one has activated`);
-            this.launching += 1;
-            try {
-                await this.drive(host, req, slot);
-            }
-            finally {
-                this.launching -= 1;
-            }
+            release ??= this.reserveLaunch();
+            await this.drive(host, req, slot);
         }
         catch (e) {
             // A slot whose drive never started is freed here; one whose drive exists is freed by that
@@ -589,6 +921,22 @@ export class RunHosting {
                 this.free(slot);
             throw e;
         }
+        finally {
+            release?.();
+        }
+    }
+    /** Take one launching unit, or refuse while stopping or at MAX_LAUNCHING. The release is idempotent. */
+    reserveLaunch() {
+        if (this.stopping)
+            throw new EpEnvelopeError("unavailable", "the manager is stopping and hosts no new drives");
+        if (this.launching >= MAX_LAUNCHING)
+            throw new EpEnvelopeError("resource-exhausted", `${MAX_LAUNCHING} workflow runs are launching on this manager already; retry once one has activated`);
+        this.launching += 1;
+        let held = true;
+        return () => { if (held) {
+            held = false;
+            this.launching -= 1;
+        } };
     }
     async drive(host, req, slot) {
         const { takeoverId, identity, mediatorIdentity } = slot;
@@ -639,6 +987,7 @@ export class RunHosting {
             // connect has already cleared the live map, and a drive started now would be nobody's.
             if (this.stopping)
                 throw new EpEnvelopeError("unavailable", "the manager is stopping and hosts no new drives");
+            this.assertNoCancel(req.runId);
         }
         catch (e) {
             await mediatorNc?.close();
@@ -662,6 +1011,9 @@ export class RunHosting {
                 epoch: req.epoch,
                 defaultCheckpointTimeout: req.timeout,
                 admission: req.admission,
+                cancellationObserved: () => this.runs.get(req.runId) === holder
+                    ? [...this.cancellations.values()].find(entry => entry.runId === req.runId && entry.observed)?.reason
+                    : undefined,
                 // The broker's own max_payload, minus headroom for the record envelope around the entry.
                 ...(typeof max === "number" && max > 4096 ? { resultBytes: max - 4096 } : {}),
             }, mediator);
@@ -674,8 +1026,10 @@ export class RunHosting {
         holder.nc = nc;
         holder.mediatorNc = mediatorNc;
         holder.drive = drive;
-        const activation = this.activated(mediator.kv, req, drive);
+        holder.activated = false;
+        const activation = this.activated(mediator.kv, req, drive).then(() => { holder.activated = true; });
         void drive.done.then(async (out) => {
+            holder.outcome = out;
             this.ctx.log(`run ${req.runId}: ${describeOutcome(out)}`);
             this.free(holder);
             // The activation wait reads the record over this connection; it finishes before the drain.