firedrill-run 2.2.4 → 3.0.0

package/dist/cli/src/native/session.js

@@ -0,0 +1,556 @@
+/**
+ * Native companion session (Plan20 W09, spec 3.5).
+ *
+ * `runCompanion(options)` runs one companion for an approved local plan:
+ *  - verifies the in-memory approval, descriptor digest, plan roots and pinned
+ *    files before anything opens;
+ *  - opens the cloud session (reconnecting with capped exponential backoff and
+ *    jitter on transient failures) and heartbeats at the server interval with jitter;
+ *  - follows one selected campaign over root's single companion channel for this
+ *    device (companion-channel.ts): its invalidation hints wake the offer loop
+ *    and the drills, so nothing polls root per drill or on an idle timer, and
+ *    nothing is read or started while the channel waits for a valid resync;
+ *  - takes the campaign's offers made to its runner device (or requests them
+ *    for its waiting slots) and accepts only those whose descriptor digest
+ *    matches the approved plan and whose offer and deadline are live, with
+ *    bounded concurrency (`effectiveParallelism`, 1 when any state domain is serial);
+ *    an offer it will not run is rejected so the drill is not left waiting;
+ *  - runs an accepted drill with the driver from the approved local plan and
+ *    this client's own step and evidence ceilings, bounded by root's deadline;
+ *    simulated dependencies go through the drill's loopback proxy, and identity
+ *    stays empty until root delivers it;
+ *  - pauses new steps while disconnected (connection gate) and never replays an
+ *    in-flight step after reconnecting (the drill runner keeps it `unknown`);
+ *  - when the plan keeps shared services running across drills, keeps the
+ *    selected campaign's local run with root (local-run.ts): reconciled at start,
+ *    registered before the first offer, takes only that campaign's offers, and
+ *    closed after every service stopped, before the session closes;
+ *  - on revocation stops admitting work and drains; on the caller's signal stops
+ *    admitting, lets in-flight steps finish within their bounds (hard abort after
+ *    a grace period), stops every supervised process once, closes the session and
+ *    removes run scratch.
+ *
+ * Cloud messages reference the plan only by digest; no command, argv or path is
+ * accepted from them. Approval stays in memory for this process.
+ */
+import { EVIDENCE_MAX_TOTAL_BYTES, NATIVE_ID, nativeError } from './contracts.js';
+import { normalizeDescriptor } from './canonical.js';
+import { assertPlanApproved, verifyPlanFiles } from './plan.js';
+import { resolvePlanRoots } from './roots.js';
+import { effectiveParallelism, planStateActions } from './target-state.js';
+import { readinessDomains } from './readiness.js';
+import { validateDependencyRoutes } from './bindings.js';
+import { Supervisor } from './supervisor.js';
+import { ConnectionGate, backoffMs, errorCode, isAbort, isRevocation, isTransient, jitteredInterval, retryTransient, sleep, } from './control-plane.js';
+import { ServicePool, freshForDrill, runDrill } from './drill-runner.js';
+import { createRunScratch } from './scratch.js';
+import { CliError } from '../http.js';
+import { LocalRun, checkRunCompatible, runOwnedAliases } from './local-run.js';
+const DEFAULT_HEARTBEAT_MS = 15_000;
+const MIN_HEARTBEAT_MS = 1_000;
+const MAX_HEARTBEAT_MS = 300_000;
+const RECONNECT_BASE_MS = 1_000;
+const RECONNECT_CAP_MS = 60_000;
+const CLAIM_IDLE_BASE_MS = 1_000;
+const CLAIM_IDLE_CAP_MS = 15_000;
+const CLOSE_TIMEOUT_MS = 10_000;
+const MAX_DECLINED_REMEMBERED = 1_024;
+/** This client's own ceilings for one drill; root bounds participant steps and the drill deadline. */
+const LOCAL_MAX_STEPS = 1_000;
+const LOCAL_STEP_TIMEOUT_MS = 10 * 60 * 1000;
+/** A drill whose deadline leaves less than this is declined rather than started. */
+const MIN_DRILL_MS = 60_000;
+function heartbeatInterval(value) {
+    return typeof value === 'number' && Number.isSafeInteger(value)
+        ? Math.min(MAX_HEARTBEAT_MS, Math.max(MIN_HEARTBEAT_MS, value)) : DEFAULT_HEARTBEAT_MS;
+}
+function abortPromise(signal) {
+    return new Promise((resolve) => {
+        if (signal.aborted)
+            resolve();
+        else
+            signal.addEventListener('abort', () => resolve(), { once: true });
+    });
+}
+/** Why an offer is declined (reason code), or null when it may be accepted. */
+function admissionProblem(offer, plan, approval) {
+    if (!offer || typeof offer.assignment_id !== 'string' || !NATIVE_ID.test(offer.assignment_id)
+        || typeof offer.attempt_id !== 'string' || !NATIVE_ID.test(offer.attempt_id))
+        return 'native_assignment_invalid';
+    if (offer.state !== 'offered')
+        return 'native_offer_not_offered';
+    // The descriptor digest also binds the driver profile and adapter digest.
+    if (offer.descriptor_sha256 !== plan.descriptor_sha256 || offer.descriptor_sha256 !== approval.descriptor_sha256) {
+        return 'native_descriptor_mismatch';
+    }
+    try {
+        assertPlanApproved(approval, plan);
+    }
+    catch (error) {
+        return errorCode(error) ?? 'native_plan_changed';
+    }
+    const now = Date.now();
+    if (!(Date.parse(offer.offer_expires_at) > now))
+        return 'native_offer_expired';
+    if (!(Date.parse(offer.deadline_at) - now > MIN_DRILL_MS))
+        return 'native_deadline_too_short';
+    return null;
+}
+/**
+ * The runnable drill for an accepted receipt: identity, fence and lease from
+ * root; driver and plan digest from the approved local plan; step, timeout and
+ * evidence ceilings from this client, with elapsed time bounded by root's deadline.
+ */
+function runnableAssignment(receipt, descriptorSha256, plan, descriptor, approval) {
+    return {
+        assignment_id: receipt.assignment_id, attempt_id: receipt.attempt_id, campaign_id: receipt.campaign_id, drill_id: receipt.attempt_id,
+        lease_epoch: receipt.lease_epoch, state_generation: receipt.state_generation, lease_expires_at: receipt.lease_expires_at,
+        descriptor_sha256: descriptorSha256, plan_sha256: approval.plan_sha256,
+        driver: { profile: plan.driver.profile, adapter_sha256: descriptor.driver.adapter_sha256 },
+        identity: null,
+        budgets: { max_steps: LOCAL_MAX_STEPS, max_elapsed_ms: Math.max(1, Date.parse(receipt.deadline_at) - Date.now()),
+            step_timeout_ms: LOCAL_STEP_TIMEOUT_MS, max_observation_bytes: EVIDENCE_MAX_TOTAL_BYTES },
+        step_cursor: 0, observation_ack: receipt.observation_sequence, observation_elapsed_ms: receipt.observation_elapsed_ms,
+    };
+}
+export async function runCompanion(options) {
+    const { controlPlane, plan, descriptor, capabilities, output, approval } = options;
+    // Local preconditions, before any cloud call.
+    assertPlanApproved(approval, plan);
+    const normalized = normalizeDescriptor(descriptor);
+    if (normalized.descriptor_sha256 !== plan.descriptor_sha256 || approval.descriptor_sha256 !== plan.descriptor_sha256) {
+        throw nativeError('plan_changed', 'The companion was not started: the approved plan was made for a different descriptor.', 2);
+    }
+    const driverCapability = capabilities.drivers.find((entry) => entry.profile === plan.driver.profile);
+    if (!driverCapability?.available) {
+        throw nativeError('driver_unavailable', `The companion was not started: the ${plan.driver.profile} driver is unavailable on this machine.`);
+    }
+    const roots = await resolvePlanRoots(plan);
+    await verifyPlanFiles(plan, normalized, roots);
+    // Checks that depend only on the approved plan, run again for every drill: a plan that fails them can never be
+    // admitted, and every drill accepted for it would use up a campaign slot. So nothing is taken.
+    readinessDomains(descriptor, planStateActions(descriptor, plan));
+    validateDependencyRoutes(descriptor, plan, { freshForDrill: freshForDrill(plan) });
+    const parallelism = Math.max(1, Math.min(effectiveParallelism(descriptor, plan), Number.isSafeInteger(capabilities.max_parallel_drills) && capabilities.max_parallel_drills > 0 ? capabilities.max_parallel_drills : 1));
+    // Shared launched services live for the campaign's local run, which root keeps per campaign and device.
+    const runAliases = runOwnedAliases(plan);
+    const campaignId = typeof options.campaignId === 'string' && options.campaignId ? options.campaignId.toLowerCase() : null;
+    // Root keeps one companion channel per device, for one campaign; without it nothing but per-drill polling would
+    // wake the drills, so a companion runs exactly the selected campaign's drills.
+    const wakes = controlPlane.wakes ?? null;
+    if (!campaignId || !wakes || wakes.campaign_id !== campaignId) {
+        throw new CliError('native_campaign_required', 'Select the campaign to run with --campaign <campaign-id>: the companion follows one campaign\'s offers and drills over a single channel. Nothing was started.', 2);
+    }
+    if (runAliases.length > 0) {
+        checkRunCompatible(descriptor, runAliases);
+    }
+    // Stop (graceful) and abort (hard) controls.
+    const stopping = new AbortController();
+    const hard = new AbortController();
+    const heartbeatStop = new AbortController();
+    let graceTimer = null;
+    let stopReason = 'stopped';
+    const beginShutdown = (reason) => {
+        if (stopping.signal.aborted)
+            return;
+        stopReason = reason;
+        stopping.abort();
+        graceTimer = setTimeout(() => hard.abort(), Math.max(0, options.shutdownGraceMs ?? 30_000));
+        graceTimer.unref?.();
+        output('native_companion_stopping', { reason }, 'Stopping: no new drills are admitted; in-flight drills finish their current step.');
+    };
+    const onCallerStop = () => beginShutdown('stopped');
+    if (options.signal.aborted)
+        onCallerStop();
+    else
+        options.signal.addEventListener('abort', onCallerStop, { once: true });
+    const counts = { completed: 0, failed: 0, cancelled: 0, incomplete: 0, declined: 0, evidence_incomplete: 0 };
+    const gate = new ConnectionGate();
+    let sessionId = null;
+    let revoked = false;
+    let fatal = null;
+    const markRevoked = () => {
+        if (revoked)
+            return;
+        revoked = true;
+        output('native_authorization_revoked', {}, 'Authorization was revoked: no new work is admitted and in-flight drills stop before their next step.');
+        beginShutdown('revoked');
+    };
+    const fail = (error) => { if (fatal === null)
+        fatal = error; beginShutdown('failed'); };
+    let scratch = null;
+    let supervisor = null;
+    let localRun = null;
+    const active = new Set();
+    const reconciliationPending = (reason) => {
+        if (stopping.signal.aborted)
+            return;
+        output('native_campaign_reconciliation_pending', { campaign_id: campaignId, reason_code: reason }, 'Cloud reconciliation remains pending. Stopping local work; original evidence, unknown outcomes and funding holds remain with the service.');
+        fail(new CliError(reason, 'Local work stopped with cloud reconciliation pending. Do not repeat an uncertain target action.', 2));
+    };
+    let heartbeat = Promise.resolve();
+    /** The offer loop's wake-ups from the campaign channel, and the channel state notices. */
+    let offers = null;
+    let unwatchChannel = () => { };
+    try {
+        scratch = await createRunScratch();
+        const perDrill = plan.services.filter((service) => service.mode === 'launch' && service.per_drill).length;
+        supervisor = new Supervisor({
+            plan, normalized, approval, roots,
+            maxProcesses: Math.min(1024, Math.max(1, plan.services.length - perDrill + perDrill * parallelism)),
+            outputBytes: options.outputBytes ?? 256 * 1024,
+            // Shared services start only under the launch ids registered for the campaign's local run.
+            runOwned: new Set(runAliases), runLaunchId: (alias) => localRun?.launchId(alias) ?? null,
+            beforeRunLaunch: async (alias) => {
+                if (!localRun)
+                    throw new CliError('native_local_run_unregistered', 'Register the local run before launching shared services.');
+                await localRun.beforeLaunch(alias);
+            },
+            captureRunObserved: async () => { if (localRun)
+                await localRun.captureObserved(); },
+            runNotStarted: async (alias) => { if (localRun)
+                await localRun.notStarted(alias); },
+        });
+        const activeSupervisor = supervisor;
+        const services = new ServicePool({ supervisor, plan, descriptor, signal: hard.signal });
+        // Open the session, reconnecting on transient failures until stopped.
+        const offered = { ...capabilities, max_parallel_drills: parallelism, isolation: 'native-trusted' };
+        let opened = null;
+        for (let attempt = 0; !opened && !stopping.signal.aborted; attempt++) {
+            try {
+                opened = await controlPlane.openSession(offered, AbortSignal.any([stopping.signal, hard.signal]));
+            }
+            catch (error) {
+                if (stopping.signal.aborted)
+                    break;
+                if (isRevocation(error)) {
+                    markRevoked();
+                    break;
+                }
+                if (!isTransient(error))
+                    throw error;
+                output('native_disconnected', { attempt: attempt + 1 }, 'The Firedrill service is unreachable; retrying the session.');
+                await sleep(backoffMs(attempt, RECONNECT_BASE_MS, RECONNECT_CAP_MS), stopping.signal);
+            }
+        }
+        if (opened) {
+            if (typeof opened.session_id !== 'string' || !NATIVE_ID.test(opened.session_id)) {
+                throw nativeError('disconnected', 'The companion session could not be confirmed: the session identifier is malformed.');
+            }
+            const id = opened.session_id;
+            sessionId = id;
+            const heartbeatMs = heartbeatInterval(opened.heartbeat_ms);
+            output('native_companion_started', { session_id: id, parallelism, driver: plan.driver.profile, isolation: 'native-trusted' }, `Companion connected (${parallelism} parallel drill${parallelism === 1 ? '' : 's'}, ${plan.driver.profile}). Native mode runs your services as trusted local processes; it is not a sandbox.`);
+            heartbeat = runHeartbeat({ controlPlane, sessionId: id, heartbeatMs, gate, output, stop: heartbeatStop.signal, hard: hard.signal,
+                onRevoked: markRevoked, onFatal: fail });
+            // The campaign's local run: reconciled with root before any offer; registered just before the first one.
+            if (runAliases.length > 0 && campaignId) {
+                // Root keeps the run per campaign and device. The session is the runner device's presence, so its id is the device key.
+                localRun = await LocalRun.open({
+                    controlPlane, campaignId, deviceKey: id, descriptorSha256: plan.descriptor_sha256, aliases: runAliases,
+                    observe: () => activeSupervisor.runStates(), output,
+                }, AbortSignal.any([stopping.signal, hard.signal]));
+                // A run that takes no more drills here (closed, uncertain, or left by an earlier companion) is only closed.
+                if (localRun.retired)
+                    fail(new CliError(localRun.retired.code, localRun.retired.message, 2));
+            }
+            const run = localRun;
+            const claimOptions = run ? { campaignOnly: true, beforeOffer: run.beforeOffer, runRegistered: () => run.registered } : undefined;
+            const declined = new Set();
+            const decline = (assignment, reason) => {
+                counts.declined++;
+                const key = `${assignment?.assignment_id}\0${assignment?.attempt_id}`;
+                declined.add(key);
+                if (declined.size > MAX_DECLINED_REMEMBERED)
+                    declined.delete(declined.values().next().value);
+                output('native_assignment_declined', { assignment_id: typeof assignment?.assignment_id === 'string' && NATIVE_ID.test(assignment.assignment_id) ? assignment.assignment_id : null, reason }, `Declined an offered drill (${reason}). Nothing was started for it.`);
+            };
+            const loopSignal = AbortSignal.any([stopping.signal, hard.signal]);
+            // The campaign channel wakes this loop; while it waits for a valid resync nothing is read, claimed or stepped.
+            if (wakes) {
+                let held = false;
+                unwatchChannel = wakes.onState((state, code) => {
+                    if (state === 'refused' || (state === 'reconnecting' && code === 'native_companion_stream_busy')) {
+                        if (held)
+                            return;
+                        held = true;
+                        output('native_channel_held', { state, error_code: code }, state === 'refused'
+                            ? 'The service refused this runner\'s campaign channel (access changed): no new drill or step starts until access is confirmed again.'
+                            : 'Another companion holds this runner\'s campaign channel: no new drill or step starts until it is free.');
+                    }
+                    else if (state === 'live' && held) {
+                        held = false;
+                        output('native_channel_resumed', {}, 'The campaign channel is confirmed again; drills continue.');
+                    }
+                });
+                offers = wakes.subscribe({ kind: 'campaign', id: campaignId });
+            }
+            let idle = 0;
+            /**
+             * An admitted offer whose accept outcome is unknown (the request may have
+             * landed). Its byte-identical accept is re-sent before anything new is
+             * claimed, so root returns the original answer and no accepted drill is stranded.
+             */
+            let acceptUnknown = null;
+            while (!stopping.signal.aborted) {
+                if (!await gate.wait(stopping.signal))
+                    break;
+                if (wakes && !await wakes.live(stopping.signal))
+                    break;
+                if (active.size >= parallelism) {
+                    await Promise.race([...active, abortPromise(stopping.signal)]);
+                    continue;
+                }
+                // Read before the canonical claim, so a wake-up that lands during it is not lost.
+                const generation = offers?.generation ?? 0;
+                const resumed = acceptUnknown;
+                acceptUnknown = null;
+                let claimed = resumed;
+                if (!resumed) {
+                    try {
+                        claimed = await controlPlane.claimAssignment(id, loopSignal, claimOptions);
+                    }
+                    catch (error) {
+                        if (stopping.signal.aborted || isAbort(error, loopSignal))
+                            break;
+                        if (isRevocation(error)) {
+                            markRevoked();
+                            break;
+                        }
+                        if (isTransient(error)) {
+                            if (gate.disconnected())
+                                output('native_disconnected', {}, 'Connection lost; new steps are paused until it returns.');
+                            continue;
+                        }
+                        fail(error);
+                        break;
+                    }
+                }
+                if (!claimed) {
+                    // Empty pages are not completion. First drain locally active drills,
+                    // then ask the authoritative campaign for sealed target-work closure.
+                    // Shared process shutdown and its signed run close still happen in
+                    // finally, independently of later cloud accounting settlement.
+                    if (active.size) {
+                        await Promise.race([...active, abortPromise(stopping.signal)]);
+                        continue;
+                    }
+                    if (controlPlane.targetWorkComplete || controlPlane.targetDispatchComplete) {
+                        try {
+                            if (await controlPlane.targetWorkComplete?.(loopSignal)) {
+                                output('native_target_work_complete', { campaign_id: campaignId }, 'All target work is closed. Stopping shared local services; cloud cleanup and accounting may still be pending.');
+                                break;
+                            }
+                            if (await controlPlane.targetDispatchComplete?.(loopSignal)) {
+                                output('native_target_dispatch_complete', { campaign_id: campaignId }, 'All target dispatch is closed. Stopping local services; cloud sealing, assessment and accounting may still be pending.');
+                                break;
+                            }
+                        }
+                        catch (error) {
+                            if (stopping.signal.aborted || isAbort(error, loopSignal))
+                                break;
+                            if (isRevocation(error)) {
+                                markRevoked();
+                                break;
+                            }
+                            if (isTransient(error)) {
+                                gate.disconnected();
+                                continue;
+                            }
+                            fail(error);
+                            break;
+                        }
+                    }
+                    // Nothing to take: the campaign's next change (an offer, a slot, admission) wakes this loop.
+                    if (offers)
+                        await offers.wait(generation, stopping.signal);
+                    else
+                        await sleep(backoffMs(idle++, CLAIM_IDLE_BASE_MS, CLAIM_IDLE_CAP_MS), stopping.signal);
+                    continue;
+                }
+                idle = 0;
+                if (stopping.signal.aborted)
+                    break;
+                const offer = claimed;
+                // Best effort: a rejected offer frees the drill now instead of at offer expiry.
+                const reject = () => { void controlPlane.rejectAssignment(offer, loopSignal).catch(() => undefined); };
+                if (!resumed) {
+                    if (declined.has(`${offer.assignment_id}\0${offer.attempt_id}`)) {
+                        if (offers)
+                            await offers.wait(generation, stopping.signal);
+                        else
+                            await sleep(backoffMs(0, CLAIM_IDLE_BASE_MS, CLAIM_IDLE_CAP_MS), stopping.signal);
+                        continue;
+                    }
+                    // Readiness preconditions: digests, driver, lease, and pinned files unchanged on disk.
+                    const problem = admissionProblem(offer, plan, approval);
+                    if (problem) {
+                        decline(offer, problem);
+                        reject();
+                        continue;
+                    }
+                    try {
+                        await verifyPlanFiles(plan, normalized, roots);
+                    }
+                    catch (error) {
+                        // A changed approved file voids the approval for every later drill.
+                        decline(offer, errorCode(error) ?? 'native_plan_changed');
+                        reject();
+                        fail(error);
+                        break;
+                    }
+                }
+                let acceptance;
+                try {
+                    acceptance = await retryTransient(() => controlPlane.acceptAssignment(offer, loopSignal), { signal: loopSignal, attempts: 3, onTransient: () => { gate.disconnected(); } });
+                }
+                catch (error) {
+                    if (stopping.signal.aborted)
+                        break;
+                    if (isRevocation(error)) {
+                        markRevoked();
+                        break;
+                    }
+                    // Outcome unknown: re-send the same accept next, after the connection returns. Nothing started.
+                    if (isTransient(error)) {
+                        acceptUnknown = offer;
+                        continue;
+                    }
+                    fail(error);
+                    break;
+                }
+                if (!acceptance?.accepted || !acceptance.receipt) {
+                    output('native_assignment_not_accepted', { assignment_id: offer.assignment_id, reason: typeof acceptance?.reason === 'string' && /^[a-z_]{1,64}$/.test(acceptance.reason) ? acceptance.reason : null }, 'The offered drill was not confirmed by the service. Nothing was started for it.');
+                    // The plane does not hand out a refused offer again. One refused for a reason other than its expiry is
+                    // rejected, so the drill is freed now (root's expiry settles an expired one). The next claim waits like
+                    // a declined one, so repeated refusals never become a tight loop of reads and signed requests.
+                    if (!acceptance?.receipt && acceptance?.reason !== 'native_offer_expired')
+                        reject();
+                    if (offers)
+                        await offers.wait(generation, stopping.signal);
+                    else
+                        await sleep(backoffMs(0, CLAIM_IDLE_BASE_MS, CLAIM_IDLE_CAP_MS), stopping.signal);
+                    continue;
+                }
+                output('native_assignment_accepted', { assignment_id: offer.assignment_id, attempt_id: offer.attempt_id,
+                    campaign_id: offer.campaign_id, deadline_at: offer.deadline_at }, 'Accepted an offered drill. It starts when the service has prepared it.');
+                const assignment = runnableAssignment(acceptance.receipt, offer.descriptor_sha256, plan, descriptor, approval);
+                const run = runDrill(assignment, {
+                    controlPlane, plan, descriptor, roots, services, scratch, gate, output,
+                    stopSignal: stopping.signal, abortSignal: hard.signal, isRevoked: () => revoked,
+                    observationBufferBytes: options.observationBufferBytes, localRun,
+                }).then((report) => {
+                    counts[report.outcome]++;
+                    if (!report.evidence?.evidence_complete)
+                        counts.evidence_incomplete++;
+                    if (report.reason_code === 'native_participant_outcome_unknown' || report.unknown_steps > 0
+                        || report.completion === 'not-reported') {
+                        reconciliationPending(report.reason_code === 'native_participant_outcome_unknown'
+                            ? report.reason_code : 'native_campaign_reconciliation_pending');
+                    }
+                }, () => {
+                    counts.failed++;
+                    counts.evidence_incomplete++;
+                    reconciliationPending('native_campaign_reconciliation_pending');
+                })
+                    .finally(() => { active.delete(run); });
+                active.add(run);
+            }
+        }
+    }
+    catch (error) {
+        fail(error);
+    }
+    finally {
+        beginShutdown(stopReason);
+        // Drain: drills stop before their next step; the grace timer hard-aborts stragglers.
+        await Promise.allSettled([...active]);
+        if (supervisor)
+            await supervisor.stopAll().catch(() => undefined);
+        if (localRun)
+            await localRun.captureObserved(true).catch((error) => {
+                output('native_local_run_journal_pending', { error_code: errorCode(error) }, 'Final shared-service observations could not be saved. Cleanup remains unresolved; do not assume the processes were recorded as stopped.');
+            });
+        // Every shared service is stopped: report how each ended and close the campaign's local run,
+        // while the device session (and its heartbeat) is still live; root refuses both once it stopped.
+        if (localRun && sessionId && !revoked) {
+            const run = localRun;
+            const wait = Math.max(0, options.runCloseWaitMs ?? 5 * 60_000);
+            await run.close(AbortSignal.timeout(wait + 60_000), AbortSignal.timeout(wait)).catch((error) => {
+                output('native_local_run_close_pending', { error_code: errorCode(error) }, `The campaign's local run was not closed. ${run.closeLaterHint}`);
+            });
+        }
+        // Every drill drained and the local run is settled: the campaign channel ends with the session.
+        offers?.close();
+        unwatchChannel();
+        wakes?.close();
+        heartbeatStop.abort();
+        await heartbeat.catch(() => undefined);
+        if (sessionId && !revoked) {
+            await controlPlane.closeSession(sessionId, AbortSignal.timeout(CLOSE_TIMEOUT_MS)).catch((error) => {
+                output('native_session_close_unconfirmed', { error_code: errorCode(error) }, 'The companion session close was not confirmed by the service.');
+            });
+        }
+        if (scratch)
+            await scratch.cleanup();
+        if (graceTimer)
+            clearTimeout(graceTimer);
+        options.signal.removeEventListener('abort', onCallerStop);
+    }
+    const result = {
+        session_id: sessionId, ended: fatal !== null ? 'failed' : revoked ? 'revoked' : stopReason,
+        error_code: fatal !== null ? (errorCode(fatal) ?? 'native_companion_failed') : revoked ? 'native_authorization_revoked' : null,
+        drills: counts,
+    };
+    output('native_companion_stopped', result, `Companion stopped (${result.ended}): ${counts.completed} completed, ${counts.failed} failed, ${counts.cancelled} cancelled, `
+        + `${counts.incomplete} incomplete, ${counts.declined} declined; ${counts.evidence_incomplete} with incomplete evidence.`);
+    if (fatal !== null)
+        throw fatal;
+    return result;
+}
+/**
+ * Heartbeat at the server interval (jittered, never above it) while connected;
+ * while disconnected, probe with capped exponential backoff and jitter and
+ * reopen the gate on the first success. A runner-reported disconnect wakes the
+ * loop early. Heartbeats continue while drills drain so their leases stay live.
+ */
+async function runHeartbeat(options) {
+    const { controlPlane, gate, output } = options;
+    let failures = 0;
+    while (!options.stop.aborted) {
+        const delay = gate.connected ? jitteredInterval(options.heartbeatMs) : backoffMs(failures, RECONNECT_BASE_MS, RECONNECT_CAP_MS);
+        const wake = new AbortController();
+        const unsubscribe = gate.onChange(() => { if (!gate.connected)
+            wake.abort(); });
+        await sleep(delay, AbortSignal.any([options.stop, wake.signal]));
+        unsubscribe();
+        if (options.stop.aborted)
+            return;
+        try {
+            const response = await controlPlane.heartbeat(options.sessionId, AbortSignal.any([options.hard, options.stop, AbortSignal.timeout(Math.max(MIN_HEARTBEAT_MS, options.heartbeatMs))]));
+            failures = 0;
+            if (gate.reconnected())
+                output('native_reconnected', {}, 'Connection restored; paused drills continue with their next step.');
+            if (response?.revoked === true) {
+                options.onRevoked();
+                return;
+            }
+        }
+        catch (error) {
+            if (options.stop.aborted)
+                return;
+            if (isRevocation(error)) {
+                options.onRevoked();
+                return;
+            }
+            if (isTransient(error) || isAbort(error)) {
+                failures++;
+                if (gate.disconnected())
+                    output('native_disconnected', {}, 'Connection lost; new steps are paused until it returns.');
+                continue;
+            }
+            options.onFatal(error);
+            return;
+        }
+    }
+}