@intentic/machine 1.325.0 → 1.327.0

package/dist/sync/mirror.js

@@ -1,17 +1,20 @@
 import { spawnSync } from "node:child_process";
 import net from "node:net";
-import { setTimeout as sleep } from "node:timers/promises";
+import { sleep } from "@intentic/base/async";
 import { errorMessage } from "@intentic/base/errors";
 import { plural } from "@intentic/base/format";
 import { writeFileAtomic } from "@intentic/base/fs";
 import { PortsListSchema } from "@intentic/sandbox-contract";
-import { mirrorHeartbeatPath, readState, removePairing, setFileSyncAutoPaused, setFileSyncSwapPaused, updateState, } from "./config.js";
+import { isAttachedPairing, mirrorHeartbeatPath, pairingKey, pairingTransport, readState, removeSandboxPairings, setFileSyncAutoPaused, setFileSyncSwapPaused, setSandboxGone, setUnreachableSince, updateState, } from "./config.js";
 import { pairingEndpoint } from "./endpoint.js";
-import { createDaemonBases, dialedPairings } from "../daemon-base.js";
+import { candidateBases, createDaemonBases, dialedPairings } from "../daemon-base.js";
+import { answeredGone, goneStep, SandboxGoneError } from "./gone.js";
+import { checkContainers, checkKeptHere, sandboxAnswered, sandboxSaidGone } from "./gone-watch.js";
+import { retireSandbox } from "./retire.js";
 import { readSwapRecords } from "../device/sandbox-rounds/swap-records.js";
 import { publishedPortsReader } from "./docker-ports.js";
 import { realBridgeExec, runGitBridge } from "./git-bridge.js";
-import { ensureMutagen, ensureSyncSession, existingSyncSessions, forwardedPorts, forwardSessionName, healDerivedConflicts, mutagenForwardArgs, ourForwardSessions, pauseRunningSync, pauseUnreachableSync, retireOrphanSessions, resumeAutoPausedSync, resumeSwapPausedSync, runMutagenAsync, syncMode, syncSessionNames, } from "./mutagen.js";
+import { ensureMutagen, ensureSyncSession, existingSyncSessions, forwardedPorts, forwardSessionName, healDerivedConflicts, MUTAGEN_CALL_TIMEOUT_MS, mutagenForwardArgs, ourForwardSessions, pauseRunningSync, pauseUnreachableSync, relabelBudget, retireOrphanSessions, resumeAutoPausedSync, resumeSwapPausedSync, runMutagenAsync, sayDaemonEventsTo, sessionLabels, syncMode, syncSessionNames, } from "./mutagen.js";
 import { quieted } from "./repeats.js";
 import { deviceReport, scopedReport } from "./report.js";
 import { pairingSshConfig, writeManagedSshConfig } from "./ssh.js";
@@ -37,6 +40,9 @@ export const fetchWorkspacePorts = async (base, syncToken) => {
     if (response.status === 401 || response.status === 403) {
         throw new SyncAuthError("the sandbox rejected the sync token: click 'Enable desktop sync' in your browser and re-run setup to mint a fresh one.");
     }
+    if (answeredGone(response)) {
+        throw new SandboxGoneError(`the platform says this sandbox no longer exists (${response.status}): ${await response.text()}`);
+    }
     if (!response.ok) {
         throw new Error(`reading the sandbox's ports failed (${response.status}): ${await response.text()}`);
     }
@@ -48,12 +54,17 @@ const localPortFree = (port) => new Promise((resolvePort) => {
     probe.listen(port, "127.0.0.1", () => probe.close(() => resolvePort(true)));
 });
 const mutagenExecutor = (mutagen, pairing, log) => ({
-    terminate: (port) => void spawnSync(mutagen, ["forward", "terminate", forwardSessionName(pairing.sandboxId, port)], { stdio: "ignore", windowsHide: true }),
+    terminate: (port) => void spawnSync(mutagen, ["forward", "terminate", forwardSessionName(pairing.sandboxId, port)], {
+        stdio: "ignore",
+        windowsHide: true,
+        timeout: MUTAGEN_CALL_TIMEOUT_MS,
+    }),
     create: async (summary) => await runMutagenAsync(mutagen, mutagenForwardArgs({
         name: forwardSessionName(pairing.sandboxId, summary.port),
         port: summary.port,
         remote: pairingEndpoint(pairing),
         host: summary.host,
+        labels: sessionLabels(pairing.sandboxId, "forward"),
     }), log),
     isLocalPortFree: localPortFree,
 });
@@ -141,8 +152,13 @@ export const strandedForwards = (held, mirrored) => {
     return held.filter((port) => !kept.has(port));
 };
 const sweepStrandedForwards = (mutagen, sandboxId, mirrored, log) => {
-    for (const port of strandedForwards(forwardedPorts(mutagen, sandboxId), mirrored)) {
-        spawnSync(mutagen, ["forward", "terminate", forwardSessionName(sandboxId, port)], { stdio: "ignore", windowsHide: true });
+    const held = forwardedPorts(mutagen, sandboxId);
+    for (const port of held === undefined ? [] : strandedForwards(held, mirrored)) {
+        spawnSync(mutagen, ["forward", "terminate", forwardSessionName(sandboxId, port)], {
+            stdio: "ignore",
+            windowsHide: true,
+            timeout: MUTAGEN_CALL_TIMEOUT_MS,
+        });
         log(`  localhost:${port}: stopped (this device was still holding a forward for it)`);
     }
 };
@@ -169,7 +185,13 @@ export const skippedPortsOf = (desired, mirrored, claimedBy, ignored) => {
         .map((summary) => {
         const heldBy = claimedBy.get(summary.port);
         const reason = ignored.has(summary.port) ? "ignored" : heldBy === undefined ? "busy" : "held-by-sandbox";
-        return { port: summary.port, host: summary.host, reason, heldBy: reason === "held-by-sandbox" ? heldBy : undefined, command: summary.command };
+        return {
+            port: summary.port,
+            host: summary.host,
+            reason,
+            heldBy: reason === "held-by-sandbox" ? heldBy : undefined,
+            command: summary.command,
+        };
     });
 };
 export const recordedForwards = (pairings) => new Map(pairings.flatMap((held) => (held.mirroredPorts ?? []).map((port) => [port.port, held.sandboxId])));
@@ -185,8 +207,8 @@ export const handOver = (holding, sandboxId, mirrored) => {
 };
 export const othersHolding = (holding, sandboxId) => new Map([...holding].filter(([, holder]) => holder !== sandboxId));
 const beat = async () => await writeFileAtomic(mirrorHeartbeatPath, String(Date.now())).catch(() => { });
-const savePorts = async (sandboxId, mirroredPorts, skippedPorts) => await updateState((state) => ({
-    pairings: state.pairings.map((held) => (held.sandboxId === sandboxId ? { ...held, mirroredPorts, skippedPorts } : held)),
+const savePorts = async (key, mirroredPorts, skippedPorts) => await updateState((state) => ({
+    pairings: state.pairings.map((held) => (pairingKey(held) === key ? { ...held, mirroredPorts, skippedPorts } : held)),
 }));
 const retireSwitchedOff = async (mutagen, pairing, log) => {
     const mirrored = pairing.mirroredPorts ?? [];
@@ -206,6 +228,9 @@ const servePairing = async (context, pairing, base) => {
     const { mutagen, claimedBy, holding, holders, say: log } = context;
     const baseline = pairing.mirroredPorts ?? [];
     const ports = pairing.syncToken === undefined ? [] : await fetchWorkspacePorts(base, pairing.syncToken);
+    if (isAttachedPairing(pairing)) {
+        return [];
+    }
     if (pairing.mirrorOff === true) {
         await retireSwitchedOff(mutagen, pairing, log);
         return [];
@@ -215,28 +240,45 @@ const servePairing = async (context, pairing, base) => {
     const next = await reconcileForwards(mutagenExecutor(mutagen, pairing, log), baseline, ports, claimedBy, ignored, holds, log);
     const skipped = skippedPortsOf(ports, next, othersHolding(holding, pairing.sandboxId), ignored);
     if (!sameMirrorSet(baseline, next) || !sameSkippedSet(pairing.skippedPorts ?? [], skipped)) {
-        await savePorts(pairing.sandboxId, next, skipped);
+        await savePorts(pairingKey(pairing), next, skipped);
     }
     return next;
 };
-const postReports = async (dialed, mutagen, unsupported, log) => {
-    const reportable = dialed.filter(({ pairing }) => pairing.syncToken !== undefined && !unsupported.has(pairing.sandboxId));
+export const reportCarriers = (dialed, unsupported) => {
+    const seen = new Set();
+    return dialed.filter(({ pairing }) => {
+        if (pairing.syncToken === undefined || pairing.goneSince !== undefined || unsupported.has(pairing.sandboxId) || seen.has(pairing.sandboxId)) {
+            return false;
+        }
+        seen.add(pairing.sandboxId);
+        return true;
+    });
+};
+const sendReport = async (base, pairing, report) => await fetch(`${base.replace(/\/$/, "")}/system/sync/report`, {
+    method: "POST",
+    headers: { "content-type": "application/json", "x-intentic-sync": pairing.syncToken ?? "" },
+    body: JSON.stringify(scopedReport(report, pairing.sandboxId)),
+    signal: AbortSignal.timeout(REPORT_TIMEOUT_MS),
+});
+const postReports = async (dialed, mutagen, unsupported, gone, log) => {
+    const reportable = reportCarriers(dialed, unsupported);
     if (reportable.length === 0) {
         return;
     }
     const report = await deviceReport(mutagen);
     for (const { pairing, base } of reportable) {
         try {
-            const response = await fetch(`${base.replace(/\/$/, "")}/system/sync/report`, {
-                method: "POST",
-                headers: { "content-type": "application/json", "x-intentic-sync": pairing.syncToken ?? "" },
-                body: JSON.stringify(scopedReport(report, pairing.sandboxId)),
-                signal: AbortSignal.timeout(REPORT_TIMEOUT_MS),
-            });
+            const response = await sendReport(base, pairing, report);
             if (response.status === 404) {
                 unsupported.add(pairing.sandboxId);
                 log(`  ${pairing.sandboxId}: this sandbox is running a daemon without machine reports, its Devices view will stay empty.`);
             }
+            else if (answeredGone(response)) {
+                await gone(pairing);
+            }
+            else if (!response.ok) {
+                log(`  ${pairing.sandboxId}: report not taken (${response.status})`);
+            }
         }
         catch (error) {
             log(`  ${pairing.sandboxId}: report skipped: ${errorMessage(error)}`);
@@ -244,9 +286,10 @@ const postReports = async (dialed, mutagen, unsupported, log) => {
     }
 };
 const dropRevokedPairing = async (mutagen, sandboxId, log) => {
-    await removePairing(sandboxId);
+    await removeSandboxPairings(sandboxId);
     retireOrphanSessions(mutagen, (await readState()).pairings, log);
 };
+const keyOfSandbox = (key, sandboxId) => key === sandboxId || key.startsWith(`${sandboxId}~`);
 const guard = async (log, what, step) => {
     try {
         await step();
@@ -268,27 +311,39 @@ const absorbRejectedPoll = async (mutagen, pairing, rejectedPolls, repos, sessio
     log(`${pairing.sandboxId} rejected the sync token ${REVOKED_POLLS} polls in a row: this machine's enrollment was revoked.`);
     rejectedPolls.delete(pairing.sandboxId);
     repos.delete(pairing.sandboxId);
-    sessionsPending.delete(pairing.sandboxId);
+    for (const key of [...sessionsPending].filter((held) => keyOfSandbox(held, pairing.sandboxId))) {
+        sessionsPending.delete(key);
+    }
     await dropRevokedPairing(mutagen, pairing.sandboxId, log);
     return true;
 };
 const absorbUnreachablePoll = async (mutagen, pairing, unreachable, log) => {
     const now = Date.now();
-    const since = unreachable.get(pairing.sandboxId)?.since ?? now;
+    const since = unreachable.get(pairing.sandboxId)?.since ?? pairing.unreachableSince ?? now;
     unreachable.set(pairing.sandboxId, { since, lastTried: now });
+    if (pairing.unreachableSince === undefined) {
+        await setUnreachableSince(pairing.sandboxId, since);
+    }
     const mirrored = (pairing.mirroredPorts ?? []).length;
     if (shouldReleaseForwards(now - since, mirrored)) {
         await retirePairingMirror(mutagen, pairing.sandboxId);
         log(`  ${pairing.sandboxId}: unreachable for ${RELEASE_FORWARDS_MS / 60_000} minutes; took its ${plural(mirrored, "port")} off localhost. They come back when it answers again.`);
     }
-    if (pairing.fileSyncAutoPaused === true || !shouldAutoPauseFileSync(now - since) || !pauseUnreachableSync(mutagen, pairing)) {
+    return await pauseIfLongUnreachable(mutagen, pairing, now - since, log);
+};
+const pauseIfLongUnreachable = async (mutagen, pairing, unreachableForMs, log) => {
+    if (pairing.fileSyncAutoPaused === true || !shouldAutoPauseFileSync(unreachableForMs) || !pauseUnreachableSync(mutagen, pairing)) {
         return false;
     }
-    await setFileSyncAutoPaused(pairing.sandboxId, true);
-    log(`  ${pairing.sandboxId}: unreachable for one hour; paused its Mutagen sessions to stop permanent reconnect/scanning load. They resume automatically when it returns.`);
+    await setFileSyncAutoPaused(pairingKey(pairing), true);
+    log(`  ${pairingKey(pairing)}: unreachable for one hour; paused its Mutagen sessions to stop permanent reconnect/scanning load. They resume automatically when it returns.`);
     return true;
 };
-const absorbPairingFailure = async (error, mutagen, pairing, tracking, log) => {
+const absorbPairingFailure = async (error, mutagen, pairing, tracking, fate, log) => {
+    if (error instanceof SandboxGoneError) {
+        await sandboxSaidGone(fate, pairing.sandboxId, "edge");
+        return { drop: true, paused: true };
+    }
     if (error instanceof SyncAuthError) {
         const drop = await absorbRejectedPoll(mutagen, pairing, tracking.rejectedPolls, tracking.repos, tracking.sessionsPending, log);
         return { drop, paused: false };
@@ -299,42 +354,97 @@ const healPairing = async (mutagen, pairing, tick, log) => {
     if (tick % HEAL_EVERY_TICKS !== 0) {
         return;
     }
-    await guard(log, `${pairing.sandboxId}: clearing derived residue`, async () => void (await healDerivedConflicts(mutagen, pairing, log)));
+    await guard(log, `${pairingKey(pairing)}: clearing derived residue`, async () => void (await healDerivedConflicts(mutagen, pairing, log)));
 };
-const setupOf = (pairing) => [pairing.sandboxId, pairing.localDir ?? "", pairing.syncToken ?? "", syncMode(pairing)].join("\n");
+const setupOf = (pairing) => [pairingKey(pairing), pairing.localDir ?? "", pairing.syncToken ?? "", syncMode(pairing), pairingTransport(pairing)].join("\n");
 export const unpreparedSetups = (pairings, prepared) => pairings.filter((held) => !prepared.has(setupOf(held)));
 export const markPrepared = (prepared, pairing) => void prepared.add(setupOf(pairing));
-const prepareSession = async (mutagen, pairing, pending, say) => {
+const prepareSession = async (mutagen, pairing, pending, relabels, say) => {
     let converged = false;
-    await guard(say, `${pairing.sandboxId}: preparing its file sync`, async () => {
-        converged = await ensureSyncSession(mutagen, pairing, say);
+    await guard(say, `${pairingKey(pairing)}: preparing its file sync`, async () => {
+        converged = await ensureSyncSession(mutagen, pairing, say, relabels);
     });
     if (converged) {
-        pending.delete(pairing.sandboxId);
+        pending.delete(pairingKey(pairing));
     }
     else {
-        pending.add(pairing.sandboxId);
+        pending.add(pairingKey(pairing));
     }
     return converged;
 };
-const dormant = (mutagen, pairing) => pairing.fileSyncAutoPaused === true && existingSyncSessions(mutagen, syncSessionNames(pairing)).length > 0;
-const prepareSessions = async (mutagen, pairings, prepared, pending, say) => {
-    for (const pairing of unpreparedSetups(pairings, prepared)) {
+const dormant = (mutagen, pairing) => pairing.goneSince !== undefined ||
+    pairing.fileSyncPausedFor !== undefined ||
+    (pairing.fileSyncAutoPaused === true && (existingSyncSessions(mutagen, syncSessionNames(pairing)) ?? [undefined]).length > 0);
+export const readyToPrepare = async (setups, firstCopyAhead, announce) => {
+    const told = new Map();
+    const ready = [];
+    for (const pairing of setups) {
+        if (isAttachedPairing(pairing) && pairing.mode === "sync" && pairing.localDir !== undefined && firstCopyAhead(pairing)) {
+            const said = told.get(pairing.sandboxId) ?? (await announce(pairing));
+            told.set(pairing.sandboxId, said);
+            if (!said) {
+                continue;
+            }
+        }
+        ready.push(pairing);
+    }
+    return ready;
+};
+const announcer = (mutagen, bases, unsupported, unreachable, gone, say) => async (pairing) => {
+    if (pairing.goneSince !== undefined) {
+        return false;
+    }
+    if (pairing.syncToken === undefined || unsupported.has(pairing.sandboxId)) {
+        return true;
+    }
+    if (!pollDue(unreachable.get(pairing.sandboxId), Date.now())) {
+        return false;
+    }
+    try {
+        const response = await sendReport(await bases.resolve(pairing), pairing, await deviceReport(mutagen));
+        if (response.status === 404) {
+            unsupported.add(pairing.sandboxId);
+            say(`  ${pairing.sandboxId}: this sandbox is running a daemon without machine reports, its Devices view will stay empty.`);
+            return true;
+        }
+        if (answeredGone(response)) {
+            await gone(pairing);
+            return false;
+        }
+        if (!response.ok) {
+            say(`  ${pairingKey(pairing)}: ${pairing.sandboxId} did not take the report naming this folder (${response.status}); its first copy waits for it.`);
+        }
+        return response.ok;
+    }
+    catch (error) {
+        say(`  ${pairingKey(pairing)}: ${pairing.sandboxId} could not be told of this folder (${errorMessage(error)}); its first copy waits until it can.`);
+        return false;
+    }
+};
+const prepareSessions = async (mutagen, pairings, prepared, pending, announce, passes, say) => {
+    const firstCopyAhead = (pairing) => (existingSyncSessions(mutagen, syncSessionNames(pairing)) ?? []).length === 0;
+    const unprepared = unpreparedSetups(pairings, prepared);
+    const docker = unprepared.filter((pairing) => pairingTransport(pairing) === "docker");
+    if (docker.length > 0) {
+        await guard(say, "asking after the containers docker pairings reach", async () => await checkContainers(passes.fate, docker, answeringOf(passes.tracking)));
+    }
+    for (const pairing of await readyToPrepare(unprepared, firstCopyAhead, announce)) {
         markPrepared(prepared, pairing);
         if (dormant(mutagen, pairing)) {
-            pending.add(pairing.sandboxId);
+            pending.add(pairingKey(pairing));
             continue;
         }
-        await prepareSession(mutagen, pairing, pending, say);
+        await prepareSession(mutagen, pairing, pending, passes.relabels, say);
     }
 };
-const retryPendingSessions = async (mutagen, pairings, sessionsPending, tick, say) => {
+const answeringOf = (tracking) => (sandboxId) => !tracking.unreachable.has(sandboxId);
+const retryPendingSessions = async (mutagen, pairings, sessionsPending, tick, relabels, say) => {
     if (sessionsPending.size === 0 || tick % SESSION_RETRY_EVERY_TICKS !== 0) {
         return;
     }
-    for (const pairing of pairings.filter((held) => sessionsPending.has(held.sandboxId) && !dormant(mutagen, held))) {
-        if (await prepareSession(mutagen, pairing, sessionsPending, say)) {
-            say(`  ${pairing.sandboxId}: file sync is running on this build's rules`);
+    for (const pairing of pairings.filter((held) => sessionsPending.has(pairingKey(held)) && !dormant(mutagen, held))) {
+        if (await prepareSession(mutagen, pairing, sessionsPending, relabels, say)) {
+            say(`  ${pairingKey(pairing)}: file sync is running on this build's rules`);
         }
     }
 };
@@ -350,19 +460,19 @@ const holdSyncDuringSwaps = async (mutagen, pairings, unreachable, say) => {
         }
         const step = swapPauseStep(pairing, holds);
         if (step === "pause" && pauseRunningSync(mutagen, pairing)) {
-            await setFileSyncSwapPaused(pairing.sandboxId, true);
-            say(`  ${pairing.sandboxId}: its sandbox is being swapped on this machine; file sync is paused until the new version has held.`);
+            await setFileSyncSwapPaused(pairingKey(pairing), true);
+            say(`  ${pairingKey(pairing)}: its sandbox is being swapped on this machine; file sync is paused until the new version has held.`);
         }
         else if (step === "resume") {
             resumeSwapPausedSync(mutagen, pairing);
-            await setFileSyncSwapPaused(pairing.sandboxId, false);
-            say(`  ${pairing.sandboxId}: its sandbox's swap is settled; file sync resumed.`);
+            await setFileSyncSwapPaused(pairingKey(pairing), false);
+            say(`  ${pairingKey(pairing)}: its sandbox's swap is settled; file sync resumed.`);
         }
     }
     return held;
 };
 const runPairingPass = async (context, pairing, base) => {
-    const { mutagen, tick, claimedBy, holding, tracking, bases, swapHeld, say } = context;
+    const { mutagen, tick, claimedBy, holding, tracking, bases, swapHeld, fate, say } = context;
     const { rejectedPolls, unreachable, repos } = tracking;
     const swapping = swapHeld.has(pairing.sandboxId);
     let pausedThisPass = false;
@@ -370,28 +480,36 @@ const runPairingPass = async (context, pairing, base) => {
         const mirrored = await servePairing(context, pairing, base);
         rejectedPolls.delete(pairing.sandboxId);
         unreachable.delete(pairing.sandboxId);
-        if (!swapping && resumeAutoPausedSync(mutagen, pairing)) {
-            await setFileSyncAutoPaused(pairing.sandboxId, false);
-            say(`  ${pairing.sandboxId}: reachable again; resumed its automatically paused file sync`);
+        if (context.pairings.some((held) => held.sandboxId === pairing.sandboxId && held.goneSince !== undefined)) {
+            await sandboxAnswered(fate, pairing.sandboxId);
         }
-        for (const port of mirrored) {
-            claimedBy.set(port.port, pairing.sandboxId);
+        if (context.pairings.some((held) => held.sandboxId === pairing.sandboxId && held.unreachableSince !== undefined)) {
+            await setUnreachableSince(pairing.sandboxId, undefined);
         }
-        handOver(holding, pairing.sandboxId, mirrored);
-        if (tick % STRANDED_SWEEP_EVERY_TICKS === 0) {
-            sweepStrandedForwards(mutagen, pairing.sandboxId, mirrored, say);
+        if (!swapping && resumeAutoPausedSync(mutagen, pairing)) {
+            await setFileSyncAutoPaused(pairingKey(pairing), false);
+            say(`  ${pairingKey(pairing)}: reachable again; resumed its automatically paused file sync`);
+        }
+        if (!isAttachedPairing(pairing)) {
+            for (const port of mirrored) {
+                claimedBy.set(port.port, pairing.sandboxId);
+            }
+            handOver(holding, pairing.sandboxId, mirrored);
+            if (tick % STRANDED_SWEEP_EVERY_TICKS === 0) {
+                sweepStrandedForwards(mutagen, pairing.sandboxId, mirrored, say);
+            }
         }
     }
     catch (error) {
         bases.failed(pairing.sandboxId);
-        const outcome = await absorbPairingFailure(error, mutagen, pairing, tracking, say);
+        const outcome = await absorbPairingFailure(error, mutagen, pairing, tracking, fate, say);
         if (outcome.drop) {
             return;
         }
         pausedThisPass = outcome.paused;
-        say(`  ${pairing.sandboxId}: reconcile skipped: ${errorMessage(error)}`);
+        say(`  ${pairingKey(pairing)}: reconcile skipped: ${errorMessage(error)}`);
     }
-    if (pairing.fileSyncAutoPaused === true || pausedThisPass || swapping) {
+    if (pairing.fileSyncAutoPaused === true || pairing.fileSyncPausedFor !== undefined || pausedThisPass || swapping) {
         return;
     }
     await healPairing(mutagen, pairing, tick, say);
@@ -409,83 +527,288 @@ const runPairingPass = async (context, pairing, base) => {
         say(`  ${pairing.sandboxId}: git bridge skipped: ${errorMessage(error)}`);
     }
 };
-export const runMirrorWatch = async (log) => {
-    const mutagen = await ensureMutagen();
-    const say = quieted(log);
-    const initial = await readState();
-    if (initial.pairings.length === 0) {
-        log("no sandboxes are paired: nothing to mirror. Enable it from a sandbox's Desktop sync card.");
+export const pollingPairings = (pairings) => {
+    const polling = new Set();
+    for (const sandboxId of new Set(pairings.map((pairing) => pairing.sandboxId))) {
+        const own = pairings.filter((pairing) => pairing.sandboxId === sandboxId);
+        const poller = own.find((pairing) => !isAttachedPairing(pairing)) ?? own[0];
+        if (poller !== undefined) {
+            polling.add(pairingKey(poller));
+        }
+    }
+    return polling;
+};
+const followSandbox = async (context, pairing) => {
+    const { mutagen, tick, tracking, swapHeld, say } = context;
+    if (pairing.goneSince !== undefined || pairing.fileSyncPausedFor !== undefined) {
         return;
     }
-    await guard(say, "refreshing the ssh configuration", async () => await writeManagedSshConfig(pairingSshConfig(initial.pairings)));
-    const tunnels = createTunnelPool(say);
-    const bases = createDaemonBases(say);
-    await guard(say, "opening the sync transports", async () => await tunnels.reconcile(tunnelTargets(await dialedPairings(initial.pairings, bases))));
-    const sessionsPending = new Set();
-    const sessionsPrepared = new Set();
-    await prepareSessions(mutagen, initial.pairings, sessionsPrepared, sessionsPending, say);
-    await guard(say, "retiring orphaned sessions", () => retireOrphanSessions(mutagen, initial.pairings, say));
-    log(`sync started; polling ${plural(initial.pairings.length, "paired sandbox")} every ${POLL_MS / 1000}s`);
-    const rejectedPolls = new Map();
-    const unreachable = new Map();
-    const repos = new Map();
-    const tracking = { rejectedPolls, unreachable, repos, sessionsPending };
-    const holders = { dockerPorts: publishedPortsReader(say), freeSince: new Map() };
-    const reportUnsupported = new Set();
-    for (let tick = 0;; tick += 1) {
-        const state = await readState().catch((error) => {
-            say(`  tick skipped: the sync state didn't read (${errorMessage(error)})`);
-            return undefined;
+    const down = tracking.unreachable.get(pairing.sandboxId)?.since ?? pairing.unreachableSince;
+    const swapping = swapHeld.has(pairing.sandboxId);
+    if (down === undefined) {
+        if (!swapping && resumeAutoPausedSync(mutagen, pairing)) {
+            await setFileSyncAutoPaused(pairingKey(pairing), false);
+            say(`  ${pairingKey(pairing)}: reachable again; resumed its automatically paused file sync`);
+        }
+    }
+    else if (await pauseIfLongUnreachable(mutagen, pairing, Date.now() - down, say)) {
+        return;
+    }
+    if (pairing.fileSyncAutoPaused === true || swapping) {
+        return;
+    }
+    await healPairing(mutagen, pairing, tick, say);
+};
+const passGone = async (context, pairing) => {
+    const now = Date.now();
+    const step = goneStep(pairing, now);
+    if (step === undefined) {
+        return "pass";
+    }
+    if (step === "recheck") {
+        await setSandboxGone(pairing.sandboxId, { since: pairing.goneSince ?? now, checkedAt: now, by: pairing.goneBy ?? "edge" });
+        return "ask";
+    }
+    if (step === "retire") {
+        const { mutagen, tracking, say } = context;
+        await guard(say, `${pairing.sandboxId}: retiring its pairing`, async () => {
+            await retireSandbox(pairing.sandboxId, say, `gone since ${new Date(pairing.goneSince ?? Date.now()).toISOString().slice(0, 10)}`, {
+                mutagen,
+            });
         });
-        if (state === undefined) {
-            await sleep(POLL_MS);
+        tracking.unreachable.delete(pairing.sandboxId);
+        tracking.rejectedPolls.delete(pairing.sandboxId);
+        tracking.repos.delete(pairing.sandboxId);
+        for (const key of [...tracking.sessionsPending].filter((held) => keyOfSandbox(held, pairing.sandboxId))) {
+            tracking.sessionsPending.delete(key);
+        }
+    }
+    return "skip";
+};
+const runPasses = async (context, dialed, pairings, live) => {
+    const polling = pollingPairings(pairings);
+    for (const { pairing, base } of dialed.filter((held) => polling.has(pairingKey(held.pairing)))) {
+        if (!live()) {
+            return;
+        }
+        const gone = await passGone(context, pairing);
+        if (gone === "skip") {
+            continue;
+        }
+        if (gone === "pass" && !pollDue(context.tracking.unreachable.get(pairing.sandboxId), Date.now())) {
             continue;
         }
-        if (state.pairings.length === 0) {
-            await tunnels.stopAll();
-            log("no sandboxes are paired any more: sync stopping. Re-enable from a sandbox's Desktop sync card.");
+        await runPairingPass(context, pairing, base);
+        context.progress(`${pairingKey(pairing)}'s pass`);
+    }
+    for (const { pairing } of dialed.filter((held) => !polling.has(pairingKey(held.pairing)))) {
+        if (!live()) {
             return;
         }
-        const dialed = await dialedPairings(state.pairings, bases);
-        await guard(say, "reconciling the sync transports", async () => await tunnels.reconcile(tunnelTargets(dialed)));
-        await prepareSessions(mutagen, state.pairings, sessionsPrepared, sessionsPending, say);
-        await retryPendingSessions(mutagen, state.pairings, sessionsPending, tick, say);
-        let swapHeld = new Set();
-        await guard(say, "holding file sync still during swaps", async () => {
-            swapHeld = await holdSyncDuringSwaps(mutagen, state.pairings, unreachable, say);
-        });
-        const claimedBy = new Map();
-        const holding = recordedForwards(state.pairings);
-        for (const { pairing, base } of dialed) {
-            if (!pollDue(unreachable.get(pairing.sandboxId), Date.now())) {
-                continue;
+        await guard(context.say, `${pairingKey(pairing)}: following its sandbox`, async () => await followSandbox(context, pairing));
+    }
+};
+const ORPHAN_SWEEP_EVERY_TICKS = 12;
+const CONTAINER_CHECK_EVERY_TICKS = 12;
+const KEPT_HERE_CHECK_EVERY_TICKS = 120;
+const reachedLocally = (dialed) => new Set(dialed.flatMap(({ pairing, base }) => {
+    const candidates = candidateBases(pairing.sandboxUrl);
+    return candidates.length > 1 && candidates[0] === base ? [pairing.sandboxId] : [];
+}));
+const standingSweeps = async (context, dialed) => {
+    const { mutagen, tick, tracking, fate, say } = context;
+    if (tick > 0 && tick % ORPHAN_SWEEP_EVERY_TICKS === 0) {
+        await guard(say, "retiring orphaned sessions", async () => retireOrphanSessions(mutagen, (await readState()).pairings, say));
+    }
+    if (tick % CONTAINER_CHECK_EVERY_TICKS === 0) {
+        await guard(say, "asking after the containers docker pairings reach", async () => await checkContainers(fate, (await readState()).pairings, answeringOf(tracking)));
+    }
+    if (tick % KEPT_HERE_CHECK_EVERY_TICKS === 0) {
+        await guard(say, "asking this machine's ic which sandboxes it keeps", async () => await checkKeptHere(fate, (await readState()).pairings, reachedLocally(dialed), undefined, answeringOf(tracking)));
+    }
+};
+export const WATCHER_STALL_MS = 20 * 60_000;
+const STALL_CHECK_MS = 30_000;
+export const watcherStalled = (progress, now) => now - progress >= WATCHER_STALL_MS;
+export const runMirrorWatch = async (log) => {
+    for (;;) {
+        if ((await watchGeneration(log)) === "done") {
+            return;
+        }
+    }
+};
+const watchGeneration = async (log) => {
+    const abandon = new AbortController();
+    const generation = { signal: abandon.signal, progress: Date.now(), step: "starting", stopTransports: undefined };
+    const loop = watchLoop(log, generation);
+    let timer;
+    const stall = new Promise((resolve) => {
+        timer = setInterval(() => {
+            if (watcherStalled(generation.progress, Date.now())) {
+                resolve("stalled");
             }
-            await runPairingPass({ mutagen, tick, claimedBy, holding, tracking, bases, swapHeld, holders, say }, pairing, base);
+        }, STALL_CHECK_MS);
+    });
+    try {
+        const outcome = await Promise.race([loop.then(() => "done"), stall]);
+        if (outcome === "stalled") {
+            abandon.abort();
+            loop.catch(() => undefined);
+            log(`sync watcher: no progress for ${WATCHER_STALL_MS / 60_000} minutes (it was at: ${generation.step}); restarting it. Its heartbeat had gone stale, which \`intentic-machine status\` reports.`);
+            await generation
+                .stopTransports?.()
+                .catch((error) => log(`sync watcher: could not stop the stalled watcher's tunnels (${errorMessage(error)})`));
         }
-        if (tick % REPORT_EVERY_TICKS === 0) {
-            await guard(say, "posting this machine's reports", async () => await postReports(dialed, mutagen, reportUnsupported, say));
+        return outcome;
+    }
+    finally {
+        clearInterval(timer);
+    }
+};
+const watchLoop = async (log, generation) => {
+    const watch = await startWatch(log, generation);
+    if (watch === undefined) {
+        return;
+    }
+    for (let tick = 0; watch.live(); tick += 1) {
+        if ((await watchTick(watch, tick)) === "stop") {
+            return;
         }
-        await beat();
-        await sleep(POLL_MS);
+        await sleep(POLL_MS, { signal: generation.signal });
+    }
+};
+const startWatch = async (log, generation) => {
+    const mutagen = await ensureMutagen();
+    const say = quieted(log);
+    sayDaemonEventsTo(say);
+    const initial = await readState();
+    if (initial.pairings.length === 0) {
+        log("no sandboxes are paired: nothing to mirror. Enable it from a sandbox's Desktop sync card.");
+        return undefined;
     }
+    const sshConfig = pairingSshConfig(initial.pairings);
+    await guard(say, "refreshing the ssh configuration", async () => await writeManagedSshConfig(sshConfig));
+    const tunnels = createTunnelPool(say, generation.signal);
+    generation.stopTransports = tunnels.stopAll;
+    const bases = createDaemonBases(say);
+    await guard(say, "opening the sync transports", async () => await tunnels.reconcile(tunnelTargets(await dialedPairings(initial.pairings, bases))));
+    const tracking = { rejectedPolls: new Map(), unreachable: new Map(), repos: new Map(), sessionsPending: new Set() };
+    const reportUnsupported = new Set();
+    const fate = { mutagen, releaseForwards: async (sandboxId) => await retirePairingMirror(mutagen, sandboxId), say };
+    const gone = async (pairing) => await sandboxSaidGone(fate, pairing.sandboxId, "edge");
+    const watch = {
+        log,
+        say,
+        mutagen,
+        live: () => !generation.signal.aborted,
+        progress: (step) => {
+            generation.signal.throwIfAborted();
+            generation.progress = Date.now();
+            generation.step = step;
+        },
+        tunnels,
+        bases,
+        tracking,
+        sessionsPrepared: new Set(),
+        reportUnsupported,
+        fate,
+        gone,
+        announce: announcer(mutagen, bases, reportUnsupported, tracking.unreachable, gone, say),
+        holders: { dockerPorts: publishedPortsReader(say), freeSince: new Map() },
+        sshConfig,
+    };
+    await prepareSessions(mutagen, initial.pairings, watch.sessionsPrepared, tracking.sessionsPending, watch.announce, { fate, relabels: relabelBudget(1), tracking }, say);
+    await guard(say, "retiring orphaned sessions", () => retireOrphanSessions(mutagen, initial.pairings, say));
+    log(`sync started; polling ${plural(initial.pairings.length, "paired sandbox")} every ${POLL_MS / 1000}s`);
+    return watch;
+};
+const watchTick = async (watch, tick) => {
+    const { mutagen, say, tracking, fate, progress } = watch;
+    progress("the start of a pass");
+    const state = await readState().catch((error) => {
+        say(`  tick skipped: the sync state didn't read (${errorMessage(error)})`);
+        return undefined;
+    });
+    if (state === undefined) {
+        return "next";
+    }
+    if (state.pairings.length === 0) {
+        await watch.tunnels.stopAll();
+        watch.log("no sandboxes are paired any more: sync stopping. Re-enable from a sandbox's Desktop sync card.");
+        return "stop";
+    }
+    const wantedConfig = pairingSshConfig(state.pairings);
+    if (wantedConfig !== watch.sshConfig) {
+        await guard(say, "refreshing the ssh configuration", async () => await writeManagedSshConfig(wantedConfig));
+        watch.sshConfig = wantedConfig;
+    }
+    const dialed = await dialedPairings(state.pairings, watch.bases);
+    await guard(say, "reconciling the sync transports", async () => await watch.tunnels.reconcile(tunnelTargets(dialed)));
+    progress("the sync transports");
+    const relabels = relabelBudget(1);
+    await prepareSessions(mutagen, state.pairings, watch.sessionsPrepared, tracking.sessionsPending, watch.announce, { fate, relabels, tracking }, say);
+    await retryPendingSessions(mutagen, state.pairings, tracking.sessionsPending, tick, relabels, say);
+    progress("preparing file sync");
+    if (!watch.live()) {
+        return "stop";
+    }
+    let swapHeld = new Set();
+    await guard(say, "holding file sync still during swaps", async () => {
+        swapHeld = await holdSyncDuringSwaps(mutagen, state.pairings, tracking.unreachable, say);
+    });
+    const context = {
+        mutagen,
+        tick,
+        claimedBy: new Map(),
+        holding: recordedForwards(state.pairings),
+        tracking,
+        bases: watch.bases,
+        swapHeld,
+        holders: watch.holders,
+        fate,
+        relabels,
+        progress,
+        pairings: state.pairings,
+        say,
+    };
+    await runPasses(context, dialed, state.pairings, watch.live);
+    if (!watch.live()) {
+        return "stop";
+    }
+    if (tick % REPORT_EVERY_TICKS === 0) {
+        await guard(say, "posting this machine's reports", async () => await postReports(dialed, mutagen, watch.reportUnsupported, watch.gone, say));
+    }
+    await standingSweeps(context, dialed);
+    progress("the end of a pass");
+    await beat();
+    return "next";
 };
 const teardownForwards = async (mutagen, sandboxId) => {
-    const names = ourForwardSessions(mutagen, sandboxId);
+    const recorded = async () => (await readState()).pairings
+        .filter((held) => sandboxId === undefined || held.sandboxId === sandboxId)
+        .flatMap((held) => (held.mirroredPorts ?? []).map((port) => forwardSessionName(held.sandboxId, port.port)));
+    const names = ourForwardSessions(mutagen, sandboxId) ?? (await recorded());
     if (names.length > 0) {
-        spawnSync(mutagen, ["forward", "terminate", ...names], { stdio: "ignore", windowsHide: true });
+        spawnSync(mutagen, ["forward", "terminate", ...names], { stdio: "ignore", windowsHide: true, timeout: MUTAGEN_CALL_TIMEOUT_MS });
     }
     await updateState((state) => ({
-        pairings: state.pairings.map((held) => sandboxId === undefined || held.sandboxId === sandboxId ? { ...held, mirroredPorts: [], skippedPorts: [] } : held),
+        pairings: state.pairings.map((held) => (sandboxId === undefined || held.sandboxId === sandboxId) && !isAttachedPairing(held)
+            ? { ...held, mirroredPorts: [], skippedPorts: [] }
+            : held),
     }));
     return names.length;
 };
 export const retirePairingMirror = async (mutagen, sandboxId) => await teardownForwards(mutagen, sandboxId);
 export const retireMirroredPort = async (mutagen, sandboxId, port) => {
-    spawnSync(mutagen, ["forward", "terminate", forwardSessionName(sandboxId, port)], { stdio: "ignore", windowsHide: true });
+    spawnSync(mutagen, ["forward", "terminate", forwardSessionName(sandboxId, port)], {
+        stdio: "ignore",
+        windowsHide: true,
+        timeout: MUTAGEN_CALL_TIMEOUT_MS,
+    });
     let wasMirrored = false;
     await updateState((state) => ({
         pairings: state.pairings.map((held) => {
-            if (held.sandboxId !== sandboxId) {
+            if (held.sandboxId !== sandboxId || isAttachedPairing(held)) {
                 return held;
             }
             const mirrored = held.mirroredPorts ?? [];