@intentic/machine 1.315.0 → 1.317.0

package/dist/device/sandbox-rounds/auto-backup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auto-backup.d.ts","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/auto-backup.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAGhE,OAAO,EAAE,KAAK,KAAK,EAA2B,KAAK,UAAU,EAAE,KAAK,MAAM,EAAyB,MAAM,gBAAgB,CAAC;AAiB1H,eAAO,MAAM,aAAa,UAAW,SAAS,aAAa,EAAE,KAAG,MAAM,EACiD,CAAC;AAGxH,eAAO,MAAM,cAAc,SAAU,MAAM,KAAG,MAAM,EAAqD,CAAC;AAsB1G,eAAO,MAAM,UAAU,SAAU,MAAM,UAAU,MAAM,KAAG,MAYzD,CAAC;AAEF,eAAO,MAAM,cAAc,QAAO,UAA6B,CAAC;AAGhE,eAAO,MAAM,cAAc,UAChB,UAAU,SACV,SAAS,aAAa,EAAE,UACvB,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,OACnC,GAAG,SACF,WAAW,CAAC,MAAM,CAAC,KAC1B,OAAO,CAAC,IAAI,CACoI,CAAC;AAMpJ,eAAO,MAAM,YAAY,QAAO,MAAM,EAAmC,CAAC;AAqB1E,eAAO,MAAM,QAAQ,QAAS,KAAK,KAAG,MAiBrC,CAAC;AAGF,eAAO,MAAM,OAAO,SAAgB,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,KAAG,OAAO,CAAC,IAAI,CAMhF,CAAC;AAIF,eAAO,MAAM,eAAe,QAAS,GAAG,KAAG,MAiB1C,CAAC"}

package/dist/device/sandbox-rounds/auto-backup.js

@@ -0,0 +1,97 @@
+import { errorMessage } from "@intentic/base/errors";
+import { plural } from "@intentic/base/format";
+import { z } from "zod";
+import { readMachineConfig } from "../../environments/machine.js";
+import { lastLine, newRoundState, runRound, startRounds } from "./ic-rounds.js";
+import { fleet, icInFlight, runIc } from "../tools/sandboxes.js";
+const FIRST_TICK_MS = 20 * 60_000;
+const FIRST_SPREAD_MS = 20 * 60_000;
+const TICK_MS = 24 * 60 * 60_000;
+const JITTER_MS = 60 * 60_000;
+export const backupTargets = (boxes) => boxes.filter((box) => box.running && box.parked !== true && !box.slug.startsWith("runner-")).map((box) => box.slug);
+export const autoBackupArgs = (slug) => ["sandbox", "backup", slug, "--auto", "--json"];
+const BackupAnswerSchema = z.object({
+    slug: z.string(),
+    result: z.string(),
+    snapshot: z.string().optional(),
+    reason: z.string().optional(),
+});
+const backupAnswer = (line) => {
+    try {
+        const parsed = BackupAnswerSchema.safeParse(JSON.parse(line));
+        return parsed.success ? parsed.data : undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
+export const backupNews = (slug, output) => {
+    const answer = output
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith("{"))
+        .map(backupAnswer)
+        .findLast((found) => found !== undefined);
+    if (answer === undefined) {
+        return `auto-backup ${slug}: ${lastLine(output) ?? "done"}`;
+    }
+    const detail = answer.result === "done" ? answer.snapshot : answer.reason;
+    return `auto-backup ${slug}: ${answer.result}${detail === undefined ? "" : ` (${detail})`}`;
+};
+export const newBackupState = () => newRoundState();
+export const runBackupRound = async (state, boxes, backup, log, busy = icInFlight) => await runRound(state, backupTargets(boxes), { name: "auto-backup", run: backup, said: (slug, run) => backupNews(slug, run.output) }, log, busy);
+export const autoTidyArgs = () => ["sandbox", "tidy", "--json"];
+const TidyAnswerSchema = z.object({
+    images: z.array(z.string()).default([]),
+    records: z.array(z.string()).default([]),
+    purgedFromTrash: z.array(z.string()).default([]),
+    orphanVolumes: z.array(z.string()).default([]),
+});
+const tidyAnswer = (line) => {
+    try {
+        const parsed = TidyAnswerSchema.safeParse(JSON.parse(line));
+        return parsed.success ? parsed.data : undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
+export const tidyNews = (run) => {
+    const answer = run.output
+        .split(/\r?\n/)
+        .map((line) => line.trim())
+        .filter((line) => line.startsWith("{"))
+        .map(tidyAnswer)
+        .findLast((found) => found !== undefined);
+    if (run.code !== 0 || answer === undefined) {
+        return `auto-tidy: ${run.code === 0 ? "" : "failed — "}${lastLine(run.output) ?? "no output"}`;
+    }
+    const removed = [
+        answer.images.length === 0 ? undefined : plural(answer.images.length, "image"),
+        answer.records.length === 0 ? undefined : `${plural(answer.records.length, "record")} of sandboxes that are gone`,
+        answer.purgedFromTrash.length === 0 ? undefined : `${plural(answer.purgedFromTrash.length, "sandbox", "sandboxes")} from the trash`,
+    ].filter((part) => part !== undefined);
+    const kept = answer.orphanVolumes.length === 0 ? "" : `; volumes no sandbox claims, kept for you to decide: ${answer.orphanVolumes.join(", ")}`;
+    return `auto-tidy: ${removed.length === 0 ? "nothing to clear" : `cleared ${removed.join(", ")}`}${kept}`;
+};
+export const runTidy = async (tidy, log) => {
+    try {
+        log(tidyNews(await tidy()));
+    }
+    catch (error) {
+        log(`auto-tidy: failed — ${errorMessage(error)}`);
+    }
+};
+export const startAutoBackup = (log) => {
+    const state = newBackupState();
+    return startRounds("auto-backup", log, FIRST_TICK_MS + Math.floor(Math.random() * FIRST_SPREAD_MS), () => TICK_MS + Math.floor(Math.random() * JITTER_MS), async () => {
+        const config = await readMachineConfig();
+        if (config.sandboxBackups !== false) {
+            await runBackupRound(state, await fleet(), async (slug) => await runIc(autoBackupArgs(slug), () => undefined), log);
+        }
+        if (config.sandboxTidy !== false) {
+            await runTidy(async () => await runIc(autoTidyArgs(), () => undefined), log);
+        }
+    });
+};
+//# sourceMappingURL=auto-backup.js.map

package/dist/device/sandbox-rounds/auto-backup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auto-backup.js","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/auto-backup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAG/C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAc,QAAQ,EAAE,aAAa,EAAgC,QAAQ,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC1H,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAQjE,MAAM,aAAa,GAAG,EAAE,GAAG,MAAM,CAAC;AAClC,MAAM,eAAe,GAAG,EAAE,GAAG,MAAM,CAAC;AAEpC,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;AACjC,MAAM,SAAS,GAAG,EAAE,GAAG,MAAM,CAAC;AAI9B,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,KAA+B,EAAY,EAAE,CACvE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAGxH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAY,EAAE,CAAC,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAI1G,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;IAClB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,CAAC,IAAY,EAAkD,EAAE;IAClF,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC9D,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QAEL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,MAAc,EAAU,EAAE;IAC/D,MAAM,MAAM,GAAG,MAAM;SAChB,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;SACtC,GAAG,CAAC,YAAY,CAAC;SACjB,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC9C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvB,OAAO,eAAe,IAAI,KAAK,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;IAChE,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;IAC1E,OAAO,eAAe,IAAI,KAAK,MAAM,CAAC,MAAM,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,GAAG,EAAE,CAAC;AAChG,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,cAAc,GAAG,GAAe,EAAE,CAAC,aAAa,EAAE,CAAC;AAGhE,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,EAC/B,KAAiB,EACjB,KAA+B,EAC/B,MAAwC,EACxC,GAAQ,EACR,IAAI,GAAwB,UAAU,EACzB,EAAE,CACf,MAAM,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;AAMpJ,MAAM,CAAC,MAAM,YAAY,GAAG,GAAa,EAAE,CAAC,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AAG1E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IACxC,eAAe,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAChD,aAAa,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;CACjD,CAAC,CAAC;AAEH,MAAM,UAAU,GAAG,CAAC,IAAY,EAAgD,EAAE;IAC9E,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5D,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QAEL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,GAAU,EAAU,EAAE;IAC3C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM;SACpB,KAAK,CAAC,OAAO,CAAC;SACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;SACtC,GAAG,CAAC,UAAU,CAAC;SACf,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC9C,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzC,OAAO,cAAc,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;IACnG,CAAC;IACD,MAAM,OAAO,GAAG;QACZ,MAAM,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;QAC9E,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,6BAA6B;QACjH,MAAM,CAAC,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,MAAM,EAAE,SAAS,EAAE,WAAW,CAAC,iBAAiB;KACtI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,wDAAwD,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAChJ,OAAO,cAAc,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,WAAW,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC;AAC9G,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EAAE,IAA0B,EAAE,GAAQ,EAAiB,EAAE;IACjF,IAAI,CAAC;QACD,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,GAAG,CAAC,uBAAuB,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;AACL,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,GAAQ,EAAU,EAAE;IAChD,MAAM,KAAK,GAAG,cAAc,EAAE,CAAC;IAC/B,OAAO,WAAW,CACd,aAAa,EACb,GAAG,EACH,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,eAAe,CAAC,EAC3D,GAAG,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,EACrD,KAAK,IAAI,EAAE;QACP,MAAM,MAAM,GAAG,MAAM,iBAAiB,EAAE,CAAC;QACzC,IAAI,MAAM,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;YAClC,MAAM,cAAc,CAAC,KAAK,EAAE,MAAM,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,CAAC;QACxH,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,KAAK,KAAK,EAAE,CAAC;YAC/B,MAAM,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,YAAY,EAAE,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,CAAC;QACjF,CAAC;IACL,CAAC,CACJ,CAAC;AACN,CAAC,CAAC"}

package/dist/device/sandbox-rounds/auto-prepare.d.ts

@@ -0,0 +1,11 @@
+import type { Log } from "@intentic/local-agent";
+import type { DeviceSandbox } from "@intentic/sandbox-contract";
+import { type IcRun, type RoundState, type Rounds, ticksToSkip } from "./ic-rounds.js";
+export { ticksToSkip };
+export declare const prepareTargets: (boxes: readonly DeviceSandbox[]) => string[];
+export declare const autoPrepareArgs: (slug: string) => string[];
+export type AutoPrepareState = RoundState;
+export declare const newState: () => AutoPrepareState;
+export declare const runTick: (state: AutoPrepareState, boxes: readonly DeviceSandbox[], prepare: (slug: string) => Promise<IcRun>, log: Log, busy?: ReadonlySet<string>) => Promise<void>;
+export declare const startAutoPrepare: (log: Log) => Rounds;
+//# sourceMappingURL=auto-prepare.d.ts.map

package/dist/device/sandbox-rounds/auto-prepare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auto-prepare.d.ts","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/auto-prepare.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEhE,OAAO,EAAE,KAAK,KAAK,EAA2B,KAAK,UAAU,EAAE,KAAK,MAAM,EAAyB,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAcvI,OAAO,EAAE,WAAW,EAAE,CAAC;AAIvB,eAAO,MAAM,cAAc,UAAW,SAAS,aAAa,EAAE,KAAG,MAAM,EACyB,CAAC;AAGjG,eAAO,MAAM,eAAe,SAAU,MAAM,KAAG,MAAM,EAA4C,CAAC;AAElG,MAAM,MAAM,gBAAgB,GAAG,UAAU,CAAC;AAE1C,eAAO,MAAM,QAAQ,QAAO,gBAAmC,CAAC;AAGhE,eAAO,MAAM,OAAO,UACT,gBAAgB,SAChB,SAAS,aAAa,EAAE,WACtB,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,OACpC,GAAG,SACF,WAAW,CAAC,MAAM,CAAC,KAC1B,OAAO,CAAC,IAAI,CAQV,CAAC;AAIN,eAAO,MAAM,gBAAgB,QAAS,GAAG,KAAG,MAc3C,CAAC"}

package/dist/device/sandbox-rounds/auto-prepare.js

@@ -0,0 +1,20 @@
+import { readMachineConfig } from "../../environments/machine.js";
+import { lastLine, newRoundState, runRound, startRounds, ticksToSkip } from "./ic-rounds.js";
+import { fleet, icInFlight, runIc } from "../tools/sandboxes.js";
+const FIRST_TICK_MS = 5 * 60_000;
+const TICK_MS = 6 * 60 * 60_000;
+const JITTER_MS = 30 * 60_000;
+export { ticksToSkip };
+export const prepareTargets = (boxes) => boxes.filter((box) => box.running && !box.slug.startsWith("runner-")).map((box) => box.slug);
+export const autoPrepareArgs = (slug) => ["sandbox", "prepare", slug, "--auto"];
+export const newState = () => newRoundState();
+export const runTick = async (state, boxes, prepare, log, busy = icInFlight) => await runRound(state, prepareTargets(boxes), { name: "auto-prepare", run: prepare, said: (slug, run) => `auto-prepare ${slug}: ${lastLine(run.output) ?? "done"}` }, log, busy);
+export const startAutoPrepare = (log) => {
+    const state = newState();
+    return startRounds("auto-prepare", log, FIRST_TICK_MS + Math.floor(Math.random() * JITTER_MS), () => TICK_MS + Math.floor(Math.random() * JITTER_MS), async () => {
+        if ((await readMachineConfig()).sandboxUpdates !== false) {
+            await runTick(state, await fleet(), async (slug) => await runIc(autoPrepareArgs(slug), () => undefined), log);
+        }
+    });
+};
+//# sourceMappingURL=auto-prepare.js.map

package/dist/device/sandbox-rounds/auto-prepare.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auto-prepare.js","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/auto-prepare.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAc,QAAQ,EAAE,aAAa,EAAgC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACvI,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAOjE,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC;AAEjC,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;AAEhC,MAAM,SAAS,GAAG,EAAE,GAAG,MAAM,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,CAAC;AAIvB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,KAA+B,EAAY,EAAE,CACxE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;AAGjG,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAY,EAAE,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;AAIlG,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAqB,EAAE,CAAC,aAAa,EAAE,CAAC;AAGhE,MAAM,CAAC,MAAM,OAAO,GAAG,KAAK,EACxB,KAAuB,EACvB,KAA+B,EAC/B,OAAyC,EACzC,GAAQ,EACR,IAAI,GAAwB,UAAU,EACzB,EAAE,CACf,MAAM,QAAQ,CACV,KAAK,EACL,cAAc,CAAC,KAAK,CAAC,EAErB,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,gBAAgB,IAAI,KAAK,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,MAAM,EAAE,EAAE,EACtH,GAAG,EACH,IAAI,CACP,CAAC;AAIN,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,GAAQ,EAAU,EAAE;IACjD,MAAM,KAAK,GAAG,QAAQ,EAAE,CAAC;IACzB,OAAO,WAAW,CACd,cAAc,EACd,GAAG,EACH,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,EACrD,GAAG,EAAE,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,EACrD,KAAK,IAAI,EAAE;QAEP,IAAI,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC,cAAc,KAAK,KAAK,EAAE,CAAC;YACvD,MAAM,OAAO,CAAC,KAAK,EAAE,MAAM,KAAK,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,CAAC;QAClH,CAAC;IACL,CAAC,CACJ,CAAC;AACN,CAAC,CAAC"}

package/dist/device/sandbox-rounds/ic-rounds.d.ts

@@ -0,0 +1,23 @@
+import type { Log } from "@intentic/local-agent";
+export declare const ticksToSkip: (failures: number) => number;
+export interface RoundState {
+    readonly failures: Map<string, number>;
+    readonly waits: Map<string, number>;
+}
+export declare const newRoundState: () => RoundState;
+export interface IcRun {
+    readonly code: number;
+    readonly output: string;
+}
+export interface SlugJob {
+    readonly name: string;
+    readonly run: (slug: string) => Promise<IcRun>;
+    readonly said: (slug: string, run: IcRun) => string | undefined;
+}
+export declare const lastLine: (output: string) => string | undefined;
+export declare const runRound: (state: RoundState, targets: readonly string[], job: SlugJob, log: Log, busy?: ReadonlySet<string>) => Promise<void>;
+export interface Rounds {
+    readonly stop: () => void;
+}
+export declare const startRounds: (name: string, log: Log, first: number, next: () => number, round: () => Promise<void>) => Rounds;
+//# sourceMappingURL=ic-rounds.d.ts.map

package/dist/device/sandbox-rounds/ic-rounds.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ic-rounds.d.ts","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/ic-rounds.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AAYjD,eAAO,MAAM,WAAW,aAAc,MAAM,KAAG,MAA6E,CAAC;AAI7H,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvC,QAAQ,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACvC;AAED,eAAO,MAAM,aAAa,QAAO,UAAyD,CAAC;AAE3F,MAAM,WAAW,KAAK;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAID,MAAM,WAAW,OAAO;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC,CAAC;IAC/C,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,KAAK,MAAM,GAAG,SAAS,CAAC;CACnE;AAGD,eAAO,MAAM,QAAQ,WAAY,MAAM,KAAG,MAAM,GAAG,SAAyE,CAAC;AAgC7H,eAAO,MAAM,QAAQ,UAAiB,UAAU,WAAW,SAAS,MAAM,EAAE,OAAO,OAAO,OAAO,GAAG,SAAQ,WAAW,CAAC,MAAM,CAAC,KAAgB,OAAO,CAAC,IAAI,CAoB1J,CAAC;AAEF,MAAM,WAAW,MAAM;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC;CAC7B;AAKD,eAAO,MAAM,WAAW,SAAU,MAAM,OAAO,GAAG,SAAS,MAAM,QAAQ,MAAM,MAAM,SAAS,MAAM,OAAO,CAAC,IAAI,CAAC,KAAG,MAuBnH,CAAC"}

package/dist/device/sandbox-rounds/ic-rounds.js

@@ -0,0 +1,78 @@
+import { errorMessage } from "@intentic/base/errors";
+import { plural } from "@intentic/base/format";
+import { icInFlight } from "../tools/sandboxes.js";
+const MAX_SKIP_TICKS = 8;
+export const ticksToSkip = (failures) => (failures <= 0 ? 0 : Math.min(2 ** (failures - 1), MAX_SKIP_TICKS));
+export const newRoundState = () => ({ failures: new Map(), waits: new Map() });
+export const lastLine = (output) => output.split(/\r?\n/).findLast((line) => line.trim() !== "");
+const runOne = async (state, slug, job, log) => {
+    icInFlight.add(slug);
+    let run;
+    try {
+        run = await job.run(slug);
+    }
+    catch (error) {
+        run = { code: 1, output: errorMessage(error) };
+    }
+    finally {
+        icInFlight.delete(slug);
+    }
+    if (run.code === 0) {
+        state.failures.delete(slug);
+        state.waits.delete(slug);
+        const said = job.said(slug, run);
+        if (said !== undefined) {
+            log(said);
+        }
+        return;
+    }
+    const failures = (state.failures.get(slug) ?? 0) + 1;
+    state.failures.set(slug, failures);
+    state.waits.set(slug, ticksToSkip(failures));
+    log(`${job.name} ${slug}: failed (attempt ${failures}, retrying after ${plural(ticksToSkip(failures), "tick")}) — ${lastLine(run.output) ?? "no output"}`);
+};
+export const runRound = async (state, targets, job, log, busy = icInFlight) => {
+    for (const slug of [...state.failures.keys(), ...state.waits.keys()]) {
+        if (!targets.includes(slug)) {
+            state.failures.delete(slug);
+            state.waits.delete(slug);
+        }
+    }
+    for (const slug of targets) {
+        if (busy.has(slug)) {
+            continue;
+        }
+        const wait = state.waits.get(slug) ?? 0;
+        if (wait > 0) {
+            state.waits.set(slug, wait - 1);
+            continue;
+        }
+        await runOne(state, slug, job, log);
+    }
+};
+export const startRounds = (name, log, first, next, round) => {
+    let timer;
+    let stopped = false;
+    const schedule = (delay) => {
+        timer = setTimeout(() => void tick(), delay);
+    };
+    const tick = async () => {
+        try {
+            await round();
+        }
+        catch (error) {
+            log(`${name}: skipped this round — ${errorMessage(error)}`);
+        }
+        if (!stopped) {
+            schedule(next());
+        }
+    };
+    schedule(first);
+    return {
+        stop: () => {
+            stopped = true;
+            clearTimeout(timer);
+        },
+    };
+};
+//# sourceMappingURL=ic-rounds.js.map

package/dist/device/sandbox-rounds/ic-rounds.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ic-rounds.js","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/ic-rounds.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAQnD,MAAM,cAAc,GAAG,CAAC,CAAC;AAGzB,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,QAAgB,EAAU,EAAE,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC;AAS7H,MAAM,CAAC,MAAM,aAAa,GAAG,GAAe,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;AAgB3F,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAsB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;AAE7H,MAAM,MAAM,GAAG,KAAK,EAAE,KAAiB,EAAE,IAAY,EAAE,GAAY,EAAE,GAAQ,EAAiB,EAAE;IAC5F,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACrB,IAAI,GAAU,CAAC;IACf,IAAI,CAAC;QACD,GAAG,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAEb,GAAG,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;IACnD,CAAC;YAAS,CAAC;QACP,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IACD,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5B,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACjC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,GAAG,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;QACD,OAAO;IACX,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IACrD,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACnC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC7C,GAAG,CACC,GAAG,GAAG,CAAC,IAAI,IAAI,IAAI,qBAAqB,QAAQ,oBAAoB,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,OAAO,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,EAAE,CACxJ,CAAC;AACN,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,QAAQ,GAAG,KAAK,EAAE,KAAiB,EAAE,OAA0B,EAAE,GAAY,EAAE,GAAQ,EAAE,IAAI,GAAwB,UAAU,EAAiB,EAAE;IAE3J,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QACnE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1B,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC5B,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACL,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QACD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACX,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YAChC,SAAS;QACb,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;AACL,CAAC,CAAC;AASF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAE,GAAQ,EAAE,KAAa,EAAE,IAAkB,EAAE,KAA0B,EAAU,EAAE;IACzH,IAAI,KAAiC,CAAC;IACtC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACrC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,EAAE,EAAE,KAAK,CAAC,CAAC;IACjD,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,KAAK,IAAmB,EAAE;QACnC,IAAI,CAAC;YACD,MAAM,KAAK,EAAE,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,GAAG,CAAC,GAAG,IAAI,0BAA0B,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,CAAC,OAAO,EAAE,CAAC;YACX,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QACrB,CAAC;IACL,CAAC,CAAC;IACF,QAAQ,CAAC,KAAK,CAAC,CAAC;IAChB,OAAO;QACH,IAAI,EAAE,GAAS,EAAE;YACb,OAAO,GAAG,IAAI,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;QACxB,CAAC;KACJ,CAAC;AACN,CAAC,CAAC"}

package/dist/device/sandbox-rounds/probation-watch.d.ts

@@ -0,0 +1,32 @@
+import type { Log } from "@intentic/local-agent";
+import { z } from "zod";
+import { type IcRun, type RoundState, type Rounds } from "./ic-rounds.js";
+import { type SwapRecord } from "./swap-records.js";
+export declare const watchArgs: (slug?: string) => string[];
+export declare const watchTargets: (records: readonly SwapRecord[], busy: ReadonlySet<string>) => string[];
+declare const WatchAnswerSchema: z.ZodObject<{
+    slug: z.ZodString;
+    action: z.ZodString;
+    reason: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export type WatchAnswer = z.infer<typeof WatchAnswerSchema>;
+export declare const watchAnswers: (output: string) => WatchAnswer[];
+export declare const watchNews: (answers: readonly WatchAnswer[], last: Map<string, string>) => string[];
+export declare const sweepDue: (dueAt: number, now: number, busy: ReadonlySet<string>) => boolean;
+interface WatchState {
+    readonly slugs: RoundState;
+    readonly last: Map<string, string>;
+    sweepDueAt: number;
+    sweepFailures: number;
+}
+export declare const runWatchRound: (state: WatchState, records: readonly SwapRecord[], watch: (slug: string | undefined) => Promise<IcRun>, log: Log, { now, sweeps, busy }: {
+    readonly now: number;
+    readonly sweeps: boolean;
+    readonly busy?: ReadonlySet<string>;
+}) => Promise<void>;
+export declare const newWatchState: (now: number) => WatchState;
+export declare const startProbationWatch: (log: Log, { sweeps }: {
+    readonly sweeps: boolean;
+}) => Rounds;
+export {};
+//# sourceMappingURL=probation-watch.d.ts.map

package/dist/device/sandbox-rounds/probation-watch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"probation-watch.d.ts","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/probation-watch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,KAAK,KAAK,EAA2B,KAAK,UAAU,EAAE,KAAK,MAAM,EAAsC,MAAM,gBAAgB,CAAC;AACvI,OAAO,EAAmB,KAAK,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAerE,eAAO,MAAM,SAAS,UAAW,MAAM,KAAG,MAAM,EAA2E,CAAC;AAI5H,eAAO,MAAM,YAAY,YAAa,SAAS,UAAU,EAAE,QAAQ,WAAW,CAAC,MAAM,CAAC,KAAG,MAAM,EACkB,CAAC;AAIlH,QAAA,MAAM,iBAAiB;;;;iBAAoF,CAAC;AAC5G,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAY5D,eAAO,MAAM,YAAY,WAAY,MAAM,KAAG,WAAW,EAMR,CAAC;AAQlD,eAAO,MAAM,SAAS,YAAa,SAAS,WAAW,EAAE,QAAQ,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,KAAG,MAAM,EAQvF,CAAC;AAIP,eAAO,MAAM,QAAQ,UAAW,MAAM,OAAO,MAAM,QAAQ,WAAW,CAAC,MAAM,CAAC,KAAG,OAA0C,CAAC;AAE5H,UAAU,UAAU;IAChB,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,UAAU,EAAE,MAAM,CAAC;IAGnB,aAAa,EAAE,MAAM,CAAC;CACzB;AAGD,eAAO,MAAM,aAAa,UACf,UAAU,WACR,SAAS,UAAU,EAAE,SACvB,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,KAAK,OAAO,CAAC,KAAK,CAAC,OAC9C,GAAG,yBAC4B;IAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAA;CAAE,KAC5H,OAAO,CAAC,IAAI,CAoBd,CAAC;AAEF,eAAO,MAAM,aAAa,QAAS,MAAM,KAAG,UAA+G,CAAC;AAG5J,eAAO,MAAM,mBAAmB,QAAS,GAAG,cAAc;IAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAA;CAAE,KAAG,MAgBxF,CAAC"}

package/dist/device/sandbox-rounds/probation-watch.js

@@ -0,0 +1,60 @@
+import { z } from "zod";
+import { lastLine, newRoundState, runRound, startRounds, ticksToSkip } from "./ic-rounds.js";
+import { readSwapRecords } from "./swap-records.js";
+import { icInFlight, runIc } from "../tools/sandboxes.js";
+const WATCH_EVERY_MS = 60_000;
+const SWEEP_FIRST_MS = 30_000;
+const SWEEP_EVERY_MS = 10 * 60_000;
+export const watchArgs = (slug) => ["sandbox", "watch", ...(slug === undefined ? [] : [slug]), "--json"];
+export const watchTargets = (records, busy) => records.filter((record) => record.phase !== undefined && !busy.has(record.slug)).map((record) => record.slug);
+const WatchAnswerSchema = z.object({ slug: z.string(), action: z.string(), reason: z.string().optional() });
+const watchAnswer = (line) => {
+    try {
+        const parsed = WatchAnswerSchema.safeParse(JSON.parse(line));
+        return parsed.success ? parsed.data : undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
+export const watchAnswers = (output) => output
+    .split(/\r?\n/)
+    .map((line) => line.trim())
+    .filter((line) => line.startsWith("{"))
+    .map(watchAnswer)
+    .filter((answer) => answer !== undefined);
+const STANDING = new Set(["watching", "busy"]);
+export const watchNews = (answers, last) => answers.flatMap((answer) => {
+    const before = last.get(answer.slug);
+    last.set(answer.slug, answer.action);
+    if (answer.action === "none" || (STANDING.has(answer.action) && before === answer.action)) {
+        return [];
+    }
+    return [`probation watch ${answer.slug}: ${answer.action}${answer.reason === undefined ? "" : ` — ${answer.reason}`}`];
+});
+export const sweepDue = (dueAt, now, busy) => now >= dueAt && busy.size === 0;
+export const runWatchRound = async (state, records, watch, log, { now, sweeps, busy = icInFlight }) => {
+    if (sweeps && sweepDue(state.sweepDueAt, now, busy)) {
+        const run = await watch(undefined);
+        state.sweepFailures = run.code === 0 ? 0 : state.sweepFailures + 1;
+        state.sweepDueAt = now + SWEEP_EVERY_MS * (1 + ticksToSkip(state.sweepFailures));
+        if (run.code !== 0) {
+            log(`probation watch: the sweep over every sandbox failed (attempt ${state.sweepFailures}, next in ${(SWEEP_EVERY_MS * (1 + ticksToSkip(state.sweepFailures))) / 60_000} minutes) — ${lastLine(run.output) ?? "no output"}`);
+        }
+        for (const line of watchNews(watchAnswers(run.output), state.last)) {
+            log(line);
+        }
+        return;
+    }
+    const said = (_slug, run) => {
+        const lines = watchNews(watchAnswers(run.output), state.last);
+        return lines.length === 0 ? undefined : lines.join("\n");
+    };
+    await runRound(state.slugs, watchTargets(records, busy), { name: "probation watch", run: watch, said }, log, busy);
+};
+export const newWatchState = (now) => ({ slugs: newRoundState(), last: new Map(), sweepDueAt: now + SWEEP_FIRST_MS, sweepFailures: 0 });
+export const startProbationWatch = (log, { sweeps }) => {
+    const state = newWatchState(Date.now());
+    return startRounds("probation watch", log, SWEEP_FIRST_MS, () => WATCH_EVERY_MS, async () => await runWatchRound(state, await readSwapRecords(), async (slug) => await runIc(watchArgs(slug), () => undefined), log, { now: Date.now(), sweeps }));
+};
+//# sourceMappingURL=probation-watch.js.map

package/dist/device/sandbox-rounds/probation-watch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"probation-watch.js","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/probation-watch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAc,QAAQ,EAAE,aAAa,EAAgC,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACvI,OAAO,EAAE,eAAe,EAAmB,MAAM,mBAAmB,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAS1D,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B,MAAM,cAAc,GAAG,MAAM,CAAC;AAC9B,MAAM,cAAc,GAAG,EAAE,GAAG,MAAM,CAAC;AAEnC,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAa,EAAY,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;AAI5H,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,OAA8B,EAAE,IAAyB,EAAY,EAAE,CAChG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAIlH,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;AAG5G,MAAM,WAAW,GAAG,CAAC,IAAY,EAA2B,EAAE;IAC1D,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC7D,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QAEL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,MAAc,EAAiB,EAAE,CAC1D,MAAM;KACD,KAAK,CAAC,OAAO,CAAC;KACd,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;KAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;KACtC,GAAG,CAAC,WAAW,CAAC;KAChB,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAIlD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;AAI/C,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,OAA+B,EAAE,IAAyB,EAAY,EAAE,CAC9F,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;IACvB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QACxF,OAAO,EAAE,CAAC;IACd,CAAC;IACD,OAAO,CAAC,mBAAmB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAC3H,CAAC,CAAC,CAAC;AAIP,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,GAAW,EAAE,IAAyB,EAAW,EAAE,CAAC,GAAG,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC;AAY5H,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAC9B,KAAiB,EACjB,OAA8B,EAC9B,KAAmD,EACnD,GAAQ,EACR,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,GAAG,UAAU,EAA2F,EAC9G,EAAE;IACf,IAAI,MAAM,IAAI,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;QACnC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC;QACnE,KAAK,CAAC,UAAU,GAAG,GAAG,GAAG,cAAc,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC;QACjF,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YACjB,GAAG,CACC,iEAAiE,KAAK,CAAC,aAAa,aAAa,CAAC,cAAc,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,MAAM,eAAe,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,EAAE,CAC1N,CAAC;QACN,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,GAAG,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;QACD,OAAO;IACX,CAAC;IACD,MAAM,IAAI,GAAG,CAAC,KAAa,EAAE,GAAU,EAAsB,EAAE;QAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9D,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,CAAC,CAAC;IACF,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;AACvH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAW,EAAc,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,UAAU,EAAE,GAAG,GAAG,cAAc,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;AAG5J,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,GAAQ,EAAE,EAAE,MAAM,EAAgC,EAAU,EAAE;IAC9F,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACxC,OAAO,WAAW,CACd,iBAAiB,EACjB,GAAG,EACH,cAAc,EACd,GAAG,EAAE,CAAC,cAAc,EACpB,KAAK,IAAI,EAAE,CACP,MAAM,aAAa,CACf,KAAK,EACL,MAAM,eAAe,EAAE,EACvB,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,MAAM,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,EAC7D,GAAG,EACH,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,CAC9B,CACR,CAAC;AACN,CAAC,CAAC"}

package/dist/device/sandbox-rounds/swap-records.d.ts

@@ -0,0 +1,14 @@
+export declare const icHome: (env?: NodeJS.ProcessEnv) => string;
+export interface SwapRecord {
+    readonly slug: string;
+    readonly phase?: string | undefined;
+    readonly at?: number | undefined;
+    readonly probationUntil?: number | undefined;
+}
+export declare const slugOfRecord: (name: string) => string | undefined;
+export declare const parseSwapRecord: (slug: string, text: string) => SwapRecord;
+export declare const readSwapRecords: (home?: string) => Promise<SwapRecord[]>;
+export declare const CUTOVER_HOLDS_MS: number;
+export declare const cutoverUnderway: (record: SwapRecord, now: number) => boolean;
+export declare const swapsUnderway: (records: readonly SwapRecord[], now: number) => string[];
+//# sourceMappingURL=swap-records.d.ts.map

package/dist/device/sandbox-rounds/swap-records.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"swap-records.d.ts","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/swap-records.ts"],"names":[],"mappings":"AAUA,eAAO,MAAM,MAAM,SAAS,MAAM,CAAC,UAAU,KAAiB,MAG7D,CAAC;AAIF,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAChD;AAMD,eAAO,MAAM,YAAY,SAAU,MAAM,KAAG,MAAM,GAAG,SAGlC,CAAC;AASpB,eAAO,MAAM,eAAe,SAAU,MAAM,QAAQ,MAAM,KAAG,UAe5D,CAAC;AAIF,eAAO,MAAM,eAAe,UAAgB,MAAM,KAAc,OAAO,CAAC,UAAU,EAAE,CAYnF,CAAC;AAIF,eAAO,MAAM,gBAAgB,QAAc,CAAC;AAI5C,eAAO,MAAM,eAAe,WAAY,UAAU,OAAO,MAAM,KAAG,OACuC,CAAC;AAG1G,eAAO,MAAM,aAAa,YAAa,SAAS,UAAU,EAAE,OAAO,MAAM,KAAG,MAAM,EACO,CAAC"}

package/dist/device/sandbox-rounds/swap-records.js

@@ -0,0 +1,45 @@
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { homeDir } from "@intentic/local-agent";
+export const icHome = (env = process.env) => {
+    const set = env["INTENTIC_HOME"];
+    return set === undefined || set === "" ? join(homeDir(), ".intentic") : set;
+};
+const RECORD_PREFIX = "sandbox-";
+const RECORD_SUFFIX = ".channel";
+export const slugOfRecord = (name) => name.startsWith(RECORD_PREFIX) && name.endsWith(RECORD_SUFFIX) && name.length > RECORD_PREFIX.length + RECORD_SUFFIX.length
+    ? name.slice(RECORD_PREFIX.length, -RECORD_SUFFIX.length)
+    : undefined;
+const millis = (value) => {
+    const parsed = value === undefined ? Number.NaN : Number(value);
+    return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined;
+};
+export const parseSwapRecord = (slug, text) => {
+    const values = new Map();
+    for (const line of text.split(/\r?\n/)) {
+        const at = line.indexOf("=");
+        if (at > 0) {
+            values.set(line.slice(0, at), line.slice(at + 1).trim());
+        }
+    }
+    const phase = values.get("swap_phase");
+    return {
+        slug,
+        phase: phase === "" ? undefined : phase,
+        at: millis(values.get("swap_at")),
+        probationUntil: millis(values.get("probation_until")),
+    };
+};
+export const readSwapRecords = async (home = icHome()) => {
+    const names = await readdir(home).catch(() => []);
+    const records = await Promise.all(names.map(async (name) => {
+        const slug = slugOfRecord(name);
+        const text = slug === undefined ? undefined : await readFile(join(home, name), "utf8").catch(() => undefined);
+        return slug === undefined || text === undefined ? undefined : parseSwapRecord(slug, text);
+    }));
+    return records.filter((record) => record?.phase !== undefined);
+};
+export const CUTOVER_HOLDS_MS = 30 * 60_000;
+export const cutoverUnderway = (record, now) => record.phase === "cutover" && record.at !== undefined && Math.abs(now - record.at) < CUTOVER_HOLDS_MS;
+export const swapsUnderway = (records, now) => records.filter((record) => cutoverUnderway(record, now)).map((record) => record.slug);
+//# sourceMappingURL=swap-records.js.map

package/dist/device/sandbox-rounds/swap-records.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"swap-records.js","sourceRoot":"","sources":["../../../src/device/sandbox-rounds/swap-records.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAC;AAQhD,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,GAAG,GAAsB,OAAO,CAAC,GAAG,EAAU,EAAE;IACnE,MAAM,GAAG,GAAG,GAAG,CAAC,eAAe,CAAC,CAAC;IACjC,OAAO,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAChF,CAAC,CAAC;AAWF,MAAM,aAAa,GAAG,UAAU,CAAC;AACjC,MAAM,aAAa,GAAG,UAAU,CAAC;AAGjC,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAY,EAAsB,EAAE,CAC7D,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;IACvH,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC;IACzD,CAAC,CAAC,SAAS,CAAC;AAEpB,MAAM,MAAM,GAAG,CAAC,KAAyB,EAAsB,EAAE;IAC7D,MAAM,MAAM,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChE,OAAO,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3E,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,IAAY,EAAE,IAAY,EAAc,EAAE;IACtE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;YACT,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IACvC,OAAO;QACH,IAAI;QACJ,KAAK,EAAE,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK;QACvC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACjC,cAAc,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;KACxD,CAAC;AACN,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAAE,IAAI,GAAW,MAAM,EAAE,EAAyB,EAAE;IAEpF,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAa,EAAE,CAAC,EAAE,CAAC,CAAC;IAC5D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAC7B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACrB,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;QAEhC,MAAM,IAAI,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9G,OAAO,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC9F,CAAC,CAAC,CACL,CAAC;IACF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAwB,EAAE,CAAC,MAAM,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC;AACzF,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,GAAG,MAAM,CAAC;AAI5C,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAkB,EAAE,GAAW,EAAW,EAAE,CACxE,MAAM,CAAC,KAAK,KAAK,SAAS,IAAI,MAAM,CAAC,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC;AAG1G,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,OAA8B,EAAE,GAAW,EAAY,EAAE,CACnF,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC"}

package/dist/device/tools/agent.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../src/device/tools/agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAY9E,eAAO,MAAM,UAAU,EAAE,MAAM,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAI/D,CAAC;AAsCF,eAAO,MAAM,UAAU,OAAc,aAAa,UAAU,YAAY,UAAU,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,KAAG,OAAO,CAAC,MAAM,CAuBxH,CAAC"}
+{"version":3,"file":"agent.d.ts","sourceRoot":"","sources":["../../../src/device/tools/agent.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAc9E,eAAO,MAAM,UAAU,EAAE,MAAM,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAI/D,CAAC;AAmDF,eAAO,MAAM,UAAU,OAAc,aAAa,UAAU,YAAY,UAAU,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,KAAG,OAAO,CAAC,MAAM,CA0BxH,CAAC"}

package/dist/device/tools/agent.js

@@ -5,6 +5,8 @@ import { agentLogPath } from "../../config.js";
 import { installedBuild } from "../../installed.js";
 import { machineLauncher } from "../../supervision.js";
 import { assertScope } from "../policy.js";
+import { CUTOVER_HOLDS_MS } from "../sandbox-rounds/swap-records.js";
+import { icSwapsInFlight } from "./sandboxes.js";
 export const AGENT_VERB = {
     upgrade: ["upgrade"],
     restart: ["run"],
@@ -31,8 +33,19 @@ const settled = {
     restart: `The agent was restarted on this device.`,
     "forget-unreachable": `The drop ran on this device. What it removed is in the lines above; this device's link count catches up within a few seconds.`,
 };
+const RESTARTS = new Set(["upgrade", "restart"]);
+const waitOutOwnSwaps = async (onLine) => {
+    if (icSwapsInFlight.size === 0) {
+        return;
+    }
+    onLine(`Waiting for ${[...icSwapsInFlight].join(", ")} to finish first: restarting this agent in the middle of a swap could leave that sandbox down.`);
+    await pollUntil(() => icSwapsInFlight.size === 0, { intervalMs: 1_000, timeoutMs: CUTOVER_HOLDS_MS });
+};
 export const runAgentOp = async (op, scopes, onLine) => {
     assertScope(scopes, "shell");
+    if (RESTARTS.has(op)) {
+        await waitOutOwnSwaps(onLine);
+    }
     onLine(started(op, installedBuild()));
     let at = (await readFrom(agentLogPath, 0)).at;
     const argv = AGENT_VERB[op];

package/dist/device/tools/agent.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../src/device/tools/agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAQ3C,MAAM,CAAC,MAAM,UAAU,GAA6C;IAChE,OAAO,EAAE,CAAC,SAAS,CAAC;IACpB,OAAO,EAAE,CAAC,KAAK,CAAC;IAChB,oBAAoB,EAAE,CAAC,QAAQ,EAAE,oBAAoB,CAAC;CACzD,CAAC;AAIF,MAAM,QAAQ,GAAG,KAAK,CAAC;AACvB,MAAM,OAAO,GAAG,GAAG,CAAC;AAIpB,MAAM,gBAAgB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAGxC,MAAM,QAAQ,GAAG,KAAK,EAAE,IAAY,EAAE,IAAY,EAAyC,EAAE;IACzF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACzD,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AACrD,CAAC,CAAC;AAKF,MAAM,OAAO,GAAG,CAAC,EAAiB,EAAE,SAA6B,EAAU,EAAE;IACzE,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACnB,OAAO,sEAAsE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,oBAAoB,SAAS,GAAG,4EAA4E,CAAC;IAC7N,CAAC;IACD,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACnB,OAAO,2EAA2E,CAAC;IACvF,CAAC;IACD,OAAO,0IAA0I,CAAC;AACtJ,CAAC,CAAC;AAIF,MAAM,OAAO,GAAkC;IAC3C,OAAO,EAAE,6IAA6I;IACtJ,OAAO,EAAE,yCAAyC;IAClD,oBAAoB,EAAE,+HAA+H;CACxJ,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,EAAiB,EAAE,MAAoB,EAAE,MAA8B,EAAmB,EAAE;IACzH,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;IAEtC,IAAI,EAAE,GAAG,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,oEAAoE,YAAY,EAAE,CAAC,CAAC;IAChI,IAAI,MAA0B,CAAC;IAC/B,MAAM,OAAO,GAAG,KAAK,IAAsB,EAAE;QACzC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC5D,EAAE,GAAG,IAAI,CAAC;QACV,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC1C,CAAC,CAAC;IACF,MAAM,SAAS,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC/E,OAAO,OAAO,CAAC,EAAE,CAAC,CAAC;AACvB,CAAC,CAAC"}
+{"version":3,"file":"agent.js","sourceRoot":"","sources":["../../../src/device/tools/agent.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAQjD,MAAM,CAAC,MAAM,UAAU,GAA6C;IAChE,OAAO,EAAE,CAAC,SAAS,CAAC;IACpB,OAAO,EAAE,CAAC,KAAK,CAAC;IAChB,oBAAoB,EAAE,CAAC,QAAQ,EAAE,oBAAoB,CAAC;CACzD,CAAC;AAIF,MAAM,QAAQ,GAAG,KAAK,CAAC;AACvB,MAAM,OAAO,GAAG,GAAG,CAAC;AAIpB,MAAM,gBAAgB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAGxC,MAAM,QAAQ,GAAG,KAAK,EAAE,IAAY,EAAE,IAAY,EAAyC,EAAE;IACzF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;IACzD,OAAO,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AACrD,CAAC,CAAC;AAKF,MAAM,OAAO,GAAG,CAAC,EAAiB,EAAE,SAA6B,EAAU,EAAE;IACzE,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACnB,OAAO,sEAAsE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,oBAAoB,SAAS,GAAG,4EAA4E,CAAC;IAC7N,CAAC;IACD,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;QACnB,OAAO,2EAA2E,CAAC;IACvF,CAAC;IACD,OAAO,0IAA0I,CAAC;AACtJ,CAAC,CAAC;AAIF,MAAM,OAAO,GAAkC;IAC3C,OAAO,EAAE,6IAA6I;IACtJ,OAAO,EAAE,yCAAyC;IAClD,oBAAoB,EAAE,+HAA+H;CACxJ,CAAC;AAGF,MAAM,QAAQ,GAA+B,IAAI,GAAG,CAAgB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;AAI5F,MAAM,eAAe,GAAG,KAAK,EAAE,MAA8B,EAAiB,EAAE;IAC5E,IAAI,eAAe,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO;IACX,CAAC;IACD,MAAM,CAAC,eAAe,CAAC,GAAG,eAAe,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,gGAAgG,CAAC,CAAC;IACvJ,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,IAAI,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,CAAC;AAC1G,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,EAAiB,EAAE,MAAoB,EAAE,MAA8B,EAAmB,EAAE;IACzH,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;QACnB,MAAM,eAAe,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IACD,MAAM,CAAC,OAAO,CAAC,EAAE,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;IAEtC,IAAI,EAAE,GAAG,CAAC,MAAM,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,MAAM,CAAC,WAAW,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,oEAAoE,YAAY,EAAE,CAAC,CAAC;IAChI,IAAI,MAA0B,CAAC;IAC/B,MAAM,OAAO,GAAG,KAAK,IAAsB,EAAE;QACzC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;QAC5D,EAAE,GAAG,IAAI,CAAC;QACV,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,IAAI,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC1C,CAAC,CAAC;IACF,MAAM,SAAS,CAAC,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC/E,OAAO,OAAO,CAAC,EAAE,CAAC,CAAC;AACvB,CAAC,CAAC"}

package/dist/device/tools/command-ledger.d.ts

@@ -0,0 +1,14 @@
+import type { Log } from "@intentic/local-agent";
+import { z } from "zod";
+declare const GroupSchema: z.ZodObject<{
+    pgid: z.ZodNumber;
+    started: z.ZodString;
+}, z.core.$strip>;
+type Group = z.infer<typeof GroupSchema>;
+export declare const startTimeOf: (stat: string) => string | undefined;
+export declare const stillLeftOver: (group: Group, leaderStarted: string | undefined, groupAnswers: boolean) => boolean;
+export declare const recordCommand: (pid: number) => Promise<void>;
+export declare const forgetCommand: (pid: number) => Promise<void>;
+export declare const takeOverCommandLedger: (log: Log) => Promise<void>;
+export {};
+//# sourceMappingURL=command-ledger.d.ts.map

package/dist/device/tools/command-ledger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"command-ledger.d.ts","sourceRoot":"","sources":["../../../src/device/tools/command-ledger.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,QAAA,MAAM,WAAW;;;iBAAsD,CAAC;AAExE,KAAK,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AAIzC,eAAO,MAAM,WAAW,SAAU,MAAM,KAAG,MAAM,GAAG,SAGnD,CAAC;AAIF,eAAO,MAAM,aAAa,UAAW,KAAK,iBAAiB,MAAM,GAAG,SAAS,gBAAgB,OAAO,KAAG,OACvB,CAAC;AAuCjF,eAAO,MAAM,aAAa,QAAe,MAAM,KAAG,OAAO,CAAC,IAAI,CAM7D,CAAC;AAEF,eAAO,MAAM,aAAa,QAAe,MAAM,KAAG,OAAO,CAAC,IAAI,CAI7D,CAAC;AAIF,eAAO,MAAM,qBAAqB,QAAe,GAAG,KAAG,OAAO,CAAC,IAAI,CAiClE,CAAC"}

package/dist/device/tools/command-ledger.js

@@ -0,0 +1,85 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { errorMessage } from "@intentic/base/errors";
+import { writeFileAtomic } from "@intentic/base/fs";
+import { plural } from "@intentic/base/format";
+import { z } from "zod";
+import { baseDir } from "../../config.js";
+const ledgerPath = join(baseDir, "commands.running");
+const GroupSchema = z.object({ pgid: z.number(), started: z.string() });
+const LedgerSchema = z.object({ boot: z.string(), groups: z.array(GroupSchema) });
+export const startTimeOf = (stat) => {
+    const close = stat.lastIndexOf(")");
+    return close < 0 ? undefined : stat.slice(close + 2).split(" ")[19];
+};
+export const stillLeftOver = (group, leaderStarted, groupAnswers) => leaderStarted === undefined ? groupAnswers : leaderStarted === group.started;
+const LINUX = process.platform === "linux";
+const bootId = async () => (await readFile("/proc/sys/kernel/random/boot_id", "utf8").catch(() => "")).trim();
+const leaderStart = async (pid) => startTimeOf(await readFile(`/proc/${pid}/stat`, "utf8").catch(() => ""));
+const groupAnswers = (pgid) => {
+    try {
+        process.kill(-pgid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+};
+const running = new Map();
+let kept = false;
+let boot;
+let writes = Promise.resolve();
+const save = () => {
+    writes = writes.then(async () => {
+        const groups = [...running].map(([pgid, started]) => ({ pgid, started }));
+        await writeFileAtomic(ledgerPath, JSON.stringify({ boot: await (boot ??= bootId()), groups }));
+    });
+    writes = writes.catch(() => undefined);
+    return writes;
+};
+export const recordCommand = async (pid) => {
+    const started = LINUX && kept ? await leaderStart(pid) : undefined;
+    if (started !== undefined) {
+        running.set(pid, started);
+        await save();
+    }
+};
+export const forgetCommand = async (pid) => {
+    if (running.delete(pid)) {
+        await save();
+    }
+};
+export const takeOverCommandLedger = async (log) => {
+    if (!LINUX) {
+        return;
+    }
+    kept = true;
+    const raw = await readFile(ledgerPath, "utf8").catch(() => undefined);
+    let ledger;
+    try {
+        const parsed = raw === undefined ? undefined : LedgerSchema.safeParse(JSON.parse(raw));
+        ledger = parsed?.success === true ? parsed.data : undefined;
+    }
+    catch {
+        ledger = undefined;
+    }
+    const current = await (boot ??= bootId());
+    const left = ledger === undefined || ledger.boot !== current ? [] : ledger.groups;
+    let ended = 0;
+    for (const group of left) {
+        if (stillLeftOver(group, await leaderStart(group.pgid), groupAnswers(group.pgid))) {
+            try {
+                process.kill(-group.pgid, "SIGKILL");
+                ended += 1;
+            }
+            catch (error) {
+                log(`could not end a command the last agent left running (group ${group.pgid}): ${errorMessage(error)}`);
+            }
+        }
+    }
+    if (ended > 0) {
+        log(`ended ${plural(ended, "command")} the last agent was running when it stopped without ending them.`);
+    }
+    await save();
+};
+//# sourceMappingURL=command-ledger.js.map