@nodii/dev 0.6.35 → 0.6.36

package/dist/gather.js

@@ -2,8 +2,8 @@ import { execFile } from "node:child_process";
 import { homedir, platform } from "node:os";
 import { promisify } from "node:util";
 import { randomUUID } from "node:crypto";
-import { link, lstat, mkdir, open, readFile, rm, stat, writeFile } from "node:fs/promises";
-import { join } from "node:path";
+import { appendFile, link, lstat, mkdir, open, readFile, rm, stat, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
 import { CliError } from "./core.js";
 
 const execFileAsync = promisify(execFile);
@@ -21,6 +21,10 @@ export const PICKER_CONTEXT = 8192;
 // A hard per-pick deadline: a pick that has not answered by then is stopped
 // (PICKER_TIMEOUT) so the next one never queues behind it.
 export const PICKER_TIMEOUT_MS = 30_000;
+// A lane Gather's first pick, timed only once the model is warm: real lanes
+// hit the 60 s Gather cap on a cold 12B (GATHER_TIMEOUT, 30 Oct).
+export const PICKER_FIRST_PICK_TIMEOUT_MS = 120_000;
+// Dev's requests in a Gather; the local warm-up and pick run on their own budgets.
 export const GATHER_TIMEOUT_MS = 60_000;
 export const PICKER_WARM_TIMEOUT_MS = 180_000;
 export const OLLAMA_URL = "http://127.0.0.1:11434";
@@ -79,9 +83,8 @@ async function jsonRequest(path, { baseUrl = ollamaUrl(), fetchImpl = fetch, tim
     }), requestSignal);
   } catch (error) {
     const timedOut = requestSignal.aborted || error?.name === "TimeoutError" || error?.name === "AbortError";
-    throw new CliError("PICKER_UNAVAILABLE", timedOut
-      ? `Local Gather picker timed out after ${timeoutMs} ms.`
-      : `Local Gather picker is unavailable at ${baseUrl}: ${error.message}. ${SETUP_FIX}`, 1);
+    if (timedOut) throw new CliError("PICKER_UNAVAILABLE", `Local Gather picker timed out after ${timeoutMs} ms.`, 1);
+    throw new CliError("PICKER_UNAVAILABLE", `Local Gather picker is unavailable at ${baseUrl}: ${error.message}. ${SETUP_FIX}`, 1, { unreachable: true, url: baseUrl });
   }
   let raw;
   try {
@@ -139,8 +142,41 @@ export async function requirePicker(options = {}) {
  */
 export async function warmPicker(options = {}) {
   const status = await requirePicker(options);
-  await jsonRequest("/api/generate", { ...options, timeoutMs: options.timeoutMs ?? PICKER_WARM_TIMEOUT_MS, body: { model: (options.model ?? PINNED).name, prompt: "", stream: false } });
-  return status;
+  const started = Date.now();
+  const payload = await jsonRequest("/api/generate", { ...options, timeoutMs: options.timeoutMs ?? PICKER_WARM_TIMEOUT_MS, body: { model: (options.model ?? PINNED).name, prompt: "", stream: false } });
+  return { ...status, latency: pickLatency(payload, Date.now() - started) };
+}
+
+/**
+ * Gather's deadline for Dev's requests. It stops while `hold` runs the local
+ * pick, which has its own budgets (warm-up, then the first pick), so a cold
+ * model never spends the requests' time.
+ */
+export function gatherDeadline(ms) {
+  const controller = new AbortController();
+  let remaining = ms;
+  let since = Date.now();
+  let timer = setTimeout(() => controller.abort(), remaining);
+  let holds = 0;
+  return {
+    ms,
+    signal: controller.signal,
+    async hold(task) {
+      if (holds++ === 0) {
+        clearTimeout(timer);
+        remaining -= Date.now() - since;
+      }
+      try {
+        return await task();
+      } finally {
+        if (--holds === 0 && !controller.signal.aborted) {
+          since = Date.now();
+          timer = setTimeout(() => controller.abort(), Math.max(0, remaining));
+        }
+      }
+    },
+    clear: () => clearTimeout(timer),
+  };
 }
 
 function pickerPrompt({ objective, files = [], triggers }) {
@@ -156,6 +192,16 @@ function pickerPrompt({ objective, files = [], triggers }) {
 
 const forced = (trigger) => Array.isArray(trigger?.forcedBy) ? trigger.forcedBy.length > 0 : Boolean(trigger?.forcedBy);
 
+/**
+ * Why a lane Gather hands the pick to the agent instead (hub-58, 30 Oct): this
+ * Mac has no picker model, or no Ollama at all. Null for every other failure.
+ */
+export function agentFallbackReason(error) {
+  if (error?.code === "PICKER_MODEL_MISSING") return `${error.details?.model ?? PICKER_MODEL} not installed; run nodii-dev setup`;
+  if (error?.code === "PICKER_UNAVAILABLE" && error.details?.unreachable) return `no local Ollama at ${error.details.url}; run nodii-dev setup`;
+  return null;
+}
+
 export async function pickDocuments(input, options = {}) {
   if (!input || typeof input.objective !== "string" || !input.objective.trim())
     throw new CliError("PICKER_INPUT_INVALID", "Gather needs a non-empty objective.", 2);
@@ -287,7 +333,7 @@ async function removeIfStale(lock, token, { alive, now, staleMs }) {
  * long). Takeovers are serialized, so the lock path is never emptied under a
  * live pick.
  */
-export async function withPickLock(lane, task, { dir = pickLockDir(), pid = process.pid, alive = processAlive, now = Date.now, staleMs = 2 * PICKER_TIMEOUT_MS, attempts = 40, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
+export async function withPickLock(lane, task, { dir = pickLockDir(), pid = process.pid, alive = processAlive, now = Date.now, staleMs = 2 * PICKER_FIRST_PICK_TIMEOUT_MS, attempts = 40, sleep = (ms) => new Promise((r) => setTimeout(r, ms)) } = {}) {
   await privateDir(dir);
   const name = encodeURIComponent(lane);
   const lock = join(dir, `${name}.lock`);
@@ -306,7 +352,7 @@ export async function withPickLock(lane, task, { dir = pickLockDir(), pid = proc
       }
       const verdict = await removeIfStale(lock, token, { alive, now, staleMs });
       if (!verdict.retry)
-        throw new CliError("PICKER_BUSY", `A Gather pick for lane ${lane} is already running (pid ${verdict.holder ?? "unknown"}); it ends within ${Math.round(PICKER_TIMEOUT_MS / 1000)} s. Rerun after it finishes (lock: ${lock}).`, 1, { pid: verdict.holder, lock });
+        throw new CliError("PICKER_BUSY", `A Gather pick for lane ${lane} is already running (pid ${verdict.holder ?? "unknown"}); it ends within ${Math.round(PICKER_FIRST_PICK_TIMEOUT_MS / 1000)} s. Rerun after it finishes (lock: ${lock}).`, 1, { pid: verdict.holder, lock });
       await sleep(5 + Math.random() * 20);
     }
   } finally {
@@ -325,6 +371,56 @@ const processAlive = (pid) => {
   catch (error) { return error?.code === "EPERM"; }
 };
 
+/** This Mac's Gather pick timings, one JSON line per lane pick (the p50/p95 evidence). */
+export const pickTimingsPath = (env = process.env) => env.NODII_DEV_GATHER_TIMINGS || join(homedir(), ".cache", "nodii-dev", "gather-timings.jsonl");
+
+export async function recordPickTiming(entry, { path = pickTimingsPath() } = {}) {
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  await appendFile(path, `${JSON.stringify(entry)}\n`, { mode: 0o600 });
+}
+
+/** The newest `last` timings, optionally for one model; unreadable lines are skipped. */
+export async function readPickTimings({ path = pickTimingsPath(), model, last = 50 } = {}) {
+  const raw = await readFile(path, "utf8").catch((error) => { if (error?.code === "ENOENT") return ""; throw error; });
+  const entries = raw.split("\n").flatMap((line) => { try { return line ? [JSON.parse(line)] : []; } catch { return []; } });
+  return entries.filter((x) => !model || x.model === model).slice(-last);
+}
+
+/** Nearest-rank percentile; null for no values. */
+export function percentile(values, p) {
+  const sorted = values.filter((x) => typeof x === "number").sort((a, b) => a - b);
+  if (!sorted.length) return null;
+  return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))];
+}
+
+export function pickTimingStats(entries) {
+  const ok = entries.filter((x) => x.outcome === "ok");
+  const spread = (key) => ({ p50: percentile(ok.map((x) => x[key]), 50), p95: percentile(ok.map((x) => x[key]), 95) });
+  const failed = {};
+  for (const x of entries) if (x.outcome !== "ok") failed[x.outcome] = (failed[x.outcome] ?? 0) + 1;
+  return { picks: entries.length, ok: ok.length, warmMs: spread("warmMs"), pickMs: spread("pickMs"), gatherPickMs: spread("totalMs"), failed };
+}
+
+const seconds = (value) => value === null || value === undefined ? "?" : `${(value / 1000).toFixed(1)} s`;
+
+export function pickTimingText(stats, { model, path } = {}) {
+  if (!stats.picks) return `${model ?? "Gather picker"}: no lane picks recorded on this Mac yet${path ? ` (${path})` : ""}.`;
+  const failed = Object.entries(stats.failed).map(([code, n]) => `${n} ${code}`).join(", ");
+  return [
+    `${model ?? "Gather picker"} · ${stats.picks} lane pick${stats.picks === 1 ? "" : "s"} on this Mac${path ? ` (${path})` : ""}`,
+    `  warm-up        p50 ${seconds(stats.warmMs.p50)}  p95 ${seconds(stats.warmMs.p95)}`,
+    `  first pick     p50 ${seconds(stats.pickMs.p50)}  p95 ${seconds(stats.pickMs.p95)}  (budget ${seconds(PICKER_FIRST_PICK_TIMEOUT_MS)} once warm)`,
+    `  warm-up + pick p50 ${seconds(stats.gatherPickMs.p50)}  p95 ${seconds(stats.gatherPickMs.p95)}`,
+    ...(failed ? [`  failed         ${failed}`] : []),
+  ].join("\n");
+}
+
+/** One line for a lane Gather's output: this pick, then this Mac's spread. */
+export function pickTimingLine(timing, stats) {
+  const spread = stats?.ok ? `; this Mac p50 ${seconds(stats.pickMs.p50)}, p95 ${seconds(stats.pickMs.p95)} over ${stats.ok} pick${stats.ok === 1 ? "" : "s"} (nodii-dev gather timings)` : "";
+  return `Picker ${timing.model}: warm-up ${seconds(timing.warmMs)}, pick ${seconds(timing.pickMs)} of its ${seconds(timing.budgetMs)} budget${spread}.`;
+}
+
 const LATENCY_PROBE = {
   objective: "Add a contacts list endpoint with cursor paging.",
   triggers: [

package/dist/lanes.js

@@ -4,7 +4,7 @@ import { promisify } from "node:util";
 import { CliError, encode, devRequest, query, readConfig, requireFlag } from "./core.js";
 import { launchRecovery } from "./runtime-launcher.js";
 import { recordReviewReceipt, reviewCommitContext } from "./safety.js";
-import { GATHER_TIMEOUT_MS, PICKER_MODEL, gatherText, pickDocuments, withPickLock } from "./gather.js";
+import { GATHER_TIMEOUT_MS, PICKER_FIRST_PICK_TIMEOUT_MS, PICKER_MODEL, agentFallbackReason, gatherDeadline, gatherText, pickDocuments, pickTimingLine, pickTimingStats, readPickTimings, recordPickTiming, warmPicker, withPickLock } from "./gather.js";
 import { storeGatherFreeze } from "./freeze.js";
 import { SWAP_BOUNDARIES, componentsForScope, guidelineLookup, laneFigmaScope, loadDsCatalog } from "./figma-scope.js";
 
@@ -207,7 +207,41 @@ async function currentGatherLane(request, config, gitContext) {
   return matches[0].key;
 }
 
-export async function laneCommand(action, args, flags, { request = devRequest, config = readConfig, gitContext = inferGitContext, commitsOf = branchCommits, read = (path) => readFile(path, "utf8"), reviewContext = reviewCommitContext, reviewReceipt = recordReviewReceipt, picker = pickDocuments, changedPaths = laneChangedPaths, gatherTimeoutMs = GATHER_TIMEOUT_MS, gatherSignal, storeFreeze = storeGatherFreeze, launch = launchRecovery, catalog = loadDsCatalog, pickLock = withPickLock } = {}) {
+const timedWarm = async (warm, signal) => {
+  const started = Date.now();
+  const status = await warm({ signal });
+  return { wallMs: Date.now() - started, loadMs: status?.latency?.loadMs ?? null };
+};
+
+/**
+ * The lane's local pick: wait for the warm-up begun at Gather start, then the
+ * first pick on its own budget. Every pick that reached the model is logged
+ * for this Mac's p50/p95, failures included.
+ */
+async function localPick(lane, warming, pick, { budgetMs, timings }) {
+  const started = Date.now();
+  const warmed = await warming;
+  const pickStarted = Date.now();
+  const entry = (outcome, picked) => ({
+    at: new Date().toISOString(), lane, model: picked?.model ?? PICKER_MODEL, outcome,
+    warmMs: pickStarted - started, warmLoadMs: warmed?.loadMs ?? null,
+    pickMs: picked?.latency?.wallMs ?? Date.now() - pickStarted, loadMs: picked?.latency?.loadMs ?? null,
+    totalMs: Date.now() - started, budgetMs,
+  });
+  let picked;
+  try {
+    picked = await pick();
+  } catch (error) {
+    if (error instanceof CliError && ["PICKER_TIMEOUT", "PICKER_RESPONSE_INVALID"].includes(error.code)) await timings.record(entry(error.code)).catch(() => {});
+    throw error;
+  }
+  const timing = entry("ok", picked);
+  await timings.record(timing).catch(() => {});
+  const stats = pickTimingStats(await timings.read({ model: timing.model }).catch(() => []));
+  return { ...picked, timing, stats };
+}
+
+export async function laneCommand(action, args, flags, { request = devRequest, config = readConfig, gitContext = inferGitContext, commitsOf = branchCommits, read = (path) => readFile(path, "utf8"), reviewContext = reviewCommitContext, reviewReceipt = recordReviewReceipt, picker = pickDocuments, warm = warmPicker, firstPickTimeoutMs = PICKER_FIRST_PICK_TIMEOUT_MS, timings = { record: recordPickTiming, read: readPickTimings }, changedPaths = laneChangedPaths, gatherTimeoutMs = GATHER_TIMEOUT_MS, deadline: outerDeadline, storeFreeze = storeGatherFreeze, launch = launchRecovery, catalog = loadDsCatalog, pickLock = withPickLock } = {}) {
   if (action === "list") {
     const c = flags.mine ? await config() : null;
     const payload = await request("GET", `/api/v1/lanes${query({ kind: flags.kind, status: flags.status, machine: flags.machine, agent: flags.mine ? c.agentId : flags.agent, "waiting-on": flags["waiting-on"], stuck: flags.stuck, cursor: flags.cursor, limit: flags.limit })}`);
@@ -248,11 +282,15 @@ export async function laneCommand(action, args, flags, { request = devRequest, c
     return { payload, human: (p) => detailText(p.data) };
   }
   if (action === "gather") {
-    const controller = gatherSignal ? null : new AbortController();
-    const deadline = controller ? setTimeout(() => controller.abort(), gatherTimeoutMs) : null;
-    const signal = gatherSignal ?? controller.signal;
+    const deadline = outerDeadline ?? gatherDeadline(gatherTimeoutMs);
+    const signal = deadline.signal;
+    // Stops a warm-up still loading when Gather fails before its pick.
+    const warmStop = new AbortController();
     try {
       const byAgent = agentPickerFlags(flags);
+      // Warm the local picker at Gather start, alongside Dev's requests (GATHER_TIMEOUT, 30 Oct).
+      const warming = byAgent ? null : timedWarm(warm, warmStop.signal);
+      warming?.catch(() => {});
       const lane = (await request("GET", `/api/v1/lanes/${encode(id)}`, undefined, { signal })).data;
       if (flags.objective !== undefined && (typeof flags.objective !== "string" || !flags.objective.trim()))
         throw new CliError("USAGE", "lane gather --objective needs non-empty text", 2);
@@ -353,7 +391,7 @@ export async function laneCommand(action, args, flags, { request = devRequest, c
         return { payload: { data: { recorded: false, picker: "agent", shortlistId, objective, triggers } }, human: (p) => agentShortlistText(ref, p.data, flags) };
       // Only the local model pick takes the per-lane pick lock; an agent pick
       // records a choice already made (AGENT-HARNESS-19 + GATHER-NA).
-      const picked = byAgent ? { documents: byAgent.documents } : await pickLock(lane.id ?? id, () => picker({
+      const pickInput = {
         objective,
         files: paths,
         triggers: triggers.map((trigger) => ({
@@ -361,7 +399,19 @@ export async function laneCommand(action, args, flags, { request = devRequest, c
           trigger: trigger.sentence,
           ...(trigger.forcedBy ? { forcedBy: trigger.forcedBy } : {}),
         })),
-      }, { signal }));
+      };
+      let picked = byAgent && { documents: byAgent.documents };
+      if (!picked) {
+        try {
+          // Dev's deadline stops while the pick runs on its own budgets.
+          picked = await deadline.hold(() => localPick(ref, warming, () => pickLock(lane.id ?? id, () => picker(pickInput, { signal, timeoutMs: firstPickTimeoutMs })), { budgetMs: firstPickTimeoutMs, timings }));
+        } catch (error) {
+          // No picker on this Mac: the agent picks from the same shortlist (hub-58).
+          const fallback = agentFallbackReason(error);
+          if (!fallback) throw error;
+          return { payload: { data: { recorded: false, picker: "agent", fallback, shortlistId, objective, triggers } }, human: (p) => `picker: agent (${p.data.fallback})\n${agentShortlistText(ref, p.data, flags)}` };
+        }
+      }
       const payload = await request("POST", `/api/v1/lanes/${encode(id)}/gather`, {
         objective,
         documentCodes: picked.documents,
@@ -401,13 +451,15 @@ export async function laneCommand(action, args, flags, { request = devRequest, c
         ...(invalidScope ? [`Notice: the launch Figma scope is not published keys (${invalidScope.replace(/\.$/, "")}), so the Freeze was stored with no Figma scope and every Figma lock and write is refused. Set the published keys with ${SWAP_BOUNDARIES(id)}.`] : []),
       ].map((line) => `\n${line}`).join("");
       // The pick's model and latency ride along in --json (morning check records them).
-      const result = { ...payload, picker: { model: picked.model ?? null, latency: picked.latency ?? null } };
-      return { payload: result, human: (p) => `${gatherText(p, objective, reasons)}${scopeNotice}` };
+      const result = { ...payload, picker: { model: picked.model ?? null, latency: picked.latency ?? null, ...(picked.timing ? { timing: picked.timing, stats: picked.stats } : {}) } };
+      const timingLine = picked.timing ? `\n${pickTimingLine(picked.timing, picked.stats)}` : "";
+      return { payload: result, human: (p) => `${gatherText(p, objective, reasons)}${scopeNotice}${timingLine}` };
     } catch (error) {
-      if (signal.aborted) throw new CliError("GATHER_TIMEOUT", `Gather exceeded its ${Math.round(gatherTimeoutMs / 1000)}-second deadline; nothing was retried.`, 1);
+      if (signal.aborted) throw new CliError("GATHER_TIMEOUT", `Gather's Dev requests exceeded their ${Math.round(deadline.ms / 1000)}-second deadline (the local pick runs on its own budget); nothing was retried.`, 1);
       throw error;
     } finally {
-      if (deadline) clearTimeout(deadline);
+      warmStop.abort();
+      if (!outerDeadline) deadline.clear();
     }
   }
   if (action === "step") {

package/dist/remote.js

@@ -0,0 +1,281 @@
+import { execFile } from "node:child_process";
+import { readFile } from "node:fs/promises";
+import { basename } from "node:path";
+import { promisify } from "node:util";
+import { CliError, devRequest, encode } from "./core.js";
+import { artifactEvidence, currentRunId, endRun, redactRunText, runKind, RUN_ID_ENV } from "./runs.js";
+
+// `nodii-dev run --remote-ssm` (AGENT-HARNESS-41): a script on an EC2
+// instance, watched like a local run. A benchmark that died silently on the
+// Dev EC2 went unheard for 8 hours (29-30 Oct); this fails it within N minutes.
+//
+// The script runs detached on the instance (its own session, so a cancel stops
+// its children), writing a log and then an exit-code file under its run
+// directory. Every poll is a short SSM command that reads the log's size and
+// tail, the exit code, and whether the process is alive. SSM keeps only the
+// first 24,000 characters of a command's output, so polling the script's own
+// invocation could neither see progress nor give the real tail.
+
+const execFileAsync = promisify(execFile);
+export const REMOTE_RUN_ROOT = "/var/tmp/nodii-dev-runs";
+const INSTANCE = /^m?i-[0-9a-f]{8,17}$/;
+const SCRIPT_MAX_BYTES = 32 * 1024;
+const TAIL_BYTES = 8 * 1024;
+const SSM_DONE = new Set(["Success", "Failed", "Cancelled", "TimedOut", "Cancelling", "Undeliverable", "Terminated", "DeliveryTimedOut", "ExecutionTimedOut", "InvalidPlatform", "AccessDenied"]);
+const sleepFor = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+const quote = (value) => `'${String(value).replace(/'/g, `'\\''`)}'`;
+
+/** The AWS CLI as nodii-dev calls it: JSON out, the caller's own credentials. */
+export function awsCli({ region, exec = execFileAsync } = {}) {
+  return async (args) => {
+    let stdout;
+    try {
+      ({ stdout } = await exec("aws", [...args, "--output", "json", ...(region ? ["--region", region] : [])], { encoding: "utf8", maxBuffer: 4 * 1024 * 1024 }));
+    } catch (error) {
+      if (error?.code === "ENOENT")
+        throw new CliError("AWS_CLI_REQUIRED", "run --remote-ssm needs the AWS CLI (aws) on PATH, signed in to the instance's account.", 2);
+      const text = String(error?.stderr || error?.message || "").trim();
+      const code = /InvocationDoesNotExist/.test(text) ? "SSM_INVOCATION_PENDING" : "SSM_CALL_FAILED";
+      throw new CliError(code, `aws ${args.slice(0, 2).join(" ")} failed: ${redactRunText(text.split("\n").at(-1) ?? "")}`, 1);
+    }
+    return stdout.trim() ? JSON.parse(stdout) : {};
+  };
+}
+
+/** One short shell command on the instance; resolves with its stdout. */
+export async function ssmShell(aws, instance, commands, { comment, waitMs = 90_000, sleep = sleepFor, now = Date.now } = {}) {
+  const sent = await aws(["ssm", "send-command", "--instance-ids", instance, "--document-name", "AWS-RunShellScript",
+    "--comment", comment.slice(0, 100), "--parameters", JSON.stringify({ commands: [commands], executionTimeout: ["120"] })]);
+  const commandId = sent.Command?.CommandId;
+  if (!commandId) throw new CliError("SSM_CALL_FAILED", "aws ssm send-command returned no command id.", 1);
+  const until = now() + waitMs;
+  for (;;) {
+    let invocation = null;
+    try {
+      invocation = await aws(["ssm", "get-command-invocation", "--command-id", commandId, "--instance-id", instance]);
+    } catch (error) {
+      if (error?.code !== "SSM_INVOCATION_PENDING") throw error;
+    }
+    if (invocation && SSM_DONE.has(invocation.Status)) {
+      if (invocation.Status !== "Success")
+        throw new CliError("SSM_COMMAND_FAILED", `SSM command on ${instance} ended ${invocation.Status}${invocation.StatusDetails && invocation.StatusDetails !== invocation.Status ? ` (${invocation.StatusDetails})` : ""}${invocation.StandardErrorContent ? `: ${redactRunText(invocation.StandardErrorContent.trim().split("\n").at(-1))}` : ""}.`, 1);
+      return { commandId, stdout: invocation.StandardOutputContent ?? "" };
+    }
+    if (now() >= until)
+      throw new CliError("SSM_UNREACHABLE", `${instance} did not finish an SSM command within ${Math.round(waitMs / 1000)} s (${invocation?.Status ?? "not delivered"}).`, 1);
+    await sleep(2_000);
+  }
+}
+
+export const remoteDir = (runId) => `${REMOTE_RUN_ROOT}/${runId}`;
+
+export function startCommand(runId, script) {
+  const dir = quote(remoteDir(runId));
+  return [
+    "set -eu",
+    `d=${dir}`,
+    'mkdir -p "$d" && chmod 700 "$d" && cd "$d"',
+    `printf '%s' ${quote(Buffer.from(script, "utf8").toString("base64"))} | base64 -d > script`,
+    "chmod 700 script",
+    // Its own session (setsid): a cancel signals the whole group.
+    `setsid sh -c 'cd "$1"; ./script > log 2>&1; echo $? > exit.tmp; mv exit.tmp exit' nodii-dev-run "$d" < /dev/null > /dev/null 2>&1 &`,
+    'echo $! > pid',
+    'echo "started $(cat pid)"',
+  ].join("\n");
+}
+
+export function pollCommand(runId, tailBytes = TAIL_BYTES) {
+  return [
+    `d=${quote(remoteDir(runId))}`,
+    'if [ ! -d "$d" ]; then echo "state=missing"; exit 0; fi',
+    'echo "size=$(wc -c < "$d/log" 2>/dev/null || echo 0)"',
+    'p=$(cat "$d/pid" 2>/dev/null || true)',
+    'if [ -f "$d/exit" ]; then echo "exit=$(cat "$d/exit")"',
+    'elif [ -n "$p" ] && kill -0 "$p" 2>/dev/null; then echo "state=running"',
+    'else echo "state=vanished"; fi',
+    'echo "--- tail ---"',
+    `tail -c ${tailBytes} "$d/log" 2>/dev/null || true`,
+  ].join("\n");
+}
+
+export function cancelCommand(runId) {
+  return [
+    `d=${quote(remoteDir(runId))}`,
+    'p=$(cat "$d/pid" 2>/dev/null || true)',
+    '[ -n "$p" ] || exit 0',
+    // `kill -SIG -PGID`, never `kill -SIG -- -PGID`: dash (Ubuntu's sh) and
+    // busybox refuse the `--` form, and SSM runs this with sh.
+    'kill -TERM "-$p" 2>/dev/null || kill -TERM "$p" 2>/dev/null || true',
+    "sleep 5",
+    'kill -KILL "-$p" 2>/dev/null || true',
+    'echo cancelled',
+  ].join("\n");
+}
+
+export function artifactCommand(runId, path) {
+  return [
+    `cd ${quote(remoteDir(runId))}`,
+    `f=${quote(path)}`,
+    'if [ ! -f "$f" ]; then echo missing; exit 0; fi',
+    'echo "bytes=$(wc -c < "$f")"',
+    'base64 < "$f" | tr -d "\\n"',
+  ].join("\n");
+}
+
+export function parsePoll(stdout) {
+  const [head, ...rest] = String(stdout).split("--- tail ---\n");
+  const fields = Object.fromEntries(head.split("\n").filter((line) => line.includes("=")).map((line) => [line.slice(0, line.indexOf("=")), line.slice(line.indexOf("=") + 1).trim()]));
+  const exitCode = fields.exit === undefined ? null : Number(fields.exit);
+  return {
+    state: exitCode !== null ? "exited" : fields.state ?? "unknown",
+    size: Number(fields.size ?? 0),
+    exitCode: Number.isInteger(exitCode) ? exitCode : exitCode === null ? null : -1,
+    tail: rest.join("--- tail ---\n"),
+  };
+}
+
+const minutes = (ms) => (ms >= 60_000 ? `${Math.round(ms / 60_000)} min` : `${Math.round(ms / 1000)} s`);
+
+/**
+ * Run SCRIPT on INSTANCE through SSM as a watched run: register it with Dev
+ * (the waiter is NODII_DEV_RUN_ID), heartbeat on every poll, and end it with
+ * the exit code, the remote log tail and, on success, the artifact. It fails
+ * on a non-zero exit, a process that vanished without an exit code, or no new
+ * output (or no answer from the instance) for `quietMinutes`.
+ */
+export async function superviseRemoteRun({
+  instance,
+  script,
+  artifact,
+  quietMinutes = 10,
+  pollSeconds = 30,
+  ssmWaitMs = 90_000,
+  kind,
+  name,
+  waiterRunId = process.env[RUN_ID_ENV],
+  request = devRequest,
+  aws = awsCli(),
+  readScript = (path) => readFile(path, "utf8"),
+  sleep = sleepFor,
+  now = Date.now,
+  stdoutWrite = (value) => process.stdout.write(value),
+  stderrWrite = (value) => process.stderr.write(value),
+} = {}) {
+  const waiter = currentRunId(waiterRunId);
+  if (!waiter)
+    throw new CliError("RUN_CONTEXT_REQUIRED", `nodii-dev run needs ${RUN_ID_ENV}, supplied by the harness launcher for the current waiter run.`, 2);
+  if (typeof instance !== "string" || !INSTANCE.test(instance))
+    throw new CliError("USAGE", "run --remote-ssm needs an instance id (i-… or mi-…).", 2);
+  if (typeof script !== "string" || !script)
+    throw new CliError("USAGE", "run --remote-ssm INSTANCE SCRIPT needs a local script file.", 2);
+  for (const [flag, value] of [["quiet-minutes", quietMinutes], ["poll-seconds", pollSeconds]])
+    if (!Number.isFinite(value) || value <= 0) throw new CliError("USAGE", `run --${flag} needs a positive number.`, 2);
+  if (artifact !== undefined && (typeof artifact !== "string" || !artifact))
+    throw new CliError("USAGE", "run --artifact needs a path on the instance.", 2);
+  let body;
+  try {
+    body = await readScript(script);
+  } catch (error) {
+    throw new CliError("USAGE", `Cannot read script ${script}: ${error?.code ?? error?.message}.`, 2);
+  }
+  if (!body.trim()) throw new CliError("USAGE", `Script ${script} is empty.`, 2);
+  if (Buffer.byteLength(body) > SCRIPT_MAX_BYTES)
+    throw new CliError("SCRIPT_TOO_LARGE", `Script ${script} is over ${SCRIPT_MAX_BYTES / 1024} KB; copy larger code to the instance and run a short script that calls it.`, 2);
+  const selectedKind = runKind(kind);
+  const label = redactRunText(String(name ?? `ssm ${instance} ${basename(script)}`)).trim().slice(0, 300);
+  const quietMs = quietMinutes * 60_000;
+  const registered = await request("POST", "/api/v1/runs", {
+    kind: selectedKind,
+    name: label,
+    externalRef: `ssm:${instance}`,
+    waiterRunId: waiter,
+    // A poll is an SSM round trip; Dev's watchdog allows a few of them.
+    expectedHeartbeatSeconds: Math.ceil(pollSeconds * 3 + 120),
+    metadata: { executable: "aws ssm", instance, script: basename(script), quietMinutes, ...(artifact ? { artifact } : {}) },
+  });
+  const runId = registered.data?.id;
+  if (!runId) throw new CliError("RUN_REGISTER_INVALID", "Dev registered a run without an id.", 1);
+
+  const ssm = (commands, what) => ssmShell(aws, instance, commands, { comment: `nodii-dev run ${runId} ${what}`, waitMs: ssmWaitMs, sleep, now });
+  const commandIds = [];
+  let tail = "";
+  const evidence = (exitCode, extra = []) => [
+    { kind: "command", ref: `ssm:${instance} ${basename(script)}` },
+    { kind: "remote", ref: `${instance}:${remoteDir(runId)}`, ...(commandIds.length ? { note: `SSM command ${commandIds[0]}` } : {}) },
+    { kind: "exit_code", ref: String(exitCode ?? -1) },
+    { kind: "log_tail", ref: "remote combined stdout/stderr", note: redactRunText(tail) || "(no output)" },
+    ...extra,
+  ];
+  const finish = async (status, exitCode, reason, extra = []) => {
+    await endRun(request, runId, status === "done"
+      ? { status, evidence: evidence(exitCode, extra) }
+      : { status, reason, evidence: evidence(exitCode, extra) });
+    if (reason) stderrWrite(`nodii-dev run: ${reason}\n`);
+    return { runId, exitCode: status === "done" ? 0 : exitCode || 1, reason: reason ?? null, tail: redactRunText(tail) };
+  };
+
+  try {
+    commandIds.push((await ssm(startCommand(runId, body), "start")).commandId);
+  } catch (error) {
+    return finish("failed", 1, `Remote script could not start on ${instance}: ${redactRunText(error?.message ?? error)}`);
+  }
+  stderrWrite(`nodii-dev run ${runId}: started on ${instance} (${remoteDir(runId)}); polling every ${pollSeconds} s, failing after ${minutes(quietMs)} without new output.\n`);
+
+  let size = 0;
+  let lastProgress = now();
+  let lastError = null;
+  let exited = null;
+  while (!exited) {
+    await sleep(pollSeconds * 1000);
+    let poll = null;
+    try {
+      poll = parsePoll((await ssm(pollCommand(runId), "poll")).stdout);
+      lastError = null;
+    } catch (error) {
+      lastError = redactRunText(error?.message ?? error);
+    }
+    if (poll) {
+      if (poll.size !== size) {
+        const added = poll.size - size;
+        const fresh = added > 0 && added <= Buffer.byteLength(poll.tail) ? Buffer.from(poll.tail).subarray(-added).toString("utf8") : poll.tail;
+        if (fresh) stdoutWrite(fresh);
+        size = poll.size;
+        lastProgress = now();
+      }
+      tail = poll.tail;
+      if (poll.state === "exited") {
+        exited = poll;
+        break;
+      }
+      if (poll.state === "vanished" || poll.state === "missing")
+        return finish("failed", 1, poll.state === "missing"
+          ? `Remote run directory ${remoteDir(runId)} is gone from ${instance} (the instance was replaced or cleaned).`
+          : `Remote script on ${instance} stopped without an exit code (killed, or the instance restarted).`);
+    }
+    if (now() - lastProgress >= quietMs) {
+      await ssm(cancelCommand(runId), "cancel").catch(() => {});
+      return finish("failed", 1, `Remote script on ${instance} went quiet: no new output for ${minutes(quietMs)}${lastError ? ` (last SSM error: ${lastError})` : ""}; it was cancelled.`);
+    }
+    try {
+      await request("POST", `/api/v1/runs/${encode(runId)}/heartbeat`, { note: lastError ? `ssm ${instance}: no answer (${lastError})` : `ssm ${instance}: running, log ${size} bytes` });
+    } catch (error) {
+      stderrWrite(`nodii-dev run heartbeat failed (${redactRunText(error?.code ?? error?.message ?? "error")}); retrying.\n`);
+    }
+    if (poll === null && lastError) stderrWrite(`nodii-dev run: no answer from ${instance} (${lastError}); retrying.\n`);
+  }
+
+  const { exitCode } = exited;
+  if (exitCode !== 0) return finish("failed", exitCode, `Remote script exited with code ${exitCode}.`);
+  if (!artifact) return finish("done", 0);
+  try {
+    const { stdout } = await ssm(artifactCommand(runId, artifact), "artifact");
+    if (stdout.trim() === "missing") throw new CliError("ARTIFACT_MISSING", `${artifact} does not exist on ${instance}.`, 1);
+    const [bytesLine, ...encoded] = stdout.split("\n");
+    const content = Buffer.from(encoded.join("").trim(), "base64").toString("utf8");
+    if (Buffer.byteLength(content) !== Number(bytesLine.replace("bytes=", "")))
+      throw new CliError("ARTIFACT_TRUNCATED", `${artifact} (${bytesLine.replace("bytes=", "")} bytes) arrived incomplete: SSM returns at most 24,000 characters of base64, and a Result carries at most 10,000 characters.`, 1);
+    return finish("done", 0, undefined, [artifactEvidence(artifact, content)]);
+  } catch (error) {
+    return finish("failed", 1, `Artifact not attached: ${redactRunText(error?.message ?? error)}`);
+  }
+}