cawdev-cli 1.0.6-beta → 1.0.8-beta

package/runner/runner.mjs

@@ -48,6 +48,11 @@ import {
 } from './auto-workspaces.mjs';
 import { agentOfCommand, needsEverything } from '../lib/agents.mjs';
 import { cawdevToolNames, codexArgs, translateCodexEvent } from '../lib/codex.mjs';
+import {
+  drivingPath, isAlive, killedByRestart, leftBehind, memoryMaxOf, probeScope, readDriving,
+  restartSummary, scoped, snapshotOf, writeDriving,
+} from './restart.mjs';
+import { COMMITS_ITS_WORK, settleBranch } from '../lib/finish-check.mjs';
 
 // --- configuration -----------------------------------------------------------
 
@@ -420,6 +425,12 @@ async function readConfig() {
      * switched on, never on by default.
      */
     autoWorkspaces: autoWorkspacesOf(file.autoWorkspaces),
+    /**
+     * R339: the memory ceiling each agent session is spawned under, as
+     * systemd reads it (`6G`, `75%`) — Linux only, see restart.mjs. Absent is
+     * none. Raw here; `main` probes whether this machine can apply it.
+     */
+    sessionMemoryMax: file.sessionMemoryMax ?? null,
     skillCache: file.skillCache ?? DEFAULTS.skillCache,
     skillPrepareSeconds: file.skillPrepareSeconds ?? DEFAULTS.skillPrepareSeconds,
     idleSeconds: file.idleSeconds ?? DEFAULTS.idleSeconds,
@@ -6755,13 +6766,22 @@ async function walkLifecycle(config, run, runToken, cwd, baseCommit, workspace,
       if (decision.allowed) {
         break;
       }
+      if (decision.expired) {
+        // R339. Unread is not refused: re-running the stage would spend a
+        // second stage on a plan nobody has looked at yet.
+        const why = `Stopped at the ${stage.stage} gate: ${decision.reason}. Nothing was `
+          + 'refused — start the run again when somebody can read it.';
+        await skipRest(config, run, lifecycle, at + 1, why);
+        await finish(config, run, 'FINISHED', why, cwd);
+        return;
+      }
       if (attempt === 1) {
         const why = `Stopped at the ${stage.stage} gate: ${decision.reason ?? 'refused twice.'}`;
         // The same trail R122 came from: a run that ends here never reaches the
         // stages after it, and stages left PENDING on a run that is over read
         // as a lifecycle still going.
         await skipRest(config, run, lifecycle, at + 1, why);
-        await finish(config, run, 'FINISHED', why);
+        await finish(config, run, 'FINISHED', why, cwd);
         return;
       }
       // Refused once: the reason is the point of refusing, so it goes into the
@@ -6798,7 +6818,7 @@ async function walkLifecycle(config, run, runToken, cwd, baseCommit, workspace,
   const current = await api(config, `/api/projects/${run.projectSlug}/runs/${run.id}`)
     .catch(() => null);
   if (current?.live) {
-    await finish(config, run, 'FINISHED', last?.text ?? 'The lifecycle finished.');
+    await finish(config, run, 'FINISHED', last?.text ?? 'The lifecycle finished.', cwd);
   }
   await settleActions(config, run, cwd);
 }
@@ -6858,6 +6878,11 @@ async function awaitGate(config, run, runToken, stage, text) {
   }
 
   log(`  waiting at the ${stage.stage} gate`);
+  // R339: no deadline of its own by default. A gate is a person reading the
+  // plan, the platform never expires one, and a run waiting on it is never
+  // swept — a timer here turned "not read yet" into a refusal and re-ran the
+  // stage, which is spend nobody asked for. An older platform that does
+  // expire it still answers EXPIRED, which is handled below.
   const deadline = Date.now() + gateTimeoutSeconds() * 1000;
   while (Date.now() < deadline) {
     const remaining = Math.ceil((deadline - Date.now()) / 1000);
@@ -6869,6 +6894,12 @@ async function awaitGate(config, run, runToken, stage, text) {
       // the transport — a 204 and a `{state: 'PENDING'}` both mean "not yet",
       // and depending on which one the platform chose would be depending on
       // something nobody promised.
+      if (decision?.state === 'EXPIRED') {
+        // A platform from before R339. Nobody said no, so this is not a
+        // refusal to re-run the stage over: the run stops at the gate.
+        log(`  the ${stage.stage} gate expired unread`);
+        return { allowed: false, expired: true, reason: 'nobody read it in time' };
+      }
       if (decision && decision.state && decision.state !== 'PENDING') {
         const allowed = decision.state === 'ALLOWED';
         log(`  the ${stage.stage} gate was ${allowed ? 'allowed' : 'refused'}`);
@@ -6879,13 +6910,29 @@ async function awaitGate(config, run, runToken, stage, text) {
       return { allowed: false, reason: `waiting failed (${failure.message})` };
     }
   }
-  return { allowed: false, reason: 'nobody answered in time' };
+  return { allowed: false, expired: true, reason: 'nobody answered in time' };
 }
 
-/** A little past the platform's own expiry, like the MCP server's. */
+/**
+ * How long a gate waits before the daemon gives up on it: for ever, unless a
+ * test says otherwise — R339. The platform decides when nobody is coming.
+ */
 function gateTimeoutSeconds() {
   const configured = Number(process.env.CAWDEV_GATE_TIMEOUT_SECONDS);
-  return Number.isFinite(configured) && configured > 0 ? configured : 960;
+  return Number.isFinite(configured) && configured > 0 ? configured : Infinity;
+}
+
+/**
+ * An agent session, spawned under the machine's memory ceiling when it has
+ * one — R339. `systemd-run --scope` execs in place, so the pid, the group and
+ * the pipes are the agent's own. `cawdevCommand` is what the next daemon reads
+ * off the driving note to tell this session's orphan from a reused pid.
+ */
+function spawnScoped(config, command, args, options) {
+  const line = scoped(command, args, config.sessionScope);
+  const child = spawn(line.command, line.args, options);
+  child.cawdevCommand = command;
+  return child;
 }
 
 async function spawnAgent(config, run, runToken, cwd, baseCommit, workspace, resume,
@@ -7306,7 +7353,7 @@ async function spawnAgent(config, run, runToken, cwd, baseCommit, workspace, res
   log(`  spawning: ${agentCmd} ${args.join(' ')} (prompt on stdin)`);
 
   return new Promise((resolvePromise) => {
-    const child = spawn(agentCmd, args, {
+    const child = spawnScoped(config, agentCmd, args, {
       cwd,
       // Its own process group, so cancelling can take down the whole tree
       // rather than leaving orphaned children behind.
@@ -7922,7 +7969,7 @@ async function spawnAgent(config, run, runToken, cwd, baseCommit, workspace, res
           const summary = signal
             ? `The agent was terminated (${signal}).`
             : `The agent exited with code ${code} without reporting. ${lastText}`.trim();
-          await finish(config, run, code === 0 ? 'FINISHED' : 'FAILED', summary);
+          await finish(config, run, code === 0 ? 'FINISHED' : 'FAILED', summary, cwd);
         }
         // R112. A STAGE ending is not the RUN ending. The loop decides that —
         // it is the only thing that knows whether there is another stage — and
@@ -8418,7 +8465,7 @@ async function spawnAntigravity(config, run, runToken, cwd, baseCommit, resume,
     let childTotals = null;
     let err = '';
     let buffer = '';
-    const child = spawn(agyCmd, args, {
+    const child = spawnScoped(config, agyCmd, args, {
       cwd,
       // Its own process group — see the same choice on the Claude Code path.
       detached: true,
@@ -8567,7 +8614,7 @@ async function spawnAntigravity(config, run, runToken, cwd, baseCommit, resume,
         .catch(() => null);
       if (current?.live && !stage) {
         await finish(config, run, code === 0 && result?.status === 'SUCCESS' ? 'FINISHED' : 'FAILED',
-          text);
+          text, cwd);
       }
       // R112: not between stages — see the same line on the Claude Code path.
       if (!stage) {
@@ -8757,7 +8804,7 @@ async function spawnCodex(config, run, runToken, cwd, baseCommit, resume, projec
     let lastSaid = null;
     let err = '';
     let buffer = '';
-    const child = spawn(codexCmd, args, {
+    const child = spawnScoped(config, codexCmd, args, {
       cwd,
       // Its own process group — see the same choice on the Claude Code path.
       detached: true,
@@ -8871,7 +8918,7 @@ async function spawnCodex(config, run, runToken, cwd, baseCommit, resume, projec
       const current = await api(config, `/api/projects/${run.projectSlug}/runs/${run.id}`)
         .catch(() => null);
       if (current?.live && !stage) {
-        await finish(config, run, code === 0 && ended && !failed ? 'FINISHED' : 'FAILED', text);
+        await finish(config, run, code === 0 && ended && !failed ? 'FINISHED' : 'FAILED', text, cwd);
       }
       // R112: not between stages — see the same line on the Claude Code path.
       if (!stage) {
@@ -8930,7 +8977,32 @@ function reportStageUsage(config, run, stageName, totals) {
     log(`  could not record what the ${stageName} stage used: ${failure.message}`));
 }
 
-async function finish(config, run, state, summary) {
+/**
+ * Ends a run on the platform.
+ *
+ * R345: a coding run reaches FINISHED only committed and pushed. Given the
+ * checkout, a FINISHED for a profile that commits its work settles the branch
+ * first — commits what the session left (unless it looks like a credential),
+ * pushes without force — and becomes FAILED, with the reason in the summary
+ * and the checkout held for Carry on (R80), when that cannot be done. Before
+ * this, R325's run finished with a file modified and nothing pushed, and the
+ * next run in that checkout refused to start on top of it.
+ */
+async function finish(config, run, state, summary, cwd = null) {
+  if (state === 'FINISHED' && cwd && run.branch && COMMITS_ITS_WORK.has(run.profile)) {
+    const settled = await settleBranch(cwd, run.branch, {
+      push: pushBranch,
+      label: run.entryRef ?? 'cawdev',
+    }).catch((failure) => ({ ok: false, reason: `Checking the branch failed: ${failure.message}` }));
+    if (!settled.ok) {
+      log(`  not finished: ${settled.reason}`);
+      state = 'FAILED';
+      summary = `${settled.reason}\n\n${summary ?? ''}`.trim();
+    } else if (settled.note) {
+      log(`  ${settled.note}`);
+      summary = `${summary ?? ''}\n\n${settled.note}`.trim();
+    }
+  }
   await api(config, `/api/projects/${run.projectSlug}/runs/${run.id}/transition`, {
     method: 'POST',
     body: { state, summary },
@@ -8939,6 +9011,101 @@ async function finish(config, run, state, summary) {
 
 // --- cancellation ------------------------------------------------------------
 
+/**
+ * What the daemon before this one was driving when it died — R339.
+ *
+ * Read off the note it kept (restart.mjs), and only when its writer is dead:
+ * a live writer is a second daemon on this runner id. Each run the platform
+ * still believes in is failed at once as DAEMON_RESTARTED, in words that name
+ * the cause — where the stale sweep would have said, five minutes later, that
+ * the runner "stopped driving it". A session the old daemon left running
+ * unattended is stopped first: two sessions writing one checkout is worse
+ * than either.
+ */
+async function reportWhatARestartKilled(config) {
+  const path = drivingPath(config.runnerId);
+  const previous = await readDriving(path);
+  const behind = leftBehind(previous, process.pid, isAlive);
+  for (const left of behind) {
+    const run = await api(config, `/api/projects/${left.projectSlug}/runs/${left.id}`)
+      .catch((failure) => {
+        log(`could not read ${left.id} after the restart: ${failure.message}`);
+        return null;
+      });
+    if (!killedByRestart(run, config.runnerId)) {
+      continue;
+    }
+    const orphanStopped = await stopOrphan(left);
+    log(`run ${left.id} was being driven when this daemon last died (pid ${previous.pid}); `
+      + 'reporting it');
+    await api(config, `/api/projects/${left.projectSlug}/runs/${left.id}/transition`, {
+      method: 'POST',
+      body: {
+        state: 'FAILED',
+        failureReason: 'DAEMON_RESTARTED',
+        summary: restartSummary({
+          name: config.name, previousPid: previous.pid, pid: process.pid, orphanStopped,
+          stopped: previous.stopped ?? null,
+        }),
+      },
+    }).catch((failure) => log(`  could not report it: ${failure.message}`));
+  }
+  await writeDriving(path, snapshotOf(process.pid, running))
+    .catch((failure) => log(`could not write ${path}: ${failure.message}`));
+}
+
+/**
+ * The old daemon's session, if it outlived it — and only if the pid still
+ * runs the command it was spawned with, because after a reboot that number
+ * may be somebody else's process.
+ */
+async function stopOrphan(left) {
+  if (!left.childPid || !left.command || !isAlive(left.childPid)) {
+    return false;
+  }
+  const command = await new Promise((resolveCommand) => {
+    const ps = spawn('ps', ['-o', 'command=', '-p', String(left.childPid)],
+      { stdio: ['ignore', 'pipe', 'ignore'] });
+    let out = '';
+    ps.stdout.on('data', (chunk) => (out += chunk));
+    ps.on('error', () => resolveCommand(''));
+    ps.on('close', () => resolveCommand(out));
+  });
+  if (!command.includes(left.command)) {
+    return false;
+  }
+  try {
+    process.kill(-left.childPid, 'SIGTERM');
+    log(`  stopped the session it left running (pid ${left.childPid})`);
+    return true;
+  } catch (failure) {
+    log(`  could not stop the session it left running: ${failure.message}`);
+    return false;
+  }
+}
+
+/**
+ * Keeps the note current: rewritten whenever the set of runs being driven
+ * changes, checked every second. A second is the window in which a death
+ * goes unreported — and the stale sweep still catches that one.
+ */
+function keepTheDrivingNote(config) {
+  const path = drivingPath(config.runnerId);
+  let written = null;
+  const timer = setInterval(() => {
+    const snapshot = snapshotOf(process.pid, running);
+    const text = JSON.stringify(snapshot);
+    if (text === written) return;
+    written = text;
+    writeDriving(path, snapshot).catch((failure) => {
+      written = null;
+      log(`could not write ${path}: ${failure.message}`);
+    });
+  }, 1000);
+  timer.unref?.();
+  return timer;
+}
+
 /**
  * A cancelled run reaches the runner on its next poll, and the child's whole
  * process group goes with it — an agent that spawns a build should not leave it
@@ -9191,6 +9358,24 @@ async function main() {
   // than buried under a session that then fails for a reason it explains.
   await probePermissionPrompt(config);
 
+  // R339. Whether this machine can put each session under a memory ceiling,
+  // and what an earlier daemon on this runner left behind when it died —
+  // both before anything is claimed: a run reported now is one the queue
+  // must not hand the checkout of to somebody else first.
+  const memoryMax = memoryMaxOf(config.sessionMemoryMax);
+  if (config.sessionMemoryMax && !memoryMax) {
+    log(`sessionMemoryMax "${config.sessionMemoryMax}" is not a size systemd reads (6G, 512M, 75%); ignored`);
+  }
+  const probed = await probeScope(memoryMax);
+  config.sessionScope = probed.scope;
+  if (probed.scope) {
+    log(`each session runs under MemoryMax=${probed.scope.memoryMax} (systemd --${probed.scope.manager} scope)`);
+  } else if (probed.why) {
+    log(`sessionMemoryMax not applied: ${probed.why}`);
+  }
+  await reportWhatARestartKilled(config);
+  const drivingNote = keepTheDrivingNote(config);
+
   // What this daemon SAYS it is, on every hello — R52's socket, R288's rule
   // that it describes the current state. One object, sent as-is by
   // `serveControl` and mutated by the SIGHUP handler below, so a terminal that
@@ -9478,6 +9663,12 @@ async function main() {
       }
       log('stopping; leaving any live run to the platform’s staleness sweep');
       stopping = true;
+      // R339. Said in the note before the sessions go, so the next daemon
+      // tells a stop somebody asked for from a kill nobody did.
+      clearInterval(drivingNote);
+      await writeDriving(drivingPath(config.runnerId),
+        { ...snapshotOf(process.pid, running), stopped: signal })
+        .catch(() => undefined);
       clearInterval(heartbeat);
       clearInterval(gitSurvey);
       clearInterval(workspaceRequests);

package/runner/sign-in.mjs

@@ -26,6 +26,9 @@ export class Session {
     this.url = String(url).replace(/\/+$/, '');
     this.cookies = new Map();
     this.email = null;
+    /** Called once when a signed-in session is answered 401 — expired, or
+     *  revoked on the platform. The terminal offers `/login` from there. */
+    this.onExpired = null;
   }
 
   get signedIn() {
@@ -63,6 +66,14 @@ export class Session {
     });
     this.#remember(response);
     const text = await response.text();
+    if (response.status === 401 && this.email !== null) {
+      // The cookie stopped meaning anything: expired, signed out elsewhere, or
+      // the account revoked. Until this every action failed one at a time with
+      // a bare "HTTP 401" while the footer still named the person — so the
+      // state is put right HERE, where every request passes, and said once.
+      this.email = null;
+      this.onExpired?.();
+    }
     if (!response.ok) {
       const failure = new Error(messageIn(text) ?? `${method} ${path} failed: HTTP ${response.status}`);
       failure.status = response.status;