claude-code-session-manager 0.26.0 → 0.28.0

package/src/main/scheduler.cjs

@@ -221,9 +221,27 @@ const ENV_CAP = process.env.SM_SCHEDULER_MAX_CONCURRENCY
   ? Math.max(1, Math.min(20, parseInt(process.env.SM_SCHEDULER_MAX_CONCURRENCY, 10) || 3))
   : null;
 
-// Each headless claude -p process can grow past 1 GB; require 1.5 GB headroom
-// per running+pending slot to avoid OOM (incident 2026-06-10).
-const MIN_FREE_MB_PER_JOB = 1500;
+// Each headless claude -p job can shell out to tsc/vite/pytest and grow well
+// past 1 GB at peak; reserve 2.5 GB per running+pending slot. Raised from 1.5 GB
+// after the 2026-06-16 OOM: 3 concurrent cross-project jobs + their build
+// subprocesses pushed a 24 GB host ~10 GB into swap and the OOM killer took
+// Electron — every pty got SIGHUP (code=0 signal=1).
+const MIN_FREE_MB_PER_JOB = 2500;
+
+// Absolute headroom kept free for the Electron host (main + renderer + GPU) and
+// the OS — NEVER lent to jobs. Without it the gate green-lights a job whenever
+// MemAvailable is just above per-job need, starving the very process that owns
+// the ptys; that's the one the OOM killer then reaps. Subtracted from
+// MemAvailable before the per-job gate runs. See availableForJobs().
+const RESERVED_HOST_MB = 3000;
+
+// oom_score_adj applied to each spawned claude -p job (range -1000..1000;
+// Electron inherits the default 0). A positive bias makes the kernel OOM killer
+// prefer a disposable, restartable job over Electron — whose death SIGHUPs every
+// pty and drops the sleep inhibitor. The job's build subprocesses (tsc/vite)
+// inherit it, so the actual memory hogs are the preferred victims. The gate caps
+// how many START; this decides who dies if a spike slips through anyway.
+const OOM_SCORE_ADJ_JOB = 500;
 
 const DEFAULT_CONFIG = {
   offsetMinutes: 15,
@@ -277,6 +295,33 @@ function memoryLimitedBatchSize(availableMb, minPerJob, runningCount, batchLen)
   return allowed;
 }
 
+/**
+ * Memory available to LAUNCH jobs with: MemAvailable minus the host reserve,
+ * floored at 0. Keeping the host reserve out of the job budget is what stops the
+ * gate from green-lighting a job into an OOM that kills Electron. Fails open
+ * (Infinity) on non-Linux where MemAvailable is unknown. Exported for tests.
+ */
+function availableForJobs(availableMb, reservedHostMb) {
+  if (availableMb === Infinity) return Infinity;
+  return Math.max(0, availableMb - reservedHostMb);
+}
+
+/**
+ * Bias a spawned job's oom_score_adj up so the kernel OOM killer sacrifices the
+ * (restartable) job before Electron. Raising a child's OWN score is privilege-
+ * free; lowering Electron's would need CAP_SYS_RESOURCE. Linux-only, best-effort
+ * — the job may have already exited (write ENOENTs), which is fine. See the
+ * 2026-06-16 OOM-kills-Electron incident.
+ */
+function biasJobOomScore(pid) {
+  if (process.platform !== 'linux' || !pid) return;
+  try {
+    fs.writeFileSync(`/proc/${pid}/oom_score_adj`, String(OOM_SCORE_ADJ_JOB));
+  } catch {
+    /* job already exited, or /proc unavailable — best-effort hardening only */
+  }
+}
+
 // ---------- fs helpers ----------
 
 /**
@@ -292,9 +337,21 @@ function safeSlugPath(slug) {
   return resolved;
 }
 
+// Bundled authoring guide seeded into the scheduler dir so the session-manager-dev
+// plugin's /develop and /prd skills — which reference this stable `~`-absolute
+// path — work on any user's machine, not just the author's.
+const PRD_AUTHORING_TEMPLATE = path.join(__dirname, 'templates', 'PRD_AUTHORING.md');
+const PRD_AUTHORING_DEST = path.join(ROOT, 'PRD_AUTHORING.md');
+
 function ensureDirs() {
   fs.mkdirSync(PRDS_DIR, { recursive: true });
   fs.mkdirSync(RUNS_DIR, { recursive: true });
+  // Seed the authoring guide once; never clobber a user's edited copy.
+  try {
+    if (!fs.existsSync(PRD_AUTHORING_DEST) && fs.existsSync(PRD_AUTHORING_TEMPLATE)) {
+      fs.copyFileSync(PRD_AUTHORING_TEMPLATE, PRD_AUTHORING_DEST);
+    }
+  } catch { /* non-fatal: the guide is a convenience, not load-bearing for a run */ }
 }
 
 // Atomic JSON write helpers delegate to config.cjs's shared implementation.
@@ -1038,6 +1095,8 @@ async function executeJob(job, runDir, defaultCwd, onPid) {
 
     if (child) {
       safeLog(`[scheduler] spawned pid=${child.pid} sessionId=${sessionId} (process group)\n\n`);
+      // Make this job the OOM killer's preferred victim over Electron.
+      biasJobOomScore(child.pid);
       // Fire-and-forget pid persistence — best effort.
       if (onPid) onPid(child.pid, sessionId, cwd).catch(() => {});
     }
@@ -1057,6 +1116,15 @@ function isFixPlanSlug(slug) {
   return /^\d+-fix-/.test(slug);
 }
 
+/**
+ * Returns true for statuses that a fix-plan completion should promote
+ * (clear) on the original job. Both 'failed' and 'needs_review' are
+ * recoverable via a fix-plan; 'completed', 'running', 'pending' are not.
+ */
+function isPromotableOriginal(status) {
+  return status === 'failed' || status === 'needs_review';
+}
+
 /**
  * Spawn an Opus investigation session for a failed job. The investigator's job
  * is to read the failure log + original PRD, identify the root cause, and write
@@ -1399,13 +1467,17 @@ async function spawnJob(job, runId, runDir, defaultCwd) {
           // though the auto-recovery did its job.
           if (effectiveStatus === 'completed' && isFixPlanSlug(job.slug)) {
             const originalSlug = job.slug.replace(/^(\d+)-fix-/, '$1-');
-            const orig = s.jobs.findIndex((x) => x.slug === originalSlug && x.status === 'failed');
+            const orig = s.jobs.findIndex((x) => x.slug === originalSlug && isPromotableOriginal(x.status));
             if (orig >= 0) {
-              console.log(`[scheduler] auto-promote: ${originalSlug} (failed) → completed because ${job.slug} succeeded`);
+              const priorStatus = s.jobs[orig].status;
+              console.log(`[scheduler] auto-promote: ${originalSlug} (${priorStatus}) → completed because ${job.slug} succeeded`);
               s.jobs[orig].status = 'completed';
               s.jobs[orig].exitCode = 0;
               s.jobs[orig].error = null;
               s.jobs[orig].completedBy = job.slug;
+              if (priorStatus === 'needs_review') {
+                delete s.jobs[orig].verifierVerdict;
+              }
             }
           }
         }
@@ -1483,17 +1555,21 @@ function tickQueue() {
     }
 
     const availableMb = getAvailableMemMb();
-    const allowed = memoryLimitedBatchSize(availableMb, MIN_FREE_MB_PER_JOB, runningSet.size, batch.length);
+    // Reserve a fixed slice for the Electron host before the per-job gate, so a
+    // job is never started into the host's own headroom (that path OOM-kills
+    // Electron and SIGHUPs every pty — 2026-06-16 incident).
+    const jobBudgetMb = availableForJobs(availableMb, RESERVED_HOST_MB);
+    const allowed = memoryLimitedBatchSize(jobBudgetMb, MIN_FREE_MB_PER_JOB, runningSet.size, batch.length);
     if (allowed === 0) {
-      const threshold = MIN_FREE_MB_PER_JOB * (runningSet.size + 1);
-      console.log(`[scheduler] memory gate: available=${availableMb} MB < threshold=${threshold} MB — deferring ${batch.length} job(s)`);
+      const threshold = RESERVED_HOST_MB + MIN_FREE_MB_PER_JOB * (runningSet.size + 1);
+      console.log(`[scheduler] memory gate: available=${availableMb} MB < threshold=${threshold} MB (host reserve ${RESERVED_HOST_MB} + ${MIN_FREE_MB_PER_JOB}/job × ${runningSet.size + 1}) — deferring ${batch.length} job(s)`);
       lastMemGate = { availableMb, threshold, deferred: true, at: new Date().toISOString() };
       return;
     }
     const gatedBatch = batch.slice(0, allowed);
     if (gatedBatch.length < batch.length) {
-      console.log(`[scheduler] memory gate: available=${availableMb} MB — clamped batch ${batch.length} → ${gatedBatch.length}`);
-      lastMemGate = { availableMb, threshold: MIN_FREE_MB_PER_JOB * (runningSet.size + gatedBatch.length), deferred: false, clamped: true, at: new Date().toISOString() };
+      console.log(`[scheduler] memory gate: available=${availableMb} MB — clamped batch ${batch.length} → ${gatedBatch.length} (host reserve ${RESERVED_HOST_MB} + ${MIN_FREE_MB_PER_JOB}/job)`);
+      lastMemGate = { availableMb, threshold: RESERVED_HOST_MB + MIN_FREE_MB_PER_JOB * (runningSet.size + gatedBatch.length), deferred: false, clamped: true, at: new Date().toISOString() };
     } else {
       // Ungated full batch: clear stale gate snapshot so status doesn't show
       // a stale deferral from a previous tick.
@@ -1739,6 +1815,31 @@ function isRescanCandidate(job) {
  *
  * @returns {Promise<{rescanned:number, healed:string[]}>}
  */
+/**
+ * Pure helper — no I/O. Returns the subset of jobs eligible for automatic
+ * fix-plan authoring after a reverify pass leaves them still in needs_review.
+ *
+ * Exclusion rules (all must pass):
+ *   - status === 'needs_review'
+ *   - truthy runId (need a run log to investigate)
+ *   - autoFixAttempted !== true (1-attempt cap)
+ *   - not itself a fix-plan slug (avoids infinite recursion)
+ *   - no fix sibling on disk (fixSlugExists) or already in the queue
+ */
+function selectAutoFixTargets(jobs, { fixSlugExists }) {
+  const slugsInQueue = new Set(jobs.map((j) => j.slug));
+  return jobs.filter((job) => {
+    if (job.status !== 'needs_review') return false;
+    if (!job.runId) return false;
+    if (job.autoFixAttempted) return false;
+    if (isFixPlanSlug(job.slug)) return false;
+    const fixSlug = `${String(job.parallelGroup ?? 99).padStart(2, '0')}-fix-${job.slug.replace(/^\d+-/, '')}`;
+    if (fixSlugExists(fixSlug)) return false;
+    if (slugsInQueue.has(fixSlug)) return false;
+    return true;
+  });
+}
+
 async function reverifyNeedsReview() {
   const snap = await readQueue();
   const candidates = snap.jobs.filter(isRescanCandidate);
@@ -1786,6 +1887,29 @@ async function reverifyNeedsReview() {
     const detail = leftForReview.map((e) => `${e.slug} (${e.reason})`).join(', ');
     console.log(`[scheduler] boot reverify: left for review: ${detail}`);
   }
+
+  // Auto-fix: spawn a fix-plan investigation for each job still in
+  // needs_review after the heal pass (kill-switch: SM_AUTOFIX_DISABLE=1).
+  if (process.env.SM_AUTOFIX_DISABLE !== '1') {
+    const afterHeal = await readQueue();
+    const targets = selectAutoFixTargets(afterHeal.jobs, {
+      fixSlugExists: (s) => fs.existsSync(path.join(PRDS_DIR, `${s}.md`)),
+    });
+    for (const job of targets) {
+      const runDir = path.join(RUNS_DIR, job.runId);
+      // Persist cap BEFORE spawning — a crash mid-investigation still counts
+      // the attempt (mirrors orphanRetries pattern).
+      await mutate((s) => {
+        const j = s.jobs.find((x) => x.slug === job.slug);
+        if (j) j.autoFixAttempted = true;
+      });
+      console.log(`[scheduler] auto-fix: needs_review ${job.slug} → authoring fix-plan (1/1)`);
+      spawnInvestigation(job, runDir).catch((e) => {
+        console.error('[scheduler] auto-fix spawnInvestigation error', job.slug, e);
+      });
+    }
+  }
+
   return { rescanned: candidates.length, healed, leftForReview };
 }
 
@@ -2302,4 +2426,4 @@ const remote = {
   },
 };
 
-module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, reverifyNeedsReview, isRescanCandidate };
+module.exports = { registerScheduleHandlers, attachWindow, init, ROOT, PRDS_DIR, selectHistoryJobs, parsePorcelain, FINISH_PROTOCOL, remote, pickNextBatch, pickForProject, reapDeadRunningJobs, pollRecoveryClearSource, memoryLimitedBatchSize, availableForJobs, reverifyNeedsReview, isRescanCandidate, isPromotableOriginal, selectAutoFixTargets };

package/src/main/templates/PRD_AUTHORING.md

@@ -0,0 +1,316 @@
+# PRD Authoring Guide — Scheduler Safety Rules
+
+This guide codifies lessons from two stuck-job incidents (fizzpop poll-hang, etch-engine post-AC overrun) into enforceable rules every PRD MUST follow. Violating these rules costs real money and wastes hours waiting at a terminal.
+
+Before queueing a new PRD, run through the §10 checklist at the bottom.
+
+---
+
+## §1 Bounded waits, never unbounded polls
+
+**Summary:** Every `until`/`while` loop that makes network calls or waits for external state MUST have a hard iteration cap that surfaces non-zero on exhaustion.
+
+**Anti-example** (verbatim from `106-fizzpop-publish.md`):
+```bash
+# WRONG — what 106-fizzpop-publish did
+PREV=288694
+until [ "$(curl -s https://bilko.run/api/health | jq .uptime)" -lt "$PREV" ]; do
+  sleep 15
+done
+# Outcome: 2h47m hang because static-content Render deploys never restart the Node API
+# so `uptime` never dropped. Loop hung until the 4h watchdog SIGKILLed.
+```
+
+**Recommended pattern:**
+```bash
+# RIGHT — bounded, surfaces failure cleanly
+for i in $(seq 1 20); do
+  if curl -sf https://bilko.run/projects/fizzpop/ > /dev/null; then
+    echo "deploy live (attempt $i)"; break
+  fi
+  echo "waiting for deploy ($i/20)..."
+  sleep 15
+done
+# Smoke test downstream will diagnose the real failure if the deploy didn't land.
+```
+
+**Rule:** Every `until`/`while` network-or-state poll MUST use `for i in $(seq 1 N); do ...; done` with a hard cap. Recommended cap: 20 × 15 s = 5 min for HTTP polls. On exhaustion, print a diagnostic line and continue (let the smoke test below catch the real failure).
+
+---
+
+## §2 Don't add work past the acceptance checklist
+
+**Summary:** Once every AC line is ticked, write the result and exit. Do not add polish, fixtures, generators, or "while we're here" improvements not enumerated in the PRD.
+
+**Anti-example** (verbatim from `112-etch-engine.md`):
+```bash
+# WRONG — what 112-etch-engine did after declaring success
+for (let seed = 100; seed < 50000 && !found; seed++) {
+  const result = generateRandom({ size, rng, maxAttempts: 3 });
+  if (result.ok && result.solution) { found = true; ... }
+}
+# Outcome: 2h44m of token burn looking for fixtures the AC did not require.
+# The agent had emitted result-success at 17:44 UTC; the bonus loop ran until 20:28.
+```
+
+**Rule:** Once every AC line is checked, write the result and exit 0. If bonus work seems genuinely valuable, write a follow-up PRD and reference it in the result. Do not add any work not explicitly enumerated in an AC line.
+
+---
+
+## §3 Smoke tests verify; spin-waits don't
+
+**Summary:** Prefer "do thing, run a test that asserts thing happened" over "do thing, spin until I detect thing happened."
+
+**Rule:** After a deployment, migration, or build step, run an actual test command (`curl -sf`, `npm test`, `pnpm typecheck`) that exits non-zero on failure. A spin-wait that polls for a condition hides the failure mode; a test command surfaces the exact error.
+
+**Pattern:**
+```bash
+# Deploy step above (bounded, §1)
+# Smoke test — will exit 1 with a clear message if deploy failed:
+curl -sf https://bilko.run/projects/fizzpop/ > /dev/null || { echo "smoke test FAILED: fizzpop not reachable"; exit 1; }
+```
+
+---
+
+## §4 Bound any generator/search loop with max-attempts and surface-on-exhaustion
+
+**Summary:** When iterating a search space (fixtures, seeds, brute-force), declare the maximum search size in the PRD AC and surface and HALT on exhaustion.
+
+**Anti-example:** Same etch-engine fixture generator from §2 — `for (let seed = 100; seed < 50000 ...)` with no AC line constraining it.
+
+**Rule:** When iterating a search space, the PRD AC MUST state the bound explicitly ("up to 1000 seeds; if exhausted, surface and HALT"). The executor knows when to give up and moves on rather than burning tokens indefinitely.
+
+**Pattern:**
+```ts
+let found = false;
+for (let seed = 0; seed < MAX_SEEDS && !found; seed++) {
+  // ...
+}
+if (!found) {
+  console.error(`HALT: exhausted ${MAX_SEEDS} seeds without finding a valid fixture`);
+  process.exit(1);
+}
+```
+
+---
+
+## §5 Render/deploy waits are bounded and always followed by a smoke test
+
+**Summary:** Render (and similar) deploys may take 2–10 minutes and may silently fail. Always bound the wait and follow it with a live endpoint test.
+
+**Pattern:**
+```bash
+DEPLOY_OK=0
+for i in $(seq 1 20); do
+  if curl -sf https://bilko.run/projects/<slug>/ > /dev/null; then
+    DEPLOY_OK=1; echo "deploy live (attempt $i)"; break
+  fi
+  echo "waiting for deploy ($i/20)..."
+  sleep 15
+done
+# Continue regardless. Smoke test below catches the real failure.
+if [ $DEPLOY_OK -eq 0 ]; then
+  echo "WARNING: deploy not detected after 20 attempts; continuing to smoke test"
+fi
+curl -sf https://bilko.run/projects/<slug>/ > /dev/null || { echo "SMOKE TEST FAILED"; exit 1; }
+```
+
+**Rule:** A static-content deploy on Render does NOT restart the backend API. Never use `uptime` or process-restart signals to detect a static-content deploy — use the actual URL that should be live.
+
+---
+
+## §6 Frontmatter rules
+
+**Summary:** Required keys are `title`, `cwd` (absolute path), `estimateMinutes`. Default to letting the filename `NN-` prefix drive grouping; include `parallelGroup` ONLY when you are deliberately interleaving across project streams.
+
+**Required frontmatter:**
+```yaml
+---
+title: <one line, plain English>
+cwd: ~/Projects/<target-repo>
+estimateMinutes: 60
+---
+```
+
+**Cross-machine portability:** Write `cwd` as `~/Projects/<name>` — the parser expands `~` to `os.homedir()` at ingest time, so the same PRD file works on Linux (`/home/<u>/...`) and macOS (`/Users/<u>/...`). Absolute paths (e.g. `/home/bilko/Projects/foo`) are passed through unchanged and will break on any machine with a different home directory.
+
+**Rules:**
+- `cwd` MUST point to the target project. Prefer `~/...` for portability; only use an absolute path if you have a specific reason to pin to one machine.
+- **`cwd` MUST already exist on disk at queue time.** The scheduler runs a dead-cwd guard (`fs.accessSync(cwd, fs.constants.X_OK)` in `src/main/scheduler.cjs:669-680`) *before* spawning the child, so a PRD whose `cwd` references a not-yet-created directory will exit with `-1: cwd no longer exists` and the body will never run — even if the first step of the body would have created the directory. If the PRD's purpose is to create a brand-new sibling project at `~/Projects/<new-slug>/`, point `cwd` at the parent (`~/Projects`) and make the first executable step `mkdir -p ~/Projects/<new-slug> && cd ~/Projects/<new-slug>`.
+- `estimateMinutes` is used for ETA display; include a realistic estimate (note: empirical median is ~10 min, p90 ~20 min — avoid wildly inflated estimates that hide real outliers).
+- `parallelGroup` in frontmatter, when present, IS honored by the scheduler (`pickNextBatch` reads `parallelGroup ?? 99` and overrides the filename NN). Use this only for cross-stream interleaving — e.g., the cellar series `122-`, `123-`, `124-` overrides to groups `113`, `114`, `115` so cellar steps fire alongside the parallel etch steps. Do NOT use it to reorder within a single stream; rename the file instead.
+
+---
+
+## §7 Self-containment
+
+**Summary:** The PRD body is the executor's entire context. It runs as `claude -p "<body>"` with no conversation history.
+
+**Rule:** Include exact file paths, function signatures if they save a Read, library versions, and the name of any sibling PRD the executor must NOT duplicate. Do not reference "the conversation", "what we discussed", "the design doc", or any other external context. If the executor would need to search for something, include the answer.
+
+---
+
+## §8 Scope sizing — target ~15 min, ceiling 30 (data-driven, 2026-06)
+
+**Summary:** One PRD ≈ **~15 wall-clock minutes** of work. Empirically (400+ runs) median real run = **~7 min**, p90 = **21 min**; authored estimates ran 5–8× too high. **If you project >30 min, SPLIT.**
+
+**Rule:** Split larger work into sequential PRDs; reference the dependency in `# Implementation notes`. PRDs in the same `NN-` group run in parallel — don't put dependent work in the same group. e2e/publish work is the failure tail: **shard test suites to one spec per PRD; never run a full suite or an endpoint-polling publish in a single PRD** (§1/§5).
+
+---
+
+## §9 Failure surfacing
+
+**Summary:** Prefer `exit 1` with a one-line diagnosis over silent retries. Note: a `rateLimited` exit-1 is the scheduler's benign auto-pause (it auto-resumes at the next 5h reset), NOT an authoring failure — don't engineer retry logic for it.
+
+**Rule:** When a step fails, print a single diagnostic line and exit 1. The scheduler marks the job `failed` and the investigator Claude reads the log. A clean failure message is worth more than a 15-minute silent retry loop. Do not swallow errors with `|| true` unless the failure is genuinely non-fatal and you explain why.
+
+```bash
+# RIGHT
+npm test || { echo "HALT: npm test failed — see above"; exit 1; }
+
+# WRONG
+npm test || true   # silently continues even if tests are broken
+```
+
+---
+
+## §10 Pre-queue checklist (the litany)
+
+Before queueing a new PRD, verify each of these:
+
+- [ ] **§1 Bounded waits:** Every `until`/`while` poll has a `for i in $(seq 1 N)` cap ≤ 20 iterations.
+- [ ] **Every command bounded:** Every test/build/dev-server/deploy command is wrapped in `timeout` (typecheck/unit 300s, e2e 120s, `curl --max-time 15`). No bare `playwright test` / `vite` / `pnpm dev` / `curl … | head`.
+- [ ] **§2 No bonus work:** AC list is the only source of work. No "while we're here" additions.
+- [ ] **§3 Smoke tests + verify-before-done:** Every deploy/migration step is followed by a test command that exits 1 on failure. Run the AC test command once before declaring done; never end the run on a red test.
+- [ ] **§4 Bounded generators:** Any search/seed loop has an explicit `MAX_ATTEMPTS` constant and surfaces failure on exhaustion.
+- [ ] **§5 Render deploys:** Deploy waits use a live URL check, not uptime/restart signals.
+- [ ] **§6 Frontmatter:** `title`, `cwd` (`~/Projects/<name>` preferred; path MUST exist on this machine), `estimateMinutes` present. `parallelGroup` only if intentionally interleaving cross-stream.
+- [ ] **§7 Self-contained:** No references to "the conversation" or external context. Paths and identifiers are inline. Body is clean UTF-8 — **no NUL/control bytes** (paste-from-PDF crashes the spawn). Quick check: `grep -qP '\x00' file && echo BAD`.
+- [ ] **§8 Scope:** Targets ~15 min, ceiling 30. If projected larger, split. e2e/publish sharded to one spec per PRD.
+- [ ] **§9 Failure surfacing:** Errors exit 1 with a diagnostic line. No silent `|| true` swallows. (`rateLimited` exit-1 is benign auto-pause, not a failure.)
+- [ ] **§11 Negative-assertion checks:** Any "this should produce NO output / NO match" check (a `grep` that should find nothing, a "no leftover X" guard) is written as an inverted conditional that exits 0 on the clean case. A bare `grep` whose success is "no match" exits 1 and trips the verifier `transcript_errors` downgrade even when the run is perfect.
+- [ ] **§12 End green:** The acceptance/test gate is the LAST thing the run does; any intentionally-failing step (TDD red test, expected-nonzero probe) runs EARLY, never after the gate, and is captured (`2>&1 | tail` inside a conditional) so it doesn't surface as a bare `is_error`/`Traceback` in the final portion of the transcript.
+- [ ] **§13 Recover/annotate errors & expected timeouts:** A throwaway probe that errors is re-run corrected (or annotated `# expected/handled`) right after — never left stranded; prefer a temp `.py` over a fragile inline `python -c`. An *expected* `timeout` cap (a long ingest/scan) handles exit 124 explicitly as success-with-note, not a bare `Exit code 124`. Both prevent the `transcript_errors` downgrade of a green deliverable.
+
+---
+
+## §11 Negative-assertion checks must exit 0 on the clean case
+
+**Summary:** A check that asserts the *absence* of something must return exit 0 when the
+thing is absent. The classic trap is `grep`: it exits **1 when it finds no match**. If your
+AC says "verify no banned phrase remains" and you write a bare `grep`, the *success* path
+(nothing found) surfaces as `is_error=true` in the transcript — and the verifier's
+`transcript_errors` heuristic downgrades the whole run to `needs_review` even though it did
+everything right.
+
+**This actually happened** (PRD `62-x-trader-doctrine`, 2026-06-13): the doctrine was cleaned
+correctly and committed, but the AC's sanity grep —
+`grep -rniE "building in public|..." data/pipelines/x_session/` — found nothing, exited 1,
+and a perfect run was flagged for review. Self-inflicted, by the PRD author.
+
+```bash
+# WRONG — exits 1 (is_error) exactly when the check PASSES
+grep -rniE "building in public|indie hacker" data/pipelines/x_session/
+
+# RIGHT — inverted: "found banned phrase" is the failure, "clean" exits 0
+if grep -rniE "building in public|indie hacker" data/pipelines/x_session/; then
+  echo "HALT: banned builder framing still present (see matches above)"; exit 1
+fi
+echo "clean: no banned framing"
+
+# ALSO RIGHT — grep -q with negation, when you don't need to see the matches
+grep -rqniE "building in public|indie hacker" data/pipelines/x_session/ \
+  && { echo "HALT: banned framing present"; exit 1; } || echo "clean"
+```
+
+**Rule:** Whenever an AC line is phrased as "verify there are no…", "confirm X does not
+appear", "no leftover…", write it as `if <detector>; then echo HALT…; exit 1; fi`. Never let
+the no-match/empty-output path be the one that carries a non-zero exit. Applies to `grep`,
+`rg`, `find ... | grep`, `diff` (exits 1 on differences), and any custom detector.
+
+---
+
+## §12 End green, and trust the verdict sentinel
+
+**Summary:** The post-run verifier (`runVerify.cjs`) scans the transcript and downgrades to
+`needs_review` on error markers (`Traceback`+`Error`, `FAIL`/`FATAL`, a tool `is_error` in the
+final portion of the run). It cannot tell an *intentional* failure from a real one. Two rules
+keep legitimate runs from false-tripping it.
+
+**12a — Run the green gate LAST.** Order the run so the final command is the acceptance/test
+gate. Do any intentionally-failing step EARLY:
+
+```bash
+# WRONG — red test reproduced AFTER the work; its Traceback lands late in the transcript
+pytest -q                      # all green
+python -m pytest tests/test_repro.py::test_bug   # ← TDD red demo, errors, trips verifier
+
+# RIGHT — red demo first (and captured), green gate last
+python -m pytest tests/test_repro.py::test_bug 2>&1 | tail -3 || true   # expected red, captured
+# ... implement the fix ...
+timeout 300 pytest -q          # ← LAST thing the run does; ends green
+```
+
+If you must show a failure late, capture it (`… 2>&1 | tail` inside a conditional, or assert
+on the captured text) so a raw `Traceback`/`is_error` never hits the transcript bare.
+
+**12b — The `SCHEDULER_VERDICT` sentinel is authoritative; emit it truthfully.** The scheduler's
+FINISH PROTOCOL ends by printing `SCHEDULER_VERDICT: PASS` once the AC gate is green AND the
+commit landed (else `SCHEDULER_VERDICT: FAIL <reason>` + `exit 1`). The verifier treats
+`PASS` + a commit landed during the run as the **authoritative** signal and overrides incidental
+transcript markers — this is what lets a deliberately-reproduced red test (PRD 77) or a grep
+result containing "Error" (PRD 68) finish `completed` instead of `needs_review`. **Never print
+`PASS` on a red gate.** The sentinel is only a safety net while it tells the truth; a lying
+`PASS` converts the verifier from "catches false failures" into "ships silent failures."
+
+**This actually happened** (PRDs 68 + 77, 2026-06-13): both committed correct work with green
+suites, but 77's `systematic-debugging` red-test repro and 68's grep-"Error" substring each
+tripped `transcript_errors → needs_review`, and the self-heal pass kept re-deriving the same
+verdict from the immutable log — stuck indefinitely. §12 (end-green + authoritative sentinel)
+is the structural fix.
+
+---
+
+## §13 Don't strand mid-run probe errors; annotate expected timeouts
+
+**Summary:** §12 keeps the *final* portion of the transcript green. §13 covers the *middle* —
+two executor habits that strand a bare `Traceback`/`Error`/`Exit code` the verifier then flags,
+even when the deliverable is correct and committed.
+
+**13a — A throwaway probe that errors must recover or be annotated in place.** Exploratory
+`python -c`/`bash` probes that error (a quoting/f-string slip, a wrong kwarg, a bad path) leave a
+bare traceback. Re-run the corrected probe immediately, or print `# expected/handled: <why>` on
+the next line, so recovery is adjacent (the heuristic looks for recovery within ~10 lines).
+Prefer a small temp `.py` file over a fragile multi-quote `python -c` one-liner — inline
+f-string/quoting errors are the top source of stranded probe tracebacks.
+
+```bash
+# WRONG — inline f-string slip strands a SyntaxError, then you move on
+python -c 'print(f"{p["title"]!r[:40]}")'        # SyntaxError, bare in transcript
+
+# RIGHT — write the probe to a temp file (no shell-quote minefield), or annotate
+cat > /tmp/probe.py <<'PY'
+print(repr(p["title"])[:40])
+PY
+python /tmp/probe.py || echo "# expected/handled: probe only, not part of the deliverable"
+```
+
+**13b — An *expected* `timeout` cap is success-with-note, not a bare `Exit code 124`.** Capping a
+genuinely long task you expect to hit the cap (a full-universe ingest, a long scan) is the
+correct §1/§8 behavior — but a bare `Exit code 124` reads as failure to the verifier. Branch on
+124 explicitly:
+
+```bash
+timeout 120 python -m project.ingest --all || { rc=$?
+  [ $rc -eq 124 ] && echo "hit time cap — idempotent/partial; rows persist incrementally; OK" \
+                  || { echo "HALT: ingest failed rc=$rc"; exit 1; }; }
+```
+
+For work that legitimately needs longer than a safe cap, run it in the background and poll a
+bounded number of times (§1) rather than capping the foreground command.
+
+**This actually happened** (PRDs 77 + 80, 2026-06-13): 77 stranded an inline-`python -c` f-string
+`SyntaxError` from a throwaway permalink probe; 80 surfaced a bare `Exit code 124` from a
+`timeout`-capped full-universe EDGAR ingest. Both committed correct, green, AC-complete work
+(77's cursor-hold fix; 80's 6 EDGAR rows + installed cron) yet were downgraded to `needs_review`
+on the incidental middle-of-run markers. 13a/13b keep the middle of the transcript clean.

package/src/preload/api.d.ts

@@ -533,16 +533,17 @@ export interface RepoAnalyzeResult {
 }
 export interface RepoAnalyzeError { ok: false; error: string }
 
-export interface HiveRole { label: string; prompt: string }
-export interface Hive {
+export interface RecipeStep { agentName: string; note?: string }
+export interface Recipe {
   slug: string;
   name: string;
   description: string;
-  roles: HiveRole[];
-  defaultPlan?: string;
+  brief?: string;
+  steps: RecipeStep[];
 }
-export interface HiveListResult { hives: Hive[]; error: string | null }
-export interface HiveGetResult { hive: Hive | null; error: string | null }
+// Channel keeps legacy "hives" name; concept is now "recipe".
+export interface HiveListResult { hives: Recipe[]; error: string | null }
+export interface HiveGetResult { hive: Recipe | null; error: string | null }
 export interface HiveMutationResult { ok: boolean; error: string | null }
 
 export interface WatcherInfo {
@@ -975,7 +976,7 @@ export interface SessionManagerAPI {
   hives: {
     list: () => Promise<HiveListResult>;
     get: (slug: string) => Promise<HiveGetResult>;
-    save: (slug: string, hive: Hive) => Promise<HiveMutationResult>;
+    save: (slug: string, hive: Recipe) => Promise<HiveMutationResult>;
     delete: (slug: string) => Promise<HiveMutationResult>;
   };
   history: {