@stigmer/runner 3.1.3 → 3.1.4

package/src/shared/filereview/progress.ts

@@ -1,8 +1,7 @@
 /**
- * Mid-run live capture — the harness-agnostic glue that turns a content-free git
- * delta ({@link ./git-substrate.js} `captureProgressDelta`) into the transient
- * `AgentExecutionStatus.file_change_progress` snapshot the "N files changed so
- * far" strip renders (DD-32).
+ * Mid-run live capture — the harness-agnostic glue that turns a per-turn
+ * workspace delta into the transient `AgentExecutionStatus.file_change_progress`
+ * snapshot the "N files changed so far" strip renders (DD-32 / DD-33).
  *
  * THE MODEL
  * ---------
@@ -13,24 +12,33 @@
  * `file_change_sets`) remains the single authoritative, reviewable diff; a mid-run
  * snapshot is no more authoritative than a streamed tool-call arg.
  *
- * Both harnesses drive this through {@link captureFileChangeProgress} from their
- * per-persist loop, each passing its OWN `excludePaths` (the runner-owned gate
- * files it writes into the workspace), so progress and the turn-boundary candidate
- * agree on what is "the agent's change". Capture is throttled by a floor +
- * tree-sha short-circuit so a quiet turn is nearly free.
+ * THE SUBSTRATE ABSTRACTION (DD-33)
+ * ---------------------------------
+ * Where the delta comes from differs by workspace: a git tree diffs cheaply with
+ * `git --numstat` (the git substrate), a non-git / gitignored workspace reads the
+ * CAS observer (the cas substrate, {@link ./cas-progress.js}), and a git tree
+ * with gitignored writes composes both (the hybrid substrate). A
+ * {@link ProgressSubstrate} hides that behind one `capture()` so the floor + attach
+ * logic below is written once. Each substrate is a per-turn object owning its own
+ * short-circuit cache; `capture()` always returns the FULL cumulative turn delta
+ * plus a `changed` flag, so the hybrid can merge both slices even when only one
+ * moved (a `ProgressDelta | undefined` would drop the unchanged slice).
  *
  * SECRET SAFETY
  * -------------
  * No file bodies are ever carried. A secret-like path ({@link isSecretLikePath})
  * is still surfaced (path visible) but with its line counts ZEROED — the same
- * "path visible, content withheld" rule the ledger uses (DD-12). Nothing new can
- * leak through this field.
+ * "path visible, content withheld" rule the ledger uses (DD-12). The cas
+ * substrate additionally excludes gate-blocked secrets up front
+ * ({@link partitionIgnoredPathsBySecret}); this zeroing is the belt-and-suspenders
+ * backstop. Nothing new can leak through this field.
  *
- * @since File-Change HITL Redesign (mid-run live capture / DD-32)
+ * @since File-Change HITL Redesign (mid-run live capture / DD-32; non-git + hybrid / DD-33)
  */
 
 import { create } from "@bufbuild/protobuf";
 import type { AgentExecutionStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/api_pb";
+import type { FileChangeKind } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import type {
   FileChangeProgress,
   FileChangeProgressEntry,
@@ -41,22 +49,74 @@ import {
 } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/filereview_pb";
 import { utcTimestamp } from "../status.js";
 import { toFileChangeKind } from "./capture.js";
-import { captureProgressDelta, type ProgressDelta } from "./git-substrate.js";
+import {
+  captureProgressDelta,
+  type GitProgressEntry,
+} from "./git-substrate.js";
 import { isSecretLikePath } from "./secret-paths.js";
 
+/**
+ * One file's slim, content-free progress row — the substrate-neutral input to
+ * {@link buildFileChangeProgress}. Carries the proto {@link FileChangeKind}
+ * directly (git and cas substrates both normalize to it), so no side re-maps.
+ * `linesAdded`/`linesRemoved` are 0 when uncountable (binary / oversized /
+ * secret-withheld).
+ */
+export interface ProgressEntry {
+  /** Path before the change (workspace-relative). Empty for ADD. */
+  readonly pathBefore: string;
+  /** Path after the change (workspace-relative). Empty for DELETE. */
+  readonly pathAfter: string;
+  readonly kind: FileChangeKind;
+  readonly linesAdded: number;
+  readonly linesRemoved: number;
+}
+
+/**
+ * The substrate-neutral progress delta the git/cas/hybrid substrates emit.
+ *
+ * `totalFilesChanged` is the honest count of changed files this turn, which may
+ * EXCEED `entries.length` when a substrate caps how many files it reads (the cas
+ * substrate reads only a bounded prefix — DD-33). Undefined means "the entries
+ * ARE every changed file" (the git substrate, whose numstat is free), so
+ * {@link buildFileChangeProgress} falls back to `entries.length`.
+ */
+export interface ProgressDelta {
+  readonly entries: readonly ProgressEntry[];
+  readonly totalFilesChanged?: number;
+}
+
+/** One mid-run capture: the FULL cumulative turn delta + whether it changed. */
+export interface ProgressCapture {
+  readonly delta: ProgressDelta;
+  /** False ⇒ nothing moved since the last capture; the caller skips re-attach. */
+  readonly changed: boolean;
+}
+
+/**
+ * A per-turn source of the mid-run delta. Implementations own their own
+ * short-circuit cache and ALWAYS return the full cumulative delta (never a
+ * bare "unchanged" sentinel), so {@link createHybridProgressSubstrate} can merge
+ * a changed slice with an unchanged one without losing the latter.
+ */
+export interface ProgressSubstrate {
+  capture(): Promise<ProgressCapture>;
+}
+
 /**
  * Cap the per-file entry list so a turn touching thousands of files does not
  * bloat the persisted status (which rides Temporal / the store). `files_changed`
  * and the aggregate counts stay honest over ALL files; the strip shows "… and K
- * more" when the list is capped.
+ * more" when the list is capped. Also the cas substrate's read budget — it reads
+ * after-bytes for at most this many files per capture (DD-33).
  */
 export const PROGRESS_MAX_ENTRIES = 200;
 
 /**
  * Minimum wall-clock gap between two mid-run captures, in ms. A floor on cost:
- * every capture stages the working tree (`write-tree`), so we bound how often
- * that runs regardless of persist frequency. Env-tunable for large-repo
- * deployments; defaults to 2s (feels live without hammering git).
+ * every capture stages the working tree (git) or reads the touched set (cas), so
+ * we bound how often that runs regardless of persist frequency. Env-tunable for
+ * large-repo deployments; defaults to 2s (feels live without hammering the disk).
  */
 export const PROGRESS_CAPTURE_MIN_INTERVAL_MS = readMinIntervalMs();
 
@@ -81,23 +141,22 @@ export function shouldCaptureProgress(
 }
 
 /**
- * Build the transient {@link FileChangeProgress} message from a content-free git
+ * Build the transient {@link FileChangeProgress} message from a substrate-neutral
  * delta. Zeroes counts for secret-like paths (path visible, magnitude withheld),
- * caps the entry list at {@link PROGRESS_MAX_ENTRIES} while keeping
- * `files_changed`/aggregate counts honest over every file, and stamps
- * `captured_at`. Pure and exported for direct testing.
+ * caps the entry list at {@link PROGRESS_MAX_ENTRIES}, and reports the honest
+ * `files_changed` (`delta.totalFilesChanged` when a substrate capped its reads,
+ * else the entry count). Aggregate counts sum over the emitted entries. Pure and
+ * exported for direct testing.
  */
 export function buildFileChangeProgress(
   delta: ProgressDelta,
   changeSetId: string,
 ): FileChangeProgress {
-  let filesChanged = 0;
   let totalAdded = 0;
   let totalRemoved = 0;
   const entries: FileChangeProgressEntry[] = [];
 
   for (const entry of delta.entries) {
-    filesChanged += 1;
     // A secret-like path is surfaced (path visible) but its magnitude is withheld
     // — counts zeroed, mirroring the ledger's "path visible, content withheld".
     const secret = isSecretLikePath(entry.pathAfter || entry.pathBefore);
@@ -111,7 +170,7 @@ export function buildFileChangeProgress(
         create(FileChangeProgressEntrySchema, {
           pathBefore: entry.pathBefore,
           pathAfter: entry.pathAfter,
-          kind: toFileChangeKind(entry.changeType),
+          kind: entry.kind,
           linesAdded,
           linesRemoved,
         }),
@@ -121,7 +180,7 @@ export function buildFileChangeProgress(
 
   return create(FileChangeProgressSchema, {
     changeSetId,
-    filesChanged,
+    filesChanged: delta.totalFilesChanged ?? delta.entries.length,
     linesAdded: totalAdded,
     linesRemoved: totalRemoved,
     entries,
@@ -130,12 +189,12 @@ export function buildFileChangeProgress(
 }
 
 /**
- * Mutable per-turn state the caller threads across persists: the last captured
- * tree sha (for the short-circuit) and the last capture time (for the floor).
- * A fresh turn starts a fresh state (`{ lastAtMs: 0 }`).
+ * Mutable per-turn state the caller threads across persists: the last capture
+ * time (the floor). Substrate-specific short-circuit state (git tree sha, cas
+ * signature) lives inside the {@link ProgressSubstrate}, not here. A fresh turn
+ * starts a fresh state (`{ lastAtMs: 0 }`).
  */
 export interface ProgressCaptureState {
-  lastTreeSha?: string;
   lastAtMs: number;
 }
 
@@ -146,11 +205,10 @@ export function newProgressCaptureState(): ProgressCaptureState {
 
 /**
  * Capture-and-attach the live progress snapshot onto `status.file_change_progress`,
- * throttled by the floor + tree-sha short-circuit. Mutates `status` and `state`
- * in place; a no-op when the floor has not elapsed or the working tree is
- * unchanged since the last capture. Content-free and secret-safe (see
- * {@link buildFileChangeProgress}). Called by both harnesses from their per-persist
- * loop; each passes its own `excludePaths`.
+ * throttled by the floor. Mutates `status` and `state` in place; a no-op when the
+ * floor has not elapsed or the substrate reports nothing changed. Content-free and
+ * secret-safe (see {@link buildFileChangeProgress}). Called by both harnesses from
+ * their per-persist loop, each passing a substrate built for its workspace shape.
  *
  * The field is set even when the delta is now EMPTY (the agent reverted its own
  * edits) so the strip reflects the reversion (it hides at zero) rather than
@@ -159,11 +217,8 @@ export function newProgressCaptureState(): ProgressCaptureState {
  */
 export async function captureFileChangeProgress(opts: {
   readonly status: AgentExecutionStatus;
-  readonly gitRoot: string;
-  readonly executionId: string;
   readonly changeSetId: string;
-  readonly baselineTree: string;
-  readonly excludePaths?: readonly string[];
+  readonly substrate: ProgressSubstrate;
   readonly state: ProgressCaptureState;
   /** Injectable clock for tests; defaults to `Date.now()`. */
   readonly nowMs?: number;
@@ -172,15 +227,84 @@ export async function captureFileChangeProgress(opts: {
   if (!shouldCaptureProgress(opts.state.lastAtMs, now)) return;
   opts.state.lastAtMs = now;
 
-  const delta = await captureProgressDelta(
-    opts.gitRoot,
-    opts.executionId,
-    opts.baselineTree,
-    opts.excludePaths,
-    opts.state.lastTreeSha,
-  );
-  if (!delta) return; // working tree unchanged since the last capture
+  const { delta, changed } = await opts.substrate.capture();
+  if (!changed) return;
 
-  opts.state.lastTreeSha = delta.afterTree;
   opts.status.fileChangeProgress = buildFileChangeProgress(delta, opts.changeSetId);
 }
+
+// ---------------------------------------------------------------------------
+// Substrates
+// ---------------------------------------------------------------------------
+
+function gitEntryToProgressEntry(e: GitProgressEntry): ProgressEntry {
+  return {
+    pathBefore: e.pathBefore,
+    pathAfter: e.pathAfter,
+    kind: toFileChangeKind(e.changeType),
+    linesAdded: e.linesAdded,
+    linesRemoved: e.linesRemoved,
+  };
+}
+
+/**
+ * The git substrate: the working-tree `--numstat` delta against the pinned
+ * baseline. `git add -A` honors `.gitignore`, so this covers exactly the
+ * git-TRACKED changes (from any source — tool or shell), disjoint from the cas
+ * substrate's gitignored set. Preserves the tree-sha short-circuit: on an
+ * unchanged tree it returns the cached full delta with `changed:false` (the
+ * caller skips the re-attach), byte-identical to the pre-DD-33 behavior.
+ */
+export function createGitProgressSubstrate(opts: {
+  readonly workspaceRoot: string;
+  readonly executionId: string;
+  readonly baselineTree: string;
+  readonly excludePaths?: readonly string[];
+}): ProgressSubstrate {
+  let lastTreeSha: string | undefined;
+  let cachedFull: ProgressDelta = { entries: [] };
+  return {
+    async capture(): Promise<ProgressCapture> {
+      const gitDelta = await captureProgressDelta(
+        opts.workspaceRoot,
+        opts.executionId,
+        opts.baselineTree,
+        opts.excludePaths,
+        lastTreeSha,
+      );
+      if (gitDelta === undefined) {
+        // Working tree unchanged since the last capture — reuse the cache so the
+        // hybrid can still merge this (unchanged) slice with a changed cas slice.
+        return { delta: cachedFull, changed: false };
+      }
+      lastTreeSha = gitDelta.afterTree;
+      cachedFull = { entries: gitDelta.entries.map(gitEntryToProgressEntry) };
+      return { delta: cachedFull, changed: true };
+    },
+  };
+}
+
+/**
+ * The hybrid substrate: a git tree whose gitignored writes are captured via CAS.
+ * Concatenates both slices (disjoint by the `.gitignore` boundary — numstat sees
+ * only tracked paths, the observer only gitignored ones) and sums the honest
+ * totals. `changed` is true when EITHER slice moved; because each child returns
+ * its full cumulative delta, the merged delta always carries both slices.
+ */
+export function createHybridProgressSubstrate(
+  git: ProgressSubstrate,
+  cas: ProgressSubstrate,
+): ProgressSubstrate {
+  return {
+    async capture(): Promise<ProgressCapture> {
+      const [g, c] = await Promise.all([git.capture(), cas.capture()]);
+      const delta: ProgressDelta = {
+        entries: [...g.delta.entries, ...c.delta.entries],
+        totalFilesChanged:
+          (g.delta.totalFilesChanged ?? g.delta.entries.length) +
+          (c.delta.totalFilesChanged ?? c.delta.entries.length),
+      };
+      return { delta, changed: g.changed || c.changed };
+    },
+  };
+}

package/src/shared/plan-mode-prompt.ts

@@ -29,6 +29,15 @@
  * message is published verbatim as a plan document (whose filename is derived
  * from the leading `#` title), so a fenced or chat-suffixed plan degrades the
  * reviewable document the user sees.
+ *
+ * Fence hygiene inside the document is part of the same contract. A plan
+ * frequently quotes file content that itself contains fenced code blocks
+ * ("insert this section into README.md"); with a same-length outer fence the
+ * inner block's closer terminates the outer fence early and corrupts the
+ * rendered document — on every client (the react SDK and the terminal
+ * renderer alike). And because the plan viewers render top-level ```mermaid
+ * fences as diagrams, the directive steers diagrams into the plan body proper
+ * rather than leaving them buried, unrendered, inside quoted file content.
  */
 export const PLAN_MODE_DIRECTIVE = [
   "IMPORTANT: You are in Plan mode — a read-only analysis turn whose " +
@@ -50,6 +59,15 @@ export const PLAN_MODE_DIRECTIVE = [
   "- Reference concrete file paths and describe the specific changes " +
     "planned for each.",
   "- Do NOT wrap the document in a code fence.",
+  "- When quoting content that itself contains fenced code blocks (e.g. a " +
+    "proposed file section with a code sample inside), open the outer fence " +
+    "with MORE backticks than any inner fence (four or more) — a same-length " +
+    "inner closer would terminate the outer fence early and corrupt the " +
+    "rendered document.",
+  "- Fenced ```mermaid blocks at the top level of the document render as " +
+    "diagrams in the plan viewer. When a diagram helps communicate the " +
+    "design (architecture, flows), include it directly in the plan body — " +
+    "not only inside quoted file content, where it stays unrendered source.",
   '- Do NOT end with conversational closers ("Let me know...", "Shall I ' +
     'proceed?") — the next step is the user\'s Build action, and trailing ' +
     "chat would be published as part of the document.",

package/src/shared/todos.ts

@@ -0,0 +1,126 @@
+/**
+ * Todo-list projection shared by every harness's status writer.
+ *
+ * All harnesses expose an agent to-do tool — the Cursor SDK emits `TodoWrite`
+ * (legacy) / `updateTodos` (current), the native deepagents harness emits
+ * `write_todos` — and each writes the SAME `AgentExecutionStatus.todos` proto
+ * map that the clients (React `TodoCard`, CLI) render. This module is the single
+ * place that maps a raw tool payload into that map, so the Cursor `TodoTracker`
+ * and the native v2/v3 status builders cannot drift in how they build todos.
+ *
+ * The mapping is a superset of what any one harness emits: the Cursor SDK sends
+ * per-item `id`/`created_at` and can send a `merge` flag and camelCase statuses;
+ * deepagents sends only `{ content, status }` (three statuses, no id, no merge).
+ * One helper handling the superset serves all three writers.
+ */
+
+import { create } from "@bufbuild/protobuf";
+import { TodoItemSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/todo_pb";
+import type { TodoItem } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/todo_pb";
+import { TodoStatus } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+import { utcTimestamp } from "./status.js";
+
+/**
+ * A single raw todo as it arrives in a to-do tool call's arguments. Every field
+ * is optional and defensively typed: the Cursor `updateTodos` schema omits
+ * per-item `id`, deepagents omits `id`/`created_at`, and the values reach us as
+ * untyped JSON (a Cursor `args` string or a protobuf `JsonObject` value).
+ */
+export interface RawTodoItem {
+  id?: string;
+  content?: string;
+  status?: string;
+  created_at?: string;
+}
+
+/**
+ * Raw status string -> proto `TodoStatus`. Keyed on the lowercased status so
+ * the current SDK's camelCase `inProgress` and the legacy snake_case
+ * `in_progress` both resolve. `cancelled` is only ever emitted by the Cursor
+ * SDK — deepagents' three-status schema never sends it — but it is kept here so
+ * the one shared mapping is faithful to every producer. Unknown values fall back
+ * to `TODO_PENDING` at the call site.
+ */
+const STATUS_MAP: Record<string, TodoStatus> = {
+  pending: TodoStatus.TODO_PENDING,
+  in_progress: TodoStatus.TODO_IN_PROGRESS,
+  inprogress: TodoStatus.TODO_IN_PROGRESS,
+  completed: TodoStatus.TODO_COMPLETED,
+  cancelled: TodoStatus.TODO_CANCELLED,
+};
+
+/**
+ * Project a to-do tool call's payload into a `status.todos` map, in place.
+ * Returns whether the map changed — the caller's "dirty" / force-persist signal.
+ *
+ * Semantics (identical across all three harness writers):
+ *   - Empty or non-array `rawTodos`: a full-replace (`!merge`) clears the map and
+ *     is a change; a merge is a no-op (nothing to merge in) and is not.
+ *   - Otherwise a full-replace clears first, then every item is (re)written.
+ *   - `id` is the provided id or a stable index fallback (`todo-<i>`); order is
+ *     the array order, so index ids are stable across full-replace updates.
+ *   - `created_at` is preserved from the prior same-id entry ONLY on merge; a
+ *     full-replace clears first, so it becomes `raw.created_at || now`. (The
+ *     native harness is always full-replace and sends no `created_at`, so a
+ *     native todo's `created_at` is `now` on each write — deliberately identical
+ *     to the Cursor non-merge path.)
+ *   - `updated_at` is always `now`.
+ *
+ * Note on clearing: an empty full-replace clears the in-memory map, but the Go
+ * `update_status` activity replaces persisted todos only when the incoming map
+ * is non-empty, so a settled list survives in history — the clear never reaches
+ * the record. This is intentional and relied upon by both harnesses.
+ */
+export function applyTodoUpdate(
+  target: { [key: string]: TodoItem },
+  rawTodos: unknown,
+  opts: { merge: boolean; now?: string },
+): boolean {
+  const { merge } = opts;
+  const now = opts.now ?? utcTimestamp();
+
+  if (!Array.isArray(rawTodos) || rawTodos.length === 0) {
+    if (!merge) {
+      clearMap(target);
+      return true;
+    }
+    return false;
+  }
+
+  if (!merge) {
+    clearMap(target);
+  }
+
+  for (let i = 0; i < rawTodos.length; i++) {
+    const raw = coerceRawTodo(rawTodos[i]);
+    const id = raw.id || `todo-${i}`;
+    const statusStr = (raw.status ?? "pending").toLowerCase();
+    const status = STATUS_MAP[statusStr] ?? TodoStatus.TODO_PENDING;
+
+    // Preserve the original creation time only when merging into an existing
+    // entry; a full-replace cleared the map above, so `existing` is undefined.
+    const existing = merge ? target[id] : undefined;
+
+    target[id] = create(TodoItemSchema, {
+      id,
+      content: raw.content ?? "",
+      status,
+      createdAt: existing?.createdAt || raw.created_at || now,
+      updatedAt: now,
+    });
+  }
+
+  return true;
+}
+
+/** Narrow an untyped array element to a {@link RawTodoItem}; non-objects become empty. */
+function coerceRawTodo(value: unknown): RawTodoItem {
+  return typeof value === "object" && value !== null ? (value as RawTodoItem) : {};
+}
+
+/** Delete every key from the map in place (proto map fields have no `.clear()`). */
+function clearMap(target: { [key: string]: TodoItem }): void {
+  for (const key of Object.keys(target)) {
+    delete target[key];
+  }
+}

package/src/shared/tool-row.ts

@@ -186,3 +186,37 @@ export function collectSubAgentToolCallIds(
   }
   return ids;
 }
+
+/** Terminal tool-call statuses — a row that has finished (however it finished). */
+const TERMINAL_TOOL_CALL_STATUSES: ReadonlySet<ToolCallStatus> = new Set([
+  ToolCallStatus.TOOL_CALL_COMPLETED,
+  ToolCallStatus.TOOL_CALL_FAILED,
+  ToolCallStatus.TOOL_CALL_SKIPPED,
+]);
+
+/**
+ * Collect the ids of tool-call rows that have already SETTLED (reached a terminal
+ * state: completed, failed, or skipped) in a transcript.
+ *
+ * The deep-agent turn-boundary provenance derivation (DD-28) snapshots this
+ * BEFORE a turn's stream to scope "this turn's tool calls" by identity: a call
+ * whose id is absent from the snapshot is this-turn's — either freshly streamed,
+ * or a prior gate that was WAITING_APPROVAL before the stream and executes now on
+ * resume (StatusBuilder updates the seeded row in place, so it keeps its id).
+ * Unlike the Cursor harness, the deep-agent cannot scope positionally — an
+ * approved command executes at its SEEDED position — so identity is the only
+ * correct scope, mirroring {@link collectSubAgentToolCallIds}. Terminal (not just
+ * completed) so a PRIOR turn's failed/skipped non-shell call is excluded and
+ * cannot spuriously disqualify this turn.
+ */
+export function collectSettledToolCallIds(
+  messages: readonly AgentMessage[],
+): Set<string> {
+  const ids = new Set<string>();
+  for (const msg of messages) {
+    for (const tc of msg.toolCalls) {
+      if (TERMINAL_TOOL_CALL_STATUSES.has(tc.status)) ids.add(tc.id);
+    }
+  }
+  return ids;
+}