@stigmer/runner 3.1.2 → 3.1.4

package/src/shared/filereview/command-provenance.ts

@@ -0,0 +1,180 @@
+/**
+ * The DD-28 approved-command turn qualification rule — the harness-agnostic core
+ * of the auto-keep policy's runner-side facts.
+ *
+ * A turn whose ONLY mutation source was shell commands the human had already
+ * authorized should not re-gate its file effects at the turn-boundary review —
+ * the user consented to the command, and the command's file effects are the
+ * consented outcome. The server cannot derive this itself (tool calls carry no
+ * turn marker), so the runner — the only component that owns turn scoping —
+ * derives the facts and attaches them to the CANDIDATE_CAPTURED event as
+ * {@link TurnCommandProvenance}.
+ *
+ * This module owns the *qualification rule* both harnesses share; each harness
+ * supplies only what genuinely differs — how "this turn's tool calls" are scoped,
+ * and how a single executed command's DIRECT consent row is identified — via
+ * {@link CommandProvenanceCoreInputs}. Keeping the rule in one place is what makes
+ * the Cursor and deep-agent qualifications provably identical (a second copy would
+ * drift on edge cases — the divergence the AI-Engineer role warns against).
+ *
+ * TRUST BOUNDARY. This module asserts turn FACTS (which tools ran this turn),
+ * the same trust level as the captured bytes themselves. It never asserts
+ * CONSENT: a resolved consent id merely POINTS at a transcript row whose
+ * `approval_action` was authored by the server's SubmitApproval (and is
+ * preserved against runner writes) — the backend re-verifies every claimed row
+ * against that server-owned record before authoring the policy decision, so a
+ * runner cannot mint authorization it was never given.
+ *
+ * FAIL-CLOSED. Every uncertainty disqualifies (returns undefined → the set
+ * reviews manually, exactly as before DD-28): a file-tool call, an MCP tool, a
+ * sub-agent delegation, an unrecognized tool name, or an executed shell command
+ * with no provable consent source. Being conservative here costs only an extra
+ * review; being permissive would silently waive one.
+ */
+
+import { create } from "@bufbuild/protobuf";
+import type { AgentMessage, ToolCall } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/message_pb";
+import { TurnCommandProvenanceSchema } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/filereview_pb";
+import type { TurnCommandProvenance } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/filereview_pb";
+import { ApprovalAction, ToolKind } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
+import { classifyTool, toolApprovalCategory } from "../tool-kind.js";
+import { isToolCallRowHidden } from "../tool-row.js";
+
+/**
+ * Tool kinds that can never mutate the workspace: their presence in a turn is
+ * irrelevant to change-set provenance. Everything OUTSIDE this set is either a
+ * consent-mapped shell command or a disqualifier — never silently ignored.
+ */
+export const NON_MUTATING_KINDS: ReadonlySet<ToolKind> = new Set([
+  ToolKind.FILE_READ,
+  ToolKind.SEARCH,
+  ToolKind.LIST,
+  ToolKind.FETCH,
+  ToolKind.WEB_SEARCH,
+  ToolKind.THINK,
+  ToolKind.TODO,
+]);
+
+/**
+ * The harness-supplied inputs to the shared DD-28 qualification rule. The two
+ * function inputs are exactly the two things that differ between harnesses
+ * (turn scoping is expressed by which calls appear in {@link turnToolCalls}).
+ */
+export interface CommandProvenanceCoreInputs {
+  /**
+   * This turn's tool calls, already scoped by the harness (Cursor: the positional
+   * slice from the turn's first streamed message; deep-agent: the calls whose id
+   * is new since a pre-stream settled snapshot). Order is preserved but irrelevant.
+   */
+  readonly turnToolCalls: readonly ToolCall[];
+  /**
+   * The WHOLE transcript, for the APPROVE_ALL lease lookup — a lease is
+   * run-lifetime and its granting row usually lives in a prior turn.
+   */
+  readonly messages: readonly AgentMessage[];
+  /**
+   * Whether a (shell) tool call actually EXECUTED this turn, as opposed to being
+   * a pending/denied gate that produced no mutation. A non-executed shell neither
+   * needs consent nor disqualifies (Cursor: not in the denial ledger; deep-agent:
+   * status is COMPLETED).
+   */
+  readonly isExecutedCommand: (tc: ToolCall) => boolean;
+  /**
+   * The tool-call id of the row whose server-authored `approval_action` DIRECTLY
+   * authorized this executed command, or undefined when none does (then the lease
+   * / global-bypass fallbacks apply). Cursor maps a reinvocation grant token back
+   * to its consent row; the deep-agent's gated command IS its own consent row
+   * (same id, `approval_action` written in place).
+   */
+  readonly resolveDirectConsent: (tc: ToolCall) => string | undefined;
+  /** True when the pre-armed spec.auto_approve_all bypassed the gate. */
+  readonly globalBypass: boolean;
+}
+
+/**
+ * Apply the DD-28 qualification rule to a harness-scoped turn, returning the
+ * {@link TurnCommandProvenance} to attach to the candidate — or undefined when
+ * the turn does not qualify (any non-shell mutation-capable call, any unknown
+ * tool, any executed command without a provable consent source, or no executed
+ * consented command at all).
+ */
+export function qualifyTurnCommandProvenance(
+  inputs: CommandProvenanceCoreInputs,
+): TurnCommandProvenance | undefined {
+  const { turnToolCalls, messages, isExecutedCommand, resolveDirectConsent, globalBypass } = inputs;
+
+  const consentIds = new Set<string>();
+  let authorizedByAutoApproveAll = false;
+  let executedCommandCount = 0;
+
+  for (const tc of turnToolCalls) {
+    // A hidden row is a collapsed twin/reaction — it never executed.
+    if (isToolCallRowHidden(tc)) continue;
+
+    const kind = classifyTool(tc.name, tc.mcpServerSlug);
+    if (NON_MUTATING_KINDS.has(kind)) continue;
+
+    if (kind !== ToolKind.SHELL) {
+      // A file tool (mixed turn), a sub-agent (unattributable mutations), an
+      // MCP tool (opaque side effects), or an unrecognized name — the turn's
+      // mutations cannot be attributed to consented commands. Fail closed.
+      return undefined;
+    }
+
+    if (!isExecutedCommand(tc)) {
+      // A pending / denied shell — this turn's gate, not a mutation source.
+      continue;
+    }
+    executedCommandCount++;
+
+    const consentId = resolveDirectConsent(tc);
+    if (consentId) {
+      consentIds.add(consentId);
+      continue;
+    }
+    const leaseConsentId = findLeaseConsentId(messages, tc.name);
+    if (leaseConsentId) {
+      consentIds.add(leaseConsentId);
+      continue;
+    }
+    if (globalBypass) {
+      authorizedByAutoApproveAll = true;
+      continue;
+    }
+    // An executed shell command with no grant, no lease, and no bypass should
+    // be impossible (every un-consented shell is gated); if it ever happens,
+    // the honest answer is a manual review, not a waived one.
+    return undefined;
+  }
+
+  // A turn that executed no consented command has nothing to attribute the
+  // change set to — whatever changed came from somewhere else. Manual review.
+  if (executedCommandCount === 0) return undefined;
+
+  return create(TurnCommandProvenanceSchema, {
+    consentToolCallIds: [...consentIds],
+    authorizedByAutoApproveAll,
+  });
+}
+
+/**
+ * Find the transcript row whose APPROVE_ALL authored the run-lifetime lease
+ * covering `toolName`'s category — the consent row a lease-executed command
+ * cites. Searched across the whole transcript (a lease usually originates in a
+ * prior turn); the first (earliest) APPROVE_ALL of the category is the lease's
+ * origin. Returns undefined when no lease row exists for the category.
+ */
+export function findLeaseConsentId(
+  messages: readonly AgentMessage[],
+  toolName: string,
+): string | undefined {
+  const category = toolApprovalCategory(toolName);
+  if (!category) return undefined;
+  for (const msg of messages) {
+    for (const tc of msg.toolCalls) {
+      if (tc.approvalAction !== ApprovalAction.APPROVE_ALL) continue;
+      if (toolApprovalCategory(tc.name) === category) return tc.id;
+    }
+  }
+  return undefined;
+}

package/src/shared/filereview/events.ts

@@ -42,6 +42,7 @@ import {
   FileReviewFailureKind,
 } from "@stigmer/protos/ai/stigmer/agentic/agentexecution/v1/enum_pb";
 import { aggregateDigest, fileDigest, sha256Bytes } from "./digest.js";
+import { countLineChanges, type LineChangeCounts } from "./line-counts.js";
 import { looksBinary } from "../file-change.js";
 
 // The runner is the author of capture/reconcile/failure events. "runner" (not
@@ -150,6 +151,15 @@ export interface CapturedChangeInput {
    * this UNSPECIFIED. See {@link FileReviewBlockReason}.
    */
   readonly blockedReason?: FileReviewBlockReason;
+  /**
+   * Display line counts for this change, for producers whose bodies are not
+   * inline here (the CAS substrate counts from the in-memory bytes before
+   * offloading them as blob refs). Omit for inline content —
+   * {@link buildCapturedFileChange} then counts the inline sides itself, with
+   * the same shared {@link countLineChanges}. Informational only, never folded
+   * into the digests.
+   */
+  readonly lineCounts?: LineChangeCounts;
 }
 
 /** Normalize the string-shorthand to a {@link CapturedContent}. */
@@ -175,6 +185,37 @@ export function contentSha256(content: CapturedContent): string {
   }
 }
 
+/**
+ * Count the display `+N −M` from a change's inline sides, or `undefined` when
+ * counting is not honest here: a side that exists without inline text (a `ref`
+ * whose bytes were already offloaded, or a `binary` side with no body), or
+ * inline text that would render as binary (the UI shows "Binary file changed"
+ * for it, never a line diff — a count would describe a diff nobody sees).
+ * An absent side is fine — it is the empty document of an ADD/DELETE.
+ */
+function countInlineSides(
+  before: CapturedContent | undefined,
+  after: CapturedContent | undefined,
+): LineChangeCounts | undefined {
+  const beforeText = inlineTextOrNull(before);
+  const afterText = inlineTextOrNull(after);
+  if (beforeText === null || afterText === null) return undefined;
+  return countLineChanges(beforeText, afterText);
+}
+
+/**
+ * A side's countable inline text: the text for a non-binary inline side,
+ * `undefined` for an absent side (countable as the empty document), and `null`
+ * when the side exists but cannot be counted (ref / binary / binary-looking).
+ */
+function inlineTextOrNull(
+  content: CapturedContent | undefined,
+): string | undefined | null {
+  if (content === undefined) return undefined;
+  if (content.kind !== "inline" || looksBinary(content.text)) return null;
+  return content.text;
+}
+
 /** Build the proto {@link FileContent} for one content side (inline, ref, or binary). */
 function toFileContent(content: CapturedContent): FileContent {
   if (content.kind === "inline") {
@@ -212,12 +253,19 @@ function toFileContent(content: CapturedContent): FileContent {
  * `file_digest`) over the captured bytes. Content may be inline (git) or a blob
  * ref (CAS) — the digests are identical either way, so the aggregate digest and
  * the reconcile enforcement compose across both substrates.
+ *
+ * Display line counts are stamped here too — the single seam every substrate
+ * flows through — from `input.lineCounts` when the producer counted at its own
+ * source (CAS), else counted from the inline sides. Counting happens BEFORE any
+ * persist-time offload, so the counts survive a body being elided later. They
+ * never enter `fileDigest` (informational, not enforcement).
  */
 export function buildCapturedFileChange(input: CapturedChangeInput): CapturedFileChange {
   const before = normalizeContent(input.before);
   const after = normalizeContent(input.after);
   const beforeSha256 = before ? contentSha256(before) : "";
   const afterSha256 = after ? contentSha256(after) : "";
+  const counts = input.lineCounts ?? countInlineSides(before, after);
 
   const fc = create(CapturedFileChangeSchema, {
     id: input.id,
@@ -229,6 +277,8 @@ export function buildCapturedFileChange(input: CapturedChangeInput): CapturedFil
     afterSha256,
     diffComplete: input.diffComplete ?? true,
     blockedReason: input.blockedReason ?? FileReviewBlockReason.UNSPECIFIED,
+    linesAdded: counts?.linesAdded ?? 0,
+    linesRemoved: counts?.linesRemoved ?? 0,
     fileDigest: fileDigest({
       pathBefore: input.pathBefore,
       pathAfter: input.pathAfter,

package/src/shared/filereview/git-substrate.ts

@@ -308,6 +308,87 @@ export async function captureChangeSet(
   return { baselineTree, afterTree, changes };
 }
 
+/**
+ * One file's slim, content-free progress delta: paths + kind + line counts, NO
+ * bytes. The mid-run analogue of {@link GitSubstrateChange} (which carries the
+ * before/after {@link CapturedContent}); this deliberately carries neither side,
+ * so it can never leak content. `linesAdded`/`linesRemoved` are 0 when
+ * uncountable (a binary file — numstat reports `-`).
+ */
+export interface GitProgressEntry {
+  /** Path before the change (repo-relative). Empty for a CREATE. */
+  readonly pathBefore: string;
+  /** Path after the change (repo-relative). Empty for a DELETE. */
+  readonly pathAfter: string;
+  readonly changeType: FileChangeType;
+  readonly linesAdded: number;
+  readonly linesRemoved: number;
+}
+
+/**
+ * Result of {@link captureProgressDelta} — the git-numstat delta. Distinct from
+ * the substrate-neutral `ProgressDelta` (in {@link ./progress.js}) that the
+ * git/cas/hybrid substrates emit: this carries the working-tree sha (for the
+ * short-circuit) and git's `FileChangeType` entries; the neutral one carries
+ * proto `FileChangeKind` entries and an honest total.
+ */
+export interface GitProgressDelta {
+  /** The post-diff working-tree sha, for the caller's short-circuit cache. */
+  readonly afterTree: string;
+  /** One count-only entry per changed file (no content). */
+  readonly entries: readonly GitProgressEntry[];
+}
+
+/**
+ * Capture a NON-AUTHORITATIVE, content-free snapshot of the working-tree delta
+ * against `baselineTree` — the substrate for the live "N files changed so far"
+ * surface (mid-run live capture). Unlike {@link captureChangeSet} this reads NO
+ * file bytes: it stages the working tree into a temp index, writes its tree, and
+ * asks git for `--numstat` + `--name-status` only. Cheap enough to run on a
+ * debounce during the turn.
+ *
+ * Short-circuit: returns `undefined` when the working tree is unchanged since
+ * `lastTreeSha` (the caller passes back the previous result's `afterTree`), so a
+ * quiet turn pays only one `write-tree`, never a diff.
+ *
+ * Uses a dedicated temp-index label ("progress") so it never collides with the
+ * `baseline`/`capture`/`approved` indexes. `--no-renames` matches
+ * {@link captureChangeSet} (a rename surfaces as delete + create). A torn read of
+ * a file being written mid-turn is acceptable — the snapshot is non-authoritative
+ * and self-corrects on the next capture.
+ */
+export async function captureProgressDelta(
+  gitRoot: string,
+  executionId: string,
+  baselineTree: string,
+  excludePaths: readonly string[] = [],
+  lastTreeSha?: string,
+): Promise<GitProgressDelta | undefined> {
+  const gitDir = await resolveGitDir(gitRoot);
+  const afterTree = await writeWorkingTree(gitRoot, gitDir, "progress", executionId, excludePaths);
+  if (lastTreeSha !== undefined && afterTree === lastTreeSha) return undefined;
+
+  const [nameStatusRaw, numstatRaw] = await Promise.all([
+    git(gitRoot, ["diff", "--no-renames", "--name-status", "-z", baselineTree, afterTree]),
+    git(gitRoot, ["diff", "--no-renames", "--numstat", "-z", baselineTree, afterTree]),
+  ]);
+
+  const counts = parseNumstatZ(numstatRaw);
+  const entries: GitProgressEntry[] = [];
+  for (const { status, path } of parseNameStatusZ(nameStatusRaw)) {
+    const changeType = nameStatusToChangeType(status);
+    const count = counts.get(path);
+    entries.push({
+      pathBefore: changeType === FileChangeType.CREATE ? "" : path,
+      pathAfter: changeType === FileChangeType.DELETE ? "" : path,
+      changeType,
+      linesAdded: count?.added ?? 0,
+      linesRemoved: count?.removed ?? 0,
+    });
+  }
+  return { afterTree, entries };
+}
+
 /**
  * Restore the given files to their `baselineTree` (pre-turn) bytes. On resume
  * this reverts the REJECTED/undecided subset so a discarded change snaps back
@@ -462,6 +543,39 @@ function parseNameStatusZ(raw: string): NameStatusEntry[] {
   return entries;
 }
 
+/**
+ * Parse `git diff --numstat -z` into a path -> {added, removed} map. Each record
+ * is a NUL-terminated `<added>\t<removed>\t<path>` triple; a binary file reports
+ * `-` for both counts, which we surface as 0 (no count) per the display contract.
+ */
+function parseNumstatZ(raw: string): Map<string, { added: number; removed: number }> {
+  const counts = new Map<string, { added: number; removed: number }>();
+  for (const record of raw.split("\u0000")) {
+    if (!record) continue;
+    const tab = record.indexOf("\t");
+    const tab2 = record.indexOf("\t", tab + 1);
+    if (tab < 0 || tab2 < 0) continue;
+    const addedStr = record.slice(0, tab);
+    const removedStr = record.slice(tab + 1, tab2);
+    const path = record.slice(tab2 + 1);
+    if (!path) continue;
+    const added = addedStr === "-" ? 0 : Number.parseInt(addedStr, 10) || 0;
+    const removed = removedStr === "-" ? 0 : Number.parseInt(removedStr, 10) || 0;
+    counts.set(path, { added, removed });
+  }
+  return counts;
+}
+
+/**
+ * Map a `--name-status` status letter to a {@link FileChangeType} — the same
+ * A -> CREATE, D -> DELETE, else MODIFY rule {@link buildCapturedChange} uses.
+ */
+function nameStatusToChangeType(status: string): FileChangeType {
+  if (status === "A") return FileChangeType.CREATE;
+  if (status === "D") return FileChangeType.DELETE;
+  return FileChangeType.MODIFY;
+}
+
 /** Read a blob's exact BYTES from a tree, or `undefined` when absent. */
 async function readBlobBytes(
   gitRoot: string,

package/src/shared/filereview/index.ts

@@ -33,19 +33,54 @@ export {
   type CaptureResumeResult,
 } from "./capture.js";
 
-export { isGitWorkTree } from "./git-substrate.js";
+export {
+  captureProgressDelta,
+  isGitWorkTree,
+  type GitProgressDelta,
+  type GitProgressEntry,
+} from "./git-substrate.js";
+
+export {
+  buildFileChangeProgress,
+  captureFileChangeProgress,
+  createGitProgressSubstrate,
+  createHybridProgressSubstrate,
+  newProgressCaptureState,
+  PROGRESS_CAPTURE_MIN_INTERVAL_MS,
+  PROGRESS_MAX_ENTRIES,
+  shouldCaptureProgress,
+  type ProgressCapture,
+  type ProgressCaptureState,
+  type ProgressDelta,
+  type ProgressEntry,
+  type ProgressSubstrate,
+} from "./progress.js";
+
+export {
+  createCasProgressSubstrate,
+  type CasTouchedReader,
+  type CasTouchedSnapshot,
+} from "./cas-progress.js";
+
+export {
+  countLineChanges,
+  LINE_COUNT_MAX_BYTES,
+  type LineChangeCounts,
+} from "./line-counts.js";
 
 export {
   applyCasApproved,
   casBlobKey,
   casBlobReader,
   casManifestKey,
+  classifyCasChange,
   loadCasManifest,
   restoreCasToBaseline,
   snapshotCasChangeSet,
   type BlobReader,
   type CasBlobRef,
   type CasCapturedFile,
+  type CasChangeClassification,
   type CasManifest,
   type CasPathCapture,
   type CasSnapshotRef,

package/src/shared/filereview/line-counts.ts

@@ -0,0 +1,83 @@
+/**
+ * Capture-time line counting for the file-review subsystem: the `+N −M` stamped
+ * onto each {@link CapturedFileChange} so review surfaces can decorate file
+ * lists without the file bodies.
+ *
+ * WHY THE RUNNER COUNTS, AND WHY WITH JSDIFF. The counts exist because the
+ * review UI's compact file lists must show per-file magnitude, but by the time
+ * the UI renders, large before/after bodies may have been offloaded to artifact
+ * storage (status-offload) — a client cannot count what it would first have to
+ * download. So the counts are computed once, at capture, where the bytes are
+ * always in hand. They are computed with the SAME `diff` (jsdiff) Myers
+ * algorithm the SDK's diff renderer uses (`computeDiff` → `structuredPatch`),
+ * NOT with `git diff --numstat`: two counting authorities can disagree on edge
+ * cases (trailing newlines being the classic), and a list that says `+37 −1`
+ * beside a rendered diff showing `+37 −0` is a self-contradicting UI. One
+ * algorithm, consistent by construction.
+ *
+ * The counts are INFORMATIONAL display data only — never an enforcement input,
+ * never folded into `file_digest`/`aggregate_digest` (the same contract as
+ * `blocked_reason`).
+ */
+
+import { structuredPatch } from "diff";
+
+/**
+ * Per-side byte ceiling above which counting is skipped. Myers is O(ND) and a
+ * pathological many-MB text pair must never stall the capture activity just to
+ * decorate a list; above this the counts stay absent and the UI hides the stat.
+ * Generous relative to the 128 KiB per-side inline-persist cap — a file can be
+ * several times that and still count quickly — while bounding the worst case.
+ */
+export const LINE_COUNT_MAX_BYTES = 1024 * 1024;
+
+/** The `+N −M` pair for one file change. */
+export interface LineChangeCounts {
+  readonly linesAdded: number;
+  readonly linesRemoved: number;
+}
+
+/**
+ * Count the added/removed lines between two text sides, exactly as the SDK's
+ * diff renderer would show them.
+ *
+ * An absent side is the empty document — an ADD counts every `after` line as
+ * added, a DELETE every `before` line as removed — mirroring how the renderer
+ * diffs a create/delete. `` markers are metadata,
+ * not content, and are excluded (the same rule as the SDK's `mapPatchHunks`).
+ *
+ * Returns `undefined` when counting is not possible or meaningful: both sides
+ * absent, or either side over {@link LINE_COUNT_MAX_BYTES}. Callers leave the
+ * proto counts at zero in that case, which consumers render as "no stat".
+ */
+export function countLineChanges(
+  before: string | undefined,
+  after: string | undefined,
+): LineChangeCounts | undefined {
+  if (before === undefined && after === undefined) return undefined;
+  const beforeText = before ?? "";
+  const afterText = after ?? "";
+  if (
+    Buffer.byteLength(beforeText, "utf8") > LINE_COUNT_MAX_BYTES ||
+    Buffer.byteLength(afterText, "utf8") > LINE_COUNT_MAX_BYTES
+  ) {
+    return undefined;
+  }
+
+  // Zero context: the +/- line population is independent of context width, and
+  // omitting context lines keeps the walk (and the patch object) minimal.
+  const patch = structuredPatch("a", "b", beforeText, afterText, "", "", {
+    context: 0,
+  });
+
+  let linesAdded = 0;
+  let linesRemoved = 0;
+  for (const hunk of patch.hunks) {
+    for (const line of hunk.lines) {
+      if (line.startsWith("\\")) continue;
+      if (line[0] === "+") linesAdded++;
+      else if (line[0] === "-") linesRemoved++;
+    }
+  }
+  return { linesAdded, linesRemoved };
+}