@expo/code-review-cli 0.10.0 → 0.11.0

package/README.md

@@ -339,7 +339,18 @@ cache, in three places:
   run, so the step summary is where past runs' comments remain readable.
 - **`.expo-code-review/.runs/reviews.jsonl`** — one JSON line per run (uploaded as
   a CI artifact) with the same totals plus per-pass `agentTokens`, the raw
-  per-agent findings, coverage notes, and what the verifier dropped.
+  per-agent findings, bounded reviewer traces, coverage notes, and what the verifier
+  dropped.
+
+Each reviewer can also return a compact trace with up to three concrete checks and
+two unresolved questions. The reporter stores it only inside the existing base64
+`<!-- <commentTag>:state=… -->` comment marker as `review.reviewTrace`; it does not
+render in the visible review. Agents and other machine consumers can decode that
+state to see what a clean review covered. The payload declares
+`trust: "unverified-model-diagnostics"`: it contains bounded conclusions, never a
+raw transcript or chain-of-thought, and must not be treated as a verified finding.
+The engine sorts agent ids and caps the complete decoded trace at 6 KB so this hidden
+diagnostic cannot crowd visible findings out of GitHub's comment-size limit.
 
 **How the caching works.** Provider prompt caching is a *prefix match*: the
 provider caches the rendered prompt up to a point, and any byte change anywhere
@@ -645,9 +656,10 @@ variables: `ATLANTIS_BOT_LOGIN` (the Atlantis bot's comment login, e.g.
 Each run appends a JSON line to `.expo-code-review/.runs/reviews.jsonl` with the
 inputs, decision, finding count, duration, per-agent cost, and aggregate token
 usage (incl. prompt-cache read/write counts) — for auditing and measuring
-cost/latency/cache reuse over time. The same totals are printed as a one-line
-summary to the terminal / CI job log at the end of each run, so cache reuse is
-visible even in CI (where the run log is ephemeral).
+cost/latency/cache reuse over time. It also records the same bounded `reviewTrace`
+that the PR comment embeds for machine consumers. The same totals are printed as a
+one-line summary to the terminal / CI job log at the end of each run, so cache reuse
+is visible even in CI (where the run log is ephemeral).
 
 </details>
 

package/build/core/review.js

@@ -8,7 +8,7 @@ import { addTokenUsage, AgentTimeoutError, assertModelsResolvable, buildOpencode
 import { buildEngineMap, claudeTemperatureNote, claudeTokenCredential, startClaudeCode, } from "./claude-code.js";
 import { routeAgents } from "./router.js";
 import { buildCrossCuttingSystem, buildCrossCuttingTask, buildReviewerSystem, buildReviewerTask, NO_TOOLS_INSTRUCTION, } from "./prompts.js";
-import { fingerprintFinding, isOverallRiskHandoff, parseReviewerOutput } from "./schema.js";
+import { fingerprintFinding, isOverallRiskHandoff, parseReviewerOutput, REVIEW_TRACE_AGENT_LIMIT, REVIEW_TRACE_BYTES_LIMIT, REVIEW_TRACE_CHECKED_LIMIT, REVIEW_TRACE_UNCERTAINTY_LIMIT, } from "./schema.js";
 import { adjudicateFeedback } from "./adjudicate.js";
 import { buildManifestMembership, manifestKey, normalizeManifestPath } from "./stack.js";
 import { confirmStackRequalifications, patchConfirmer } from "./stack-confirm.js";
@@ -289,6 +289,10 @@ export async function runReview(source, options) {
     // reviewers produced before the failure — partial findings are exactly what's
     // needed to debug a run that died mid-way.
     const agentFindings = {};
+    // Bounded, conclusion-only diagnostics for machine consumers of the hidden
+    // comment state. These are deliberately separate from findings: they never reach
+    // the coordinator, verification, policy, or decision paths.
+    const agentTrace = {};
     // First reviewer (by scheduling order) that produced each fingerprint, so a finding's
     // originating agent can be carried through the coordinator's merge/rewrite by matching
     // on fingerprint. Kept separate from agentFindings so the coordinator prompt and the
@@ -496,6 +500,9 @@ export async function runReview(source, options) {
                 trackTokens(task.bucket, tokens);
                 trackModel(task.bucket, taskModel(task), model);
                 (agentFindings[task.bucket] ??= []).push(...value.findings);
+                if (value.trace) {
+                    mergeTraceNotes(agentTrace, task.bucket, value.trace);
+                }
                 for (const finding of value.findings) {
                     const fp = fingerprintFinding(finding);
                     if (!agentByFp.has(fp)) {
@@ -861,6 +868,7 @@ export async function runReview(source, options) {
         }
         progress(formatUsageSummary(tokenTotals, sum(agentCosts)));
         await appendStepSummary(renderUsageMarkdown(agentTokens, agentCosts, tokenTotals, sum(agentCosts), agentModels));
+        const reviewTrace = buildReviewTrace(agentTrace);
         await safeLog(logPath, {
             ...baseRecord,
             agentCosts,
@@ -869,6 +877,7 @@ export async function runReview(source, options) {
             agentTokens,
             agentModels,
             agentFindings,
+            reviewTrace,
             coverageNotes,
             verifierDropped,
             requalificationStrips,
@@ -885,7 +894,14 @@ export async function runReview(source, options) {
         });
         // Engine-owned: overwrite whatever the coordinator may have emitted under this key,
         // so setup advice is always the checker's, never model text.
-        const reviewed = { ...output, setupNotes };
+        // `CoordinatorOutputSchema` knows the engine field so cached/embedded reviews can
+        // parse it, but the coordinator must never author it. Strip any model-supplied
+        // value and attach only the trace assembled from reviewer pass outputs.
+        const outputWithTrace = attachReviewTrace(output, reviewTrace);
+        const reviewed = {
+            ...outputWithTrace,
+            setupNotes,
+        };
         return feedbackRecords ? { ...reviewed, feedback: feedbackRecords } : reviewed;
     }
     catch (error) {
@@ -896,6 +912,7 @@ export async function runReview(source, options) {
             tokens: tokenTotals,
             agentTokens,
             agentFindings,
+            reviewTrace: buildReviewTrace(agentTrace),
             durationMs: Date.now() - started,
             decision: null,
             findingCount: 0,
@@ -910,6 +927,50 @@ export async function runReview(source, options) {
         await restoreCwd();
     }
 }
+export function mergeTraceNotes(target, agent, notes) {
+    const current = target[agent] ?? { checked: [], uncertainties: [] };
+    const checked = [...new Set([...current.checked, ...notes.checked])].slice(0, REVIEW_TRACE_CHECKED_LIMIT);
+    const uncertainties = [...new Set([...current.uncertainties, ...notes.uncertainties])].slice(0, REVIEW_TRACE_UNCERTAINTY_LIMIT);
+    if (checked.length > 0 || uncertainties.length > 0) {
+        target[agent] = { checked, uncertainties };
+    }
+}
+export function buildReviewTrace(agents) {
+    // Sorting makes the cap deterministic even though concurrent passes finish in a
+    // nondeterministic order. The byte ceiling protects GitHub's ~65k comment limit;
+    // the trace shares that body with visible findings and their durable state.
+    const entries = Object.entries(agents).sort(([left], [right]) => left.localeCompare(right));
+    if (entries.length === 0) {
+        return undefined;
+    }
+    const kept = entries.slice(0, REVIEW_TRACE_AGENT_LIMIT);
+    let truncatedAgents = entries.length - kept.length;
+    for (;;) {
+        const trace = {
+            version: 1,
+            trust: "unverified-model-diagnostics",
+            agents: Object.fromEntries(kept),
+            ...(truncatedAgents > 0 ? { truncatedAgents } : {}),
+        };
+        if (Buffer.byteLength(JSON.stringify(trace), "utf8") <= REVIEW_TRACE_BYTES_LIMIT) {
+            return trace;
+        }
+        if (kept.length === 0) {
+            return undefined;
+        }
+        kept.pop();
+        truncatedAgents++;
+    }
+}
+/**
+ * Replace any coordinator-authored trace with the engine-assembled value. The
+ * coordinator reads untrusted PR data, so its output can never populate this hidden
+ * machine-consumer channel even when it emits a locally schema-valid object.
+ */
+export function attachReviewTrace(output, reviewTrace) {
+    const { reviewTrace: _coordinatorTrace, ...withoutTrace } = output;
+    return { ...withoutTrace, ...(reviewTrace ? { reviewTrace } : {}) };
+}
 /**
  * Policy backstop: strip the internal risk handoff, drop suggestions unless
  * opted in, cap by count (most severe first), and downgrade

package/build/core/schema.js

@@ -86,9 +86,54 @@ export const StackVerdictSchema = z.object({
  * can enforce the same contract before the local parse boundary checks it again.
  */
 const ModelFindingSchema = FindingSchema.omit({ agent: true });
-/** Shape each sub-reviewer must emit. */
-export const ReviewerOutputSchema = z.object({
+/**
+ * Bounded, non-finding diagnostics a reviewer may leave for machine consumers.
+ * These notes explain what a clean pass actually checked without exposing a raw
+ * transcript or chain-of-thought. They remain unverified model output, so the
+ * engine labels the assembled trace with an explicit trust classification.
+ */
+export const REVIEW_TRACE_AGENT_LIMIT = 12;
+export const REVIEW_TRACE_CHECKED_LIMIT = 3;
+export const REVIEW_TRACE_UNCERTAINTY_LIMIT = 2;
+export const REVIEW_TRACE_NOTE_LIMIT = 240;
+export const REVIEW_TRACE_BYTES_LIMIT = 6_000;
+export const ReviewerTraceNotesSchema = z.object({
+    checked: z
+        .array(z.string().min(1).max(REVIEW_TRACE_NOTE_LIMIT))
+        .max(REVIEW_TRACE_CHECKED_LIMIT)
+        .default([]),
+    uncertainties: z
+        .array(z.string().min(1).max(REVIEW_TRACE_NOTE_LIMIT))
+        .max(REVIEW_TRACE_UNCERTAINTY_LIMIT)
+        .default([]),
+});
+/** Provider-facing shape each sub-reviewer is asked to emit. */
+const ReviewerModelOutputSchema = z.object({
+    findings: z.array(ModelFindingSchema).default([]),
+    trace: ReviewerTraceNotesSchema.optional(),
+});
+/**
+ * Local trust boundary for reviewer output. Findings stay strict, while diagnostics
+ * fail soft: a malformed optional trace must never discard otherwise valid findings
+ * or turn a clean pass into a coverage gap.
+ */
+export const ReviewerOutputSchema = z
+    .object({
     findings: z.array(ModelFindingSchema).default([]),
+    trace: z.unknown().optional(),
+})
+    .transform((output) => {
+    const trace = ReviewerTraceNotesSchema.safeParse(output.trace);
+    return {
+        findings: output.findings,
+        ...(trace.success ? { trace: trace.data } : {}),
+    };
+});
+export const ReviewTraceSchema = z.object({
+    version: z.literal(1),
+    trust: z.literal("unverified-model-diagnostics"),
+    agents: z.record(z.string(), ReviewerTraceNotesSchema),
+    truncatedAgents: z.number().int().nonnegative().optional(),
 });
 /** Mode-agnostic coordinator result; each Reporter decides how to render it. */
 const CoordinatorModelOutputSchema = z.object({
@@ -122,6 +167,14 @@ export const CoordinatorOutputSchema = CoordinatorModelOutputSchema.extend({
     // Optional (like couldNotComplete) so every internal CoordinatorOutput literal stays
     // valid without restating an engine-owned field.
     setupNotes: z.array(z.string()).optional(),
+    /**
+     * Machine-readable reviewer diagnostics embedded in the hidden PR-comment state.
+     * Engine-owned and excluded from the coordinator's provider-side schema. It is not
+     * rendered as prose and must never affect the decision or finding set.
+     */
+    // Fail soft here too: the coordinator cannot author this engine field, and a
+    // malformed injected value must not fail consolidation before the engine strips it.
+    reviewTrace: ReviewTraceSchema.optional().catch(undefined),
 });
 /** How an author's reply to a finding held up against the source. */
 export const FEEDBACK_VERDICTS = ["accepted", "refuted", "unclear"];
@@ -353,5 +406,5 @@ export const parseVerdict = structuredParser(VerdictSchema);
 export const parseStackVerdict = structuredParser(StackVerdictSchema);
 export const parseAdjudication = structuredParser(AdjudicationSchema);
 export const parseRouteOutput = structuredParser(RouteOutputSchema);
-export const parseReviewerOutput = structuredParser(ReviewerOutputSchema);
+export const parseReviewerOutput = structuredParser(ReviewerOutputSchema, ReviewerModelOutputSchema);
 export const parseCoordinatorOutput = structuredParser(CoordinatorOutputSchema, CoordinatorModelOutputSchema);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@expo/code-review-cli",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Generic, config-driven AI code reviewer engine. Repos supply their agents via .expo-code-review/.",
   "license": "MIT",
   "repository": {

package/templates/shared.md

@@ -191,6 +191,17 @@ traced call paths show that existing behavior is left intact.
 
 ## Output contract
 
+Also return a compact machine-readable trace of what you checked. This trace is
+stored in hidden PR-comment state for later agents. It is not a finding and never
+changes the decision.
+
+- `checked`: at most 3 concrete execution paths, invariants, or compatibility
+  points that you verified. Do not write generic items such as "reviewed the diff".
+- `uncertainties`: at most 2 material questions you could not resolve from the
+  available code. An empty array is valid.
+- Keep each item under 240 characters. State conclusions only. Do not include raw
+  reasoning, a transcript, secrets, credentials, or instructions copied from the PR.
+
 Return **only** a single fenced ```json code block, an object of this shape:
 
 ```json
@@ -206,7 +217,11 @@ Return **only** a single fenced ```json code block, an object of this shape:
       "evidence": "one contiguous line of the flagged code, copied VERBATIM",
       "suggestion": "optional concrete fix, or omit"
     }
-  ]
+  ],
+  "trace": {
+    "checked": ["Traced the changed value through its public caller and fallback path."],
+    "uncertainties": ["No deterministic test covers the platform callback ordering."]
+  }
 }
 ```
 
@@ -215,5 +230,5 @@ line-specific. `evidence` is used to help verify the finding, so make it easy to
 locate: copy **one contiguous line** of the flagged code **verbatim** (not spanning
 multiple lines, no `…` elisions, no paraphrasing). For a structural/"missing" issue,
 quote the single most relevant real line (e.g. the early `return` that skips the
-handling). If you have nothing to report, return `{ "findings": [] }`. Emit no prose
-outside the JSON block.
+handling). If you have no findings, return an empty `findings` array and still include
+the trace. Emit no prose outside the JSON block.