@skyramp/mcp 0.3.5 → 0.3.6-rc.2.ac20

package/build/tools/generate-tests/generateUIRestTool.d.ts

@@ -24,6 +24,7 @@ export declare const uiTestSchema: {
     force: z.ZodDefault<z.ZodBoolean>;
     codeReuse: z.ZodDefault<z.ZodBoolean>;
     modularizeCode: z.ZodDefault<z.ZodDefault<z.ZodBoolean>>;
+    allowTraceReuse: z.ZodDefault<z.ZodBoolean>;
     language: z.ZodNativeEnum<typeof import("../../types/TestTypes.js").ProgrammingLanguage>;
     framework: z.ZodString;
 };

package/build/tools/generate-tests/generateUIRestTool.js

@@ -5,6 +5,8 @@ import { TestGenerationService, } from "../../services/TestGenerationService.js"
 import { normalizeLanguageParams, resolveParamAliases, } from "../../utils/normalizeParams.js";
 import { getPersonaPrefix } from "../../prompts/personas.js";
 import { isTestbotEnabled } from "../../utils/featureFlags.js";
+import { toolError } from "../../utils/utils.js";
+import { checkTraceReuse, recordTraceOutput } from "./trace-reuse-guard.js";
 const TOOL_NAME = "skyramp_ui_test_generation";
 export class UITestService extends TestGenerationService {
     getTestType() {
@@ -32,6 +34,11 @@ export class UITestService extends TestGenerationService {
         });
         if (result.isError)
             return result;
+        // On modularize-first flows the base hand-off sequences the whole post-gen
+        // chain (enhance → modularize → reuse); a second block here would instruct
+        // enhance twice and re-extract the consolidated helpers.
+        if (this.emitsModularizeFirstHandOff(params))
+            return result;
         if (!params.enhanceAssertions && !params.modularizeCode)
             return result;
         const content = [...result.content];
@@ -83,6 +90,13 @@ export const uiTestSchema = {
     force: baseSchema.shape.force,
     codeReuse: codeRefactoringSchema.shape.codeReuse,
     modularizeCode: codeRefactoringSchema.shape.modularizeCode.default(true),
+    allowTraceReuse: z
+        .boolean()
+        .default(false)
+        .describe("Explicit override for the trace-reuse guard. By default, generating a second differently-named spec " +
+        "from a trace zip that already produced one is rejected — one recording covers one scenario, and reusing " +
+        "it manufactures identical tests under names the recording never covered. Set true only when the same " +
+        "recording genuinely covers the new scenario end-to-end."),
 };
 export function registerUITestTool(server) {
     server.registerTool(TOOL_NAME, {
@@ -123,8 +137,16 @@ This tells you exactly which frontend files changed so you record traces for the
             zip: "playwrightInput",
         });
         normalizeLanguageParams(params);
+        const traceReuse = await checkTraceReuse(params.playwrightInput, params.output, params.outputDir, !!params.allowTraceReuse, "ui");
+        if (traceReuse.rejection)
+            return toolError(traceReuse.rejection);
         const service = new UITestService();
         const result = await service.generateTest(params);
+        // Register only what generation actually produced: recording at check
+        // time poisoned the registry on failures (name-validation rejections,
+        // codegen errors), refusing the corrected retry for reusing its own trace.
+        if (!result.isError && traceReuse.key && traceReuse.identity)
+            recordTraceOutput(traceReuse.key, traceReuse.identity);
         AnalyticsService.pushTestGenerationToolEvent(TOOL_NAME, result, params).catch(() => {
             // Silently ignore analytics errors
         });

package/build/tools/generate-tests/scenarioLint.d.ts

@@ -9,6 +9,8 @@
  * failure family. This lint checks the bait at write time:
  *
  *  - a referenced value produced ONLY by a later step  → error (step order)
+ *    (with ONE remedy: reorder when that later step creates the referenced
+ *     resource, else raise the entropy of its fabricated id — see below)
  *  - a reference produced by no prior step at all      → warning (hardcode)
  *  - a reference produced by multiple prior steps      → warning (ambiguous)
  *

package/build/tools/generate-tests/scenarioLint.js

@@ -9,6 +9,8 @@
  * failure family. This lint checks the bait at write time:
  *
  *  - a referenced value produced ONLY by a later step  → error (step order)
+ *    (with ONE remedy: reorder when that later step creates the referenced
+ *     resource, else raise the entropy of its fabricated id — see below)
  *  - a reference produced by no prior step at all      → warning (hardcode)
  *  - a reference produced by multiple prior steps      → warning (ambiguous)
  *
@@ -16,6 +18,7 @@
  * fields with id-like keys, or keys resource-affine to a producing step's
  * collection path) to keep noise down — mirroring the reader's own matching.
  */
+import { singularize } from "../../utils/pluralization.js";
 const UUID_RE = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
 function isIdLikeKey(key) {
     return /^(id|pk|uuid)$/i.test(key) || /_id$/i.test(key) || /[a-z]Id$/.test(key);
@@ -23,6 +26,14 @@ function isIdLikeKey(key) {
 function isIdishSegment(seg) {
     return /^\d+$/.test(seg) || UUID_RE.test(seg);
 }
+/** The collection an id path-segment hangs off: "orders" in /orders/1/confirm. */
+function owningSegment(segs, idIndex) {
+    for (let i = idIndex - 1; i >= 0; i--) {
+        if (!isIdishSegment(segs[i]))
+            return segs[i].toLowerCase();
+    }
+    return "";
+}
 function lastStaticSegment(path) {
     const segs = path.split("/").filter((s) => s !== "");
     for (let i = segs.length - 1; i >= 0; i--) {
@@ -31,13 +42,45 @@ function lastStaticSegment(path) {
     }
     return "";
 }
+/** Strip the id suffix from a body key, accepting exactly the two spellings
+ *  `isIdLikeKey` accepts and no others: `_id` case-insensitively (it reads
+ *  `/_id$/i`, so `provider_ID` is a reference too) and a camel `Id` after a
+ *  lower-case letter. Words that merely end in those letters (`uuid`, `valid`)
+ *  keep their spelling — a blanket case-insensitive strip would turn `uuid`
+ *  into `uu` and match any 4-character collection. */
+function stripIdSuffix(key) {
+    if (/_id$/i.test(key))
+        return key.slice(0, -3);
+    if (/[a-z]Id$/.test(key))
+        return key.slice(0, -2);
+    return key;
+}
 /** "provider" ↔ "providers" style prefix affinity between a body key and a
  *  producing step's collection segment. Minimum overlap of 4 chars keeps
- *  short generic keys from matching everything. */
+ *  short generic keys from matching everything.
+ *
+ *  The id suffix comes off first: `product_id` and `products` are prefixes of
+ *  neither direction, and an FK is spelled with that suffix far more often than
+ *  without, so leaving it on made the affinity fail on the commonest shape it
+ *  exists to recognise — and a mis-ordered create then received the remedy
+ *  meant for a fixed-position sub-action. Both sides are then singularized,
+ *  because a prefix test cannot bridge an irregular plural: "category" is a
+ *  prefix of neither "categories" nor the reverse. */
 function keyResourceAffinity(key, segment) {
-    const k = key.toLowerCase();
-    const s = segment.toLowerCase();
-    if (k.length < 4 || s.length < 4)
+    const rawKey = stripIdSuffix(key).toLowerCase();
+    const rawSegment = segment.toLowerCase();
+    const k = singularize(rawKey);
+    const s = singularize(rawSegment);
+    // Equal singular forms are the strongest affinity there is, so they do not
+    // need the length floor — and they must skip it, because singularizing first
+    // collapses a 4-letter plural under it (`tags` -> `tag`, and `jobs`, `orgs`,
+    // `keys`, `pets` alike). `tag_id` with a later `POST /tags` then read as a
+    // sub-action and took the remedy that ships a hardcoded value.
+    if (k === s)
+        return true;
+    // The floor reads the RAW words for the same reason: `cases` -> `cas` is a
+    // 3-character over-strip that would fail a comparison the raw pair passes.
+    if (rawKey.length < 4 || rawSegment.length < 4)
         return false;
     return k.startsWith(s) || s.startsWith(k);
 }
@@ -85,27 +128,40 @@ export function lintScenarioChaining(steps) {
         if (body === undefined)
             return;
         const segment = typeof step.path === "string" ? lastStaticSegment(step.path) : "";
+        const creates = (step.method ?? "").toUpperCase() === "POST";
+        const collectionCreate = creates &&
+            typeof step.path === "string" &&
+            !step.path.split("/").some((seg) => seg !== "" && isIdishSegment(seg));
         walkPrimitives(body, "", (key, value) => {
             if (!isIdLikeKey(key))
                 return;
             const list = producers.get(value) ?? [];
-            if (!list.some((p) => p.step === idx))
-                list.push({ step: idx, segment });
+            const existing = list.find((p) => p.step === idx);
+            if (existing) {
+                if (!existing.keys.includes(key))
+                    existing.keys.push(key);
+            }
+            else {
+                list.push({ step: idx, segment, keys: [key], creates, collectionCreate });
+            }
             producers.set(value, list);
         });
     });
     steps.forEach((step, idx) => {
         // Collect this step's id-like references, one per distinct value, so a
         // value referenced in both the path and the body (or repeated in an
-        // array) yields a single classification and a single message.
-        const refs = new Map(); // value → first reference description
+        // array) yields a single classification and a single message. `resource`
+        // names the thing the reference points at ("provider", "orders") and only
+        // picks the remedy wording below — it never changes what is flagged.
+        const refs = new Map();
         // Path segments that look like ids are always FK-ish (path params are ids).
         if (typeof step.path === "string") {
-            for (const seg of step.path.split("/")) {
-                if (seg !== "" && isIdishSegment(seg) && !refs.has(seg)) {
-                    refs.set(seg, "path segment");
-                }
-            }
+            const segs = step.path.split("/").filter((s) => s !== "");
+            segs.forEach((seg, i) => {
+                if (!isIdishSegment(seg) || refs.has(seg))
+                    return;
+                refs.set(seg, { desc: "path segment", resource: owningSegment(segs, i) });
+            });
         }
         // Body values: only id-like keys, or keys resource-affine to a producer.
         const body = parseJsonObject(step.requestBody);
@@ -117,16 +173,44 @@ export function lintScenarioChaining(steps) {
                 const relevant = isIdLikeKey(key) || producerList.some((p) => keyResourceAffinity(key, p.segment));
                 if (!relevant)
                     return;
-                refs.set(value, `body field "${key}"`);
+                refs.set(value, { desc: `body field "${key}"`, resource: key, key });
             });
         }
-        for (const [value, refDesc] of refs) {
+        for (const [value, ref] of refs) {
+            const refDesc = ref.desc;
             const all = producers.get(value) ?? [];
             const earlier = all.filter((p) => p.step < idx);
             // Strictly-later producers are a fixable ordering mistake; a producer in
             // the SAME step (e.g. a create whose own response echoes the id) is just
             // unchainable — Go only chains from strictly-prior steps.
             const later = all.filter((p) => p.step > idx);
+            // An echo of the referencing field is not a producer. A sub-action
+            // response that mirrors the parent's child collection re-emits the very
+            // field the earlier step SENT — `items[].product_id` comes back on every
+            // confirm/ship/deliver — so no value of that field can clear the error,
+            // and the only escape is to delete the array from the fabricated
+            // response. Measured on run 32395388709: the agent did exactly that, and
+            // the assertion judge then docked 10 points for the missing array
+            // coverage. A DIFFERENT id key carrying the same value is still an error:
+            // that is the genuine `product_id: 1` vs `order_id: 1` ambiguity, where
+            // raising the fabricated id is both followable and correct.
+            // A later step that creates the very resource this reference points at.
+            // Found BEFORE the echo test, because a create is a genuine producer: a
+            // conventional `POST /orders` answering `{ order_id }` reuses the FK's own
+            // spelling and would otherwise satisfy the echo condition below, silently
+            // downgrading a real ordering mistake to a hardcoding warning.
+            // The second lookup catches an ALIASED foreign key. `customer_id` created
+            // by `POST /clients` shares no prefix with "clients", so affinity cannot
+            // see it, and the echo test below then downgraded a genuine ordering
+            // mistake to a warning. A collection create planting the referencing key
+            // is a producer whatever the key is called; a sub-action POST is excluded
+            // by `collectionCreate`, so the measured `/orders/{id}/confirm` echo is
+            // untouched.
+            const creator = later.find((p) => p.creates && keyResourceAffinity(ref.resource, p.segment)) ??
+                later.find((p) => p.collectionCreate && ref.key !== undefined && p.keys.includes(ref.key));
+            const echoesReferencingField = ref.key !== undefined &&
+                creator === undefined &&
+                later.every((p) => p.keys.every((k) => k === ref.key));
             if (earlier.length > 0) {
                 if (earlier.length > 1) {
                     warnings.push(`Value "${value}" referenced by ${stepLabel(steps, idx)} is produced by multiple prior steps ` +
@@ -134,10 +218,34 @@ export function lintScenarioChaining(steps) {
                         `use distinct, high-entropy ids (e.g. fresh UUIDs) in fabricated responses.`);
                 }
             }
-            else if (later.length > 0) {
-                errors.push(`${stepLabel(steps, idx)} references value "${value}" (${refDesc}) that is only produced by a ` +
-                    `later ${stepLabel(steps, later[0].step)} — reorder the steps so the producing create comes ` +
-                    `first (the generated test can only chain from PRIOR steps).`);
+            else if (later.length > 0 && !echoesReferencingField) {
+                // Two causes need OPPOSITE fixes, and only one of them can be
+                // reordered. Naming both is worse than naming one: renumbering the
+                // later id ALWAYS clears the error because it deletes the producer, so
+                // an agent offered both picks it and the mis-ordered batch goes green
+                // while still hardcoding the value. Pick the remedy here instead.
+                if (creator) {
+                    // The later step creates the very resource being referenced, so it
+                    // is a real ordering mistake and the value is already correct.
+                    errors.push(`${stepLabel(steps, idx)} references value "${value}" (${refDesc}) that is only produced by a ` +
+                        `later ${stepLabel(steps, creator.step)} — that step creates the "${creator.segment}" this ` +
+                        `reference points at, so reorder the batch to put step ${creator.step + 1} before ` +
+                        `step ${idx + 1} and keep the value unchanged (the generated test can only chain from PRIOR steps).`);
+                }
+                else {
+                    // The later step is a sub-action on a fixed rung of a state machine
+                    // (confirm/ship/deliver). It cannot move, so the collision is the
+                    // fabricated id being too guessable — raise its entropy.
+                    const producer = later[0];
+                    const shape = producer.segment
+                        ? `is a "${producer.segment}" sub-action, not a create of what this reference points at`
+                        : `does not create what this reference points at`;
+                    errors.push(`${stepLabel(steps, idx)} references value "${value}" (${refDesc}) that is only produced by a ` +
+                        `later ${stepLabel(steps, producer.step)} — that step ${shape}, so its position is fixed and ` +
+                        `reordering cannot fix this. Change the id in step ${producer.step + 1}'s fabricated ` +
+                        `responseBody to a distinct high-entropy value (e.g. a fresh UUID or a large random integer) ` +
+                        `so it stops colliding with "${value}" here.`);
+                }
             }
             else {
                 warnings.push(`${stepLabel(steps, idx)} ${refDesc} value "${value}" does not appear in any prior step's ` +

package/build/tools/generate-tests/trace-reuse-guard.d.ts

@@ -0,0 +1,20 @@
+/** Test-only: clear the in-process trace→outputs registry. */
+export declare function resetTraceReuseGuard(): void;
+export interface TraceReuseCheck {
+    /** Set when the call must be rejected (same zip, different output). */
+    rejection?: string;
+    /** Content hash of the zip — pass to recordTraceOutput on success. */
+    key?: string;
+    /** The resolved output identity — pass to recordTraceOutput on success. */
+    identity?: string;
+}
+/**
+ * Read-only reuse check. Returns a rejection when this zip's content already
+ * generated a DIFFERENT output identity and `allowTraceReuse` is not set; on a
+ * clean check, returns the hash/identity for the caller to record after
+ * generation succeeds. Fails open (empty result) when the zip is unreadable —
+ * generation will surface its own error for that.
+ */
+export declare function checkTraceReuse(playwrightInput: string | undefined, output: string | undefined, outputDir: string | undefined, allowTraceReuse: boolean, scope: "ui" | "e2e"): Promise<TraceReuseCheck>;
+/** Register a generated output for its trace. Call ONLY after generation succeeded. */
+export declare function recordTraceOutput(key: string, identity: string): void;

package/build/tools/generate-tests/trace-reuse-guard.js

@@ -0,0 +1,93 @@
+import { createHash } from "crypto";
+import { createReadStream } from "fs";
+import { pipeline } from "stream/promises";
+import * as path from "path";
+/**
+ * SKYR-4262 — trace-reuse guard. One recording captures one flow; generating
+ * several differently-named specs from the same trace zip produces byte-identical
+ * tests whose names claim scenarios the recording never covered (observed in
+ * production: one trace fanned into 4 specs, none asserting its named behavior).
+ * Keyed by CONTENT hash, not path, so re-exporting a genuinely new recording to
+ * the same zip path never trips the guard.
+ *
+ * Scope: this registry is per MCP-server process. That is narrower than "one
+ * testbot run" — testbot retries an agent up to 3 times, each attempt spawning a
+ * fresh agent CLI and therefore a fresh server (a retry also deletes the run's
+ * state files and restarts the flow, so the reset is coherent) — and broader
+ * than one recording session in a long-lived IDE/stdio server. Both are
+ * acceptable for this guard's job: reusing an IDENTICAL zip under a new name is
+ * the failure mode wherever it happens, and a fresh recording always hashes
+ * differently.
+ *
+ * Check/record are split: `checkTraceReuse` is read-only and runs BEFORE
+ * generation; `recordTraceOutput` runs only after generation SUCCEEDS.
+ * Registering at check time poisoned the registry on any generation failure —
+ * e.g. output-name validation rejects `a.ts` with "rename to a.spec.ts", and the
+ * corrected retry was then refused for reusing its own trace.
+ */
+const generatedFromTrace = new Map();
+/** Test-only: clear the in-process trace→outputs registry. */
+export function resetTraceReuseGuard() {
+    generatedFromTrace.clear();
+}
+/**
+ * The registered identity of one generated spec: the RESOLVED output path, so
+ * the same filename into two different outputDirs still counts as a second,
+ * differently-named spec. An omitted `output` (Skyramp picks a default name)
+ * maps to a per-directory sentinel: two default-named generations from the same
+ * zip into the same directory overwrite each other and are allowed, while
+ * mixing a named and an unnamed generation of the same zip is a fan-out and is
+ * not.
+ */
+function outputIdentity(output, outputDir) {
+    return path.resolve(outputDir ?? process.cwd(), output ?? "(default output name)");
+}
+/**
+ * Read-only reuse check. Returns a rejection when this zip's content already
+ * generated a DIFFERENT output identity and `allowTraceReuse` is not set; on a
+ * clean check, returns the hash/identity for the caller to record after
+ * generation succeeds. Fails open (empty result) when the zip is unreadable —
+ * generation will surface its own error for that.
+ */
+export async function checkTraceReuse(playwrightInput, output, outputDir, allowTraceReuse, scope) {
+    if (!playwrightInput)
+        return {};
+    let key;
+    try {
+        // Stream the zip through the hash — trace zips carry full HARs and can be
+        // large, so buffering the whole file would spike the server's memory.
+        const hasher = createHash("sha256");
+        await pipeline(createReadStream(playwrightInput), hasher);
+        // Namespace by generator: a UI test and an E2E test from ONE recording are
+        // different artifacts (the e2e generator adds API correlation), not the
+        // byte-identical fan-out this guard exists to stop — only same-generator
+        // reuse of a zip is a duplicate.
+        key = `${scope}:${hasher.digest("hex")}`;
+    }
+    catch {
+        // Unreadable zip fails in generation with its own error; the guard stays out of the way.
+        return {};
+    }
+    const identity = outputIdentity(output, outputDir);
+    const outputs = generatedFromTrace.get(key);
+    if (!allowTraceReuse && outputs && outputs.size > 0 && !outputs.has(identity)) {
+        const prior = [...outputs].map(p => path.basename(p)).join(", ");
+        return {
+            key,
+            identity,
+            rejection: `This trace zip already generated ${prior}. A recording captures ONE flow — generating a second, ` +
+                `differently-named spec (${path.basename(identity)}) from the same trace produces an identical test whose name claims a ` +
+                `scenario the recording never covered. Record a dedicated trace for this scenario with the browser_* tools ` +
+                `(including browser_assert on the behavior the test is named for) and skyramp_export_zip, then generate from ` +
+                `that. If this recording genuinely covers the scenario end-to-end, pass allowTraceReuse: true and be prepared ` +
+                `to differentiate the spec's assertions afterward via skyramp_enhance_assertions.`,
+        };
+    }
+    return { key, identity };
+}
+/** Register a generated output for its trace. Call ONLY after generation succeeded. */
+export function recordTraceOutput(key, identity) {
+    const outputs = generatedFromTrace.get(key) ?? new Set();
+    outputs.add(identity);
+    generatedFromTrace.set(key, outputs);
+}

package/build/tools/runExistingTestsTool.d.ts

@@ -1,6 +1,6 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { WorkspaceConfig } from "../workspace/workspace.js";
-import { ParsedExternalRun } from "../types/ExternalTestExecution.js";
+import { ExternalFileOutcome, ExternalTestResult, ParsedExternalRun } from "../types/ExternalTestExecution.js";
 import { SUPPORTED_FRAMEWORKS } from "../workspace/frameworks.js";
 /** Test-run lifecycle resolved from a service's workspace.yml runtimeDetails. */
 export interface ResolvedRunConfig {
@@ -42,7 +42,9 @@ export interface RunExistingTestsParams {
     workspacePath: string;
     service?: string;
     testSelectors?: string[];
-    mode?: "confirm" | "verify";
+    /** Required: the only thing that marks a record as the pre-edit baseline or the
+     *  post-edit re-run. A wrong label is read as fact by the report. */
+    mode: "confirm" | "verify";
     project?: string;
     stateFile?: string;
     repository?: string;
@@ -128,6 +130,36 @@ export declare function aggregateRuns(parsed: ParsedExternalRun[]): ParsedExtern
  */
 export declare function runExternalSuite(spec: ReaderSpec, cfg: ResolvedRunConfig, opts: PlaywrightRunOpts): Promise<ParsedExternalRun>;
 export declare function runPlaywright(cfg: ResolvedRunConfig, opts: PlaywrightRunOpts): Promise<ParsedExternalRun>;
+/**
+ * Reduce a run's flat result list to one outcome per spec file.
+ *
+ * This is the only place that knows BOTH what the framework reported and what the
+ * run asked it for, which is what `complete` needs. Deriving it later from the
+ * results alone is not possible: a file re-run for one test looks identical to a
+ * file whose other tests silently vanished.
+ *
+ * `partialFiles` are the files a `file::test title` selector narrowed to a subset of
+ * their tests — the one narrowing this tool applies itself, and so the only one it
+ * can state with certainty.
+ */
+export declare function summarizeFilesInRun(results: ExternalTestResult[], opts: {
+    partialFiles: Set<string>;
+}): ExternalFileOutcome[];
+/** Placeholder identity for a record that names no suite because none resolved.
+ *  Deliberately not a valid suiteIdOf output, so it can never collide with one. */
+export declare const NO_RUNNABLE_SUITE = "(no runnable suite)";
+/** Stable identity for ONE configured suite. A service can declare several
+ *  (workspace.yml testSuites), so its name alone collides — two suites under one
+ *  service would share a slot and the later would mask the other instead of both
+ *  counting. */
+export declare function suiteIdOf(cfg: {
+    serviceName: string;
+    framework: string;
+    testRunCommand: string;
+}): string;
+/** The files a selector narrowed to individual tests. A bare `file` selector runs
+ *  the whole file; `file::test title` does not. */
+export declare function partialFilesFrom(selectors: string[]): Set<string>;
 /**
  * Orchestrates an external test run: resolve config → apply maxTests cap → run →
  * persist a record to the state file. Pure of MCP/registration concerns and

package/build/tools/runExistingTestsTool.js

@@ -440,6 +440,95 @@ const defaultDeps = {
     },
     now: () => new Date().toISOString(),
 };
+/** Worst outcome wins for a file. `error` (never reached its assertions) outranks
+ *  `fail` so a collection failure is never reported as an assertion failure, and
+ *  `skipped` sits below everything: it only wins when nothing else ran. */
+const STATUS_RANK = {
+    skipped: 0,
+    pass: 1,
+    fail: 2,
+    error: 3,
+};
+/**
+ * Reduce a run's flat result list to one outcome per spec file.
+ *
+ * This is the only place that knows BOTH what the framework reported and what the
+ * run asked it for, which is what `complete` needs. Deriving it later from the
+ * results alone is not possible: a file re-run for one test looks identical to a
+ * file whose other tests silently vanished.
+ *
+ * `partialFiles` are the files a `file::test title` selector narrowed to a subset of
+ * their tests — the one narrowing this tool applies itself, and so the only one it
+ * can state with certainty.
+ */
+export function summarizeFilesInRun(results, opts) {
+    const byFile = new Map();
+    for (const r of results) {
+        if (typeof r.file !== "string" || r.file === "")
+            continue;
+        const key = r.absoluteFile ?? r.file;
+        byFile.set(key, [...(byFile.get(key) ?? []), r]);
+    }
+    return [...byFile.values()].map((group) => {
+        const counts = { pass: 0, fail: 0, error: 0, skipped: 0 };
+        for (const r of group)
+            counts[r.status] = (counts[r.status] ?? 0) + 1;
+        const status = group.reduce((a, b) => STATUS_RANK[b.status] > STATUS_RANK[a.status] ? b : a).status;
+        const first = group[0];
+        return {
+            file: first.file,
+            ...(first.absoluteFile ? { absoluteFile: first.absoluteFile } : {}),
+            status,
+            counts,
+            complete: !isPartial(first, opts.partialFiles),
+            errors: group
+                .filter((r) => r.status === "fail" || r.status === "error")
+                .map((r) => `${r.testId}: ${r.message ?? r.status}`),
+            durationMs: group.reduce((sum, r) => sum + (r.durationMs ?? 0), 0),
+        };
+    });
+}
+/** A selector spelling and a result spelling need not agree — the selector is
+ *  repo-relative, the result may be absolute or framework-root-relative — so match
+ *  on a path-segment suffix in either direction. */
+function isPartial(r, partialFiles) {
+    if (partialFiles.size === 0)
+        return false;
+    const candidates = [r.file, r.absoluteFile].filter((c) => typeof c === "string" && c !== "");
+    for (const sel of partialFiles) {
+        for (const c of candidates) {
+            if (c === sel || c.endsWith("/" + sel) || sel.endsWith("/" + c))
+                return true;
+        }
+    }
+    return false;
+}
+/** Placeholder identity for a record that names no suite because none resolved.
+ *  Deliberately not a valid suiteIdOf output, so it can never collide with one. */
+export const NO_RUNNABLE_SUITE = "(no runnable suite)";
+/** Stable identity for ONE configured suite. A service can declare several
+ *  (workspace.yml testSuites), so its name alone collides — two suites under one
+ *  service would share a slot and the later would mask the other instead of both
+ *  counting. */
+export function suiteIdOf(cfg) {
+    return `${cfg.serviceName}::${cfg.framework}::${cfg.testRunCommand}`;
+}
+/** The files a selector narrowed to individual tests. A bare `file` selector runs
+ *  the whole file; `file::test title` does not. */
+export function partialFilesFrom(selectors) {
+    const out = new Set();
+    for (const sel of selectors) {
+        const [file, ...rest] = sel.split("::");
+        // A trailing `::` names no title, so every builder drops it and the whole file
+        // runs. Treating it as partial would throw away a real status.
+        if (rest.join("::").trim() === "")
+            continue;
+        const trimmed = file.trim();
+        if (trimmed)
+            out.add(trimmed);
+    }
+    return out;
+}
 /** Appends a run record into UnifiedAnalysisState.externalTestResults for the
  *  given repo section. Best-effort: a state failure never fails the run. */
 async function persistExternalRun(stateFile, repository, record) {
@@ -463,7 +552,7 @@ async function persistExternalRun(stateFile, repository, record) {
  * fully injectable for tests.
  */
 export async function runExternalTests(params, deps = defaultDeps) {
-    const mode = params.mode ?? "confirm";
+    const mode = params.mode;
     const selectors = params.testSelectors ?? [];
     let effective = selectors;
     let truncated = false;
@@ -486,6 +575,11 @@ export async function runExternalTests(params, deps = defaultDeps) {
         if (params.stateFile) {
             await persistExternalRun(params.stateFile, params.repository, {
                 ...skippedRun,
+                files: [],
+                // No run config resolved, so there is no suite to identify — and a service
+                // name here would read as one. The reader never keys on it (the record is
+                // skipped, and skipped records are dropped first), so name it for what it is.
+                suite: NO_RUNNABLE_SUITE,
                 mode,
                 ranAt: deps.now(),
                 repository: params.repository,
@@ -518,6 +612,8 @@ export async function runExternalTests(params, deps = defaultDeps) {
             if (params.stateFile) {
                 await persistExternalRun(params.stateFile, params.repository, {
                     ...routedOut,
+                    files: [],
+                    suite: suiteIdOf(cfg),
                     mode,
                     ranAt: deps.now(),
                     repository: params.repository,
@@ -542,8 +638,13 @@ export async function runExternalTests(params, deps = defaultDeps) {
         };
         perSuite.push(suiteResult);
         if (params.stateFile) {
+            const files = summarizeFilesInRun(suiteResult.results, {
+                partialFiles: partialFilesFrom(routing.kept),
+            });
             await persistExternalRun(params.stateFile, params.repository, {
                 ...suiteResult,
+                files,
+                suite: suiteIdOf(cfg),
                 mode,
                 ranAt: deps.now(),
                 repository: params.repository,
@@ -575,8 +676,7 @@ export function registerRunExistingTestsTool(server) {
                 .describe("Spec files to run (paths relative to the run command's cwd), optionally `file::test title` to also filter by name. Omit to run the whole configured suite."),
             mode: z
                 .enum(["confirm", "verify"])
-                .optional()
-                .describe("confirm (default): scoped run to establish which tests a change breaks; disables the suite's fail-fast cap (e.g. Playwright's maxFailures) so ALL breakages are counted. verify: re-run after an edit."),
+                .describe("REQUIRED, and the only thing that marks a run as before or after the edit: confirm = the pre-edit baseline (a scoped run establishing which tests the change breaks; disables the suite's fail-fast cap, e.g. Playwright's maxFailures, so ALL breakages are counted), verify = the re-run after you edited. Passing confirm for a post-edit run overwrites the baseline with the fixed result, and the report then shows a repair as having never been broken."),
             project: z
                 .string()
                 .optional()
@@ -635,7 +735,7 @@ export function registerRunExistingTestsTool(server) {
             AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {
                 workspacePath: params.workspacePath,
                 service: params.service ?? "",
-                mode: params.mode ?? "confirm",
+                mode: params.mode,
             }).catch((err) => {
                 logger.warning("Analytics event failed", { error: String(err) });
             });