@skyramp/mcp 0.3.6-rc.1 → 0.3.6-rc.2.ac20

package/build/adapters/jestAdapter.js

@@ -1,3 +1,4 @@
+import * as path from "path";
 /**
  * Shared jest + vitest adapter for skyramp_run_existing_tests. vitest's json
  * reporter is jest-compatible, so both use ONE parser; only the reporter flag
@@ -40,6 +41,7 @@ export function parseJestJson(report, opts) {
             results.push({
                 testId: `${name} › (file failed to run)`,
                 file: name,
+                ...(path.isAbsolute(name) ? { absoluteFile: name } : {}),
                 status: "error",
                 message: stripVTControlCharacters(raw).trim() || undefined,
                 durationMs: 0,
@@ -55,6 +57,7 @@ export function parseJestJson(report, opts) {
             results.push({
                 testId: `${name} › ${titlePath}`,
                 file: name,
+                ...(path.isAbsolute(name) ? { absoluteFile: name } : {}),
                 status,
                 message,
                 durationMs: Math.round(a.duration ?? 0),

package/build/adapters/mochaAdapter.js

@@ -1,3 +1,4 @@
+import * as path from "path";
 /**
  * Mocha adapter for skyramp_run_existing_tests. Normalizes `mocha --reporter
  * json` (emitted to stdout) into the neutral shape. Mocha does not tag
@@ -29,6 +30,7 @@ function toResult(t, status) {
     return {
         testId: `${file} › ${full}`,
         file,
+        ...(path.isAbsolute(file) ? { absoluteFile: file } : {}),
         status,
         message: status === "fail" || status === "error" ? messageOf(t) : undefined,
         durationMs: Math.round(t.duration ?? 0),

package/build/adapters/playwrightAdapter.js

@@ -12,6 +12,7 @@
  * builds the flags appended to the suite's `testRunCommand`. The IO shell that
  * actually spawns the run lives in `runExistingTestsTool`.
  */
+import * as path from "path";
 import { stripVTControlCharacters } from "util";
 /**
  * A Playwright project (or spec file) is "infra" when it exists to bring up /
@@ -67,6 +68,7 @@ function messageOf(test) {
 }
 export function parsePlaywrightJson(report, opts) {
     const rep = (report ?? {});
+    const rootDir = rep.config?.rootDir;
     const collected = [];
     for (const fileSuite of rep.suites ?? []) {
         collectSpecs(fileSuite, fileSuite.file ?? "", [], collected);
@@ -118,6 +120,7 @@ export function parsePlaywrightJson(report, opts) {
             results.push({
                 testId,
                 file,
+                ...(rootDir ? { absoluteFile: path.resolve(rootDir, file) } : {}),
                 status,
                 message: status === "fail" || status === "error" ? messageOf(test) : undefined,
                 durationMs,

package/build/adapters/pytestAdapter.js

@@ -1,3 +1,4 @@
+import * as path from "path";
 /**
  * pytest adapter for skyramp_run_existing_tests. Normalizes the
  * `pytest-json-report` (`--json-report`) output into the neutral result shape.
@@ -39,6 +40,8 @@ function durationMsOf(test) {
 export function parsePytestJson(report, opts) {
     const rep = (report ?? {});
     const tests = rep.tests ?? [];
+    const root = rep.root;
+    const abs = (f) => (root ? { absoluteFile: path.resolve(root, f) } : {});
     // Real test outcomes.
     const testResults = tests.map((t) => {
         const status = mapOutcome(t.outcome);
@@ -46,6 +49,7 @@ export function parsePytestJson(report, opts) {
         return {
             testId: nodeid,
             file: fileOf(nodeid),
+            ...abs(fileOf(nodeid)),
             status,
             message: status === "fail" || status === "error" ? messageOf(t) : undefined,
             durationMs: durationMsOf(t),
@@ -61,6 +65,7 @@ export function parsePytestJson(report, opts) {
         return {
             testId: `${nodeid} › (collection error)`,
             file: fileOf(nodeid),
+            ...abs(fileOf(nodeid)),
             status: "error",
             message: lastLine(c.longrepr),
             durationMs: 0,
@@ -72,6 +77,13 @@ export function parsePytestJson(report, opts) {
     // When collection failed AND no real test ran, the suite as a whole could not be
     // collected — environmental (a wall of red), not PR signal. When some real tests
     // DID run, the collection errors above stand as ordinary PR-signal error results.
+    // NOTE: a collection error names the file that failed to import, but says nothing
+    // about WHY. A missing settings module, a database that is down or any shared
+    // dependency failure surfaces per-module, landing on exactly the files the run
+    // selected — indistinguishable here from a file the PR itself broke. So a run that
+    // collected nothing stays unhealthy even when scoped, and the file keeps its
+    // Unknown baseline. Treating it as PR signal would let an environment outage be
+    // reported as a test the change broke, and then repaired.
     if (collectionErrors.length > 0 && testResults.length === 0) {
         environmentHealthy = false;
         const first = collectionErrors[0];

package/build/prompts/testbot/testbot-prompts.js

@@ -212,7 +212,7 @@ ${maintenanceBeforeExecStep}
 
    e. Call \`skyramp_actions\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) and apply the edits it returns.
 
-   f. Verify external-test fixes. **This step is not optional and it is the easiest one to forget — you have just edited files in step 2(e), so come back here before you move on to anything else.** It applies whenever step 2(a) reported a real pass/fail result for a file you then edited. It does NOT apply when step 2(a) returned \`skipped: true\` or \`ran: 0\` for every suite — there is no baseline to compare against, so say so in your report instead of re-running. When it applies: re-run those \`[external]\` files with \`skyramp_run_existing_tests\` (\`mode: "verify"\`, \`stateFile\`) and record each file's result as its \`afterStatus\`. Editing an \`[external]\` file that step 2(a) confirmed failing and NOT re-running it leaves your own fix unverified — you would be reporting a repair you never saw work. A still-failing verify is surfaced in the report — do not loop.
+   f. Verify external-test fixes. **This step is not optional and it is the easiest one to forget — you have just edited files in step 2(e), so come back here before you move on to anything else.** It applies whenever step 2(a) reported a real pass/fail result for a file you then edited. It does NOT apply when step 2(a) returned \`skipped: true\` or \`ran: 0\` for every suite — there is no baseline to compare against, so say so in your report instead of re-running. When it applies: re-run those \`[external]\` files with \`skyramp_run_existing_tests\` (\`mode: "verify"\`, \`stateFile\`) — the server reads each file's result back as its \`afterStatus\`. Editing an \`[external]\` file that step 2(a) confirmed failing and NOT re-running it leaves your own fix unverified — you would be reporting a repair you never saw work. A still-failing verify is surfaced in the report — do not loop.
 
 3. **Code review:** Find the logic bugs in the code that this change touches. Read the implementation of each changed endpoint: the route handler, and the functions that it calls to read or write data. For a changed screen, read the component and the functions that it calls. Read these files even when the diff does not contain them — a defect often sits in the code that the change depends on. Report each finding in \`issuesFound\` with a severity, and say which file and line holds it. Common patterns to flag:
    - Computed fields not recalculated after mutation (e.g. \`total_amount\` unchanged after items are added/removed)

package/build/tools/runExistingTestsTool.d.ts

@@ -1,6 +1,6 @@
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { WorkspaceConfig } from "../workspace/workspace.js";
-import { ParsedExternalRun } from "../types/ExternalTestExecution.js";
+import { ExternalFileOutcome, ExternalTestResult, ParsedExternalRun } from "../types/ExternalTestExecution.js";
 import { SUPPORTED_FRAMEWORKS } from "../workspace/frameworks.js";
 /** Test-run lifecycle resolved from a service's workspace.yml runtimeDetails. */
 export interface ResolvedRunConfig {
@@ -42,7 +42,9 @@ export interface RunExistingTestsParams {
     workspacePath: string;
     service?: string;
     testSelectors?: string[];
-    mode?: "confirm" | "verify";
+    /** Required: the only thing that marks a record as the pre-edit baseline or the
+     *  post-edit re-run. A wrong label is read as fact by the report. */
+    mode: "confirm" | "verify";
     project?: string;
     stateFile?: string;
     repository?: string;
@@ -128,6 +130,36 @@ export declare function aggregateRuns(parsed: ParsedExternalRun[]): ParsedExtern
  */
 export declare function runExternalSuite(spec: ReaderSpec, cfg: ResolvedRunConfig, opts: PlaywrightRunOpts): Promise<ParsedExternalRun>;
 export declare function runPlaywright(cfg: ResolvedRunConfig, opts: PlaywrightRunOpts): Promise<ParsedExternalRun>;
+/**
+ * Reduce a run's flat result list to one outcome per spec file.
+ *
+ * This is the only place that knows BOTH what the framework reported and what the
+ * run asked it for, which is what `complete` needs. Deriving it later from the
+ * results alone is not possible: a file re-run for one test looks identical to a
+ * file whose other tests silently vanished.
+ *
+ * `partialFiles` are the files a `file::test title` selector narrowed to a subset of
+ * their tests — the one narrowing this tool applies itself, and so the only one it
+ * can state with certainty.
+ */
+export declare function summarizeFilesInRun(results: ExternalTestResult[], opts: {
+    partialFiles: Set<string>;
+}): ExternalFileOutcome[];
+/** Placeholder identity for a record that names no suite because none resolved.
+ *  Deliberately not a valid suiteIdOf output, so it can never collide with one. */
+export declare const NO_RUNNABLE_SUITE = "(no runnable suite)";
+/** Stable identity for ONE configured suite. A service can declare several
+ *  (workspace.yml testSuites), so its name alone collides — two suites under one
+ *  service would share a slot and the later would mask the other instead of both
+ *  counting. */
+export declare function suiteIdOf(cfg: {
+    serviceName: string;
+    framework: string;
+    testRunCommand: string;
+}): string;
+/** The files a selector narrowed to individual tests. A bare `file` selector runs
+ *  the whole file; `file::test title` does not. */
+export declare function partialFilesFrom(selectors: string[]): Set<string>;
 /**
  * Orchestrates an external test run: resolve config → apply maxTests cap → run →
  * persist a record to the state file. Pure of MCP/registration concerns and

package/build/tools/runExistingTestsTool.js

@@ -440,6 +440,95 @@ const defaultDeps = {
     },
     now: () => new Date().toISOString(),
 };
+/** Worst outcome wins for a file. `error` (never reached its assertions) outranks
+ *  `fail` so a collection failure is never reported as an assertion failure, and
+ *  `skipped` sits below everything: it only wins when nothing else ran. */
+const STATUS_RANK = {
+    skipped: 0,
+    pass: 1,
+    fail: 2,
+    error: 3,
+};
+/**
+ * Reduce a run's flat result list to one outcome per spec file.
+ *
+ * This is the only place that knows BOTH what the framework reported and what the
+ * run asked it for, which is what `complete` needs. Deriving it later from the
+ * results alone is not possible: a file re-run for one test looks identical to a
+ * file whose other tests silently vanished.
+ *
+ * `partialFiles` are the files a `file::test title` selector narrowed to a subset of
+ * their tests — the one narrowing this tool applies itself, and so the only one it
+ * can state with certainty.
+ */
+export function summarizeFilesInRun(results, opts) {
+    const byFile = new Map();
+    for (const r of results) {
+        if (typeof r.file !== "string" || r.file === "")
+            continue;
+        const key = r.absoluteFile ?? r.file;
+        byFile.set(key, [...(byFile.get(key) ?? []), r]);
+    }
+    return [...byFile.values()].map((group) => {
+        const counts = { pass: 0, fail: 0, error: 0, skipped: 0 };
+        for (const r of group)
+            counts[r.status] = (counts[r.status] ?? 0) + 1;
+        const status = group.reduce((a, b) => STATUS_RANK[b.status] > STATUS_RANK[a.status] ? b : a).status;
+        const first = group[0];
+        return {
+            file: first.file,
+            ...(first.absoluteFile ? { absoluteFile: first.absoluteFile } : {}),
+            status,
+            counts,
+            complete: !isPartial(first, opts.partialFiles),
+            errors: group
+                .filter((r) => r.status === "fail" || r.status === "error")
+                .map((r) => `${r.testId}: ${r.message ?? r.status}`),
+            durationMs: group.reduce((sum, r) => sum + (r.durationMs ?? 0), 0),
+        };
+    });
+}
+/** A selector spelling and a result spelling need not agree — the selector is
+ *  repo-relative, the result may be absolute or framework-root-relative — so match
+ *  on a path-segment suffix in either direction. */
+function isPartial(r, partialFiles) {
+    if (partialFiles.size === 0)
+        return false;
+    const candidates = [r.file, r.absoluteFile].filter((c) => typeof c === "string" && c !== "");
+    for (const sel of partialFiles) {
+        for (const c of candidates) {
+            if (c === sel || c.endsWith("/" + sel) || sel.endsWith("/" + c))
+                return true;
+        }
+    }
+    return false;
+}
+/** Placeholder identity for a record that names no suite because none resolved.
+ *  Deliberately not a valid suiteIdOf output, so it can never collide with one. */
+export const NO_RUNNABLE_SUITE = "(no runnable suite)";
+/** Stable identity for ONE configured suite. A service can declare several
+ *  (workspace.yml testSuites), so its name alone collides — two suites under one
+ *  service would share a slot and the later would mask the other instead of both
+ *  counting. */
+export function suiteIdOf(cfg) {
+    return `${cfg.serviceName}::${cfg.framework}::${cfg.testRunCommand}`;
+}
+/** The files a selector narrowed to individual tests. A bare `file` selector runs
+ *  the whole file; `file::test title` does not. */
+export function partialFilesFrom(selectors) {
+    const out = new Set();
+    for (const sel of selectors) {
+        const [file, ...rest] = sel.split("::");
+        // A trailing `::` names no title, so every builder drops it and the whole file
+        // runs. Treating it as partial would throw away a real status.
+        if (rest.join("::").trim() === "")
+            continue;
+        const trimmed = file.trim();
+        if (trimmed)
+            out.add(trimmed);
+    }
+    return out;
+}
 /** Appends a run record into UnifiedAnalysisState.externalTestResults for the
  *  given repo section. Best-effort: a state failure never fails the run. */
 async function persistExternalRun(stateFile, repository, record) {
@@ -463,7 +552,7 @@ async function persistExternalRun(stateFile, repository, record) {
  * fully injectable for tests.
  */
 export async function runExternalTests(params, deps = defaultDeps) {
-    const mode = params.mode ?? "confirm";
+    const mode = params.mode;
     const selectors = params.testSelectors ?? [];
     let effective = selectors;
     let truncated = false;
@@ -486,6 +575,11 @@ export async function runExternalTests(params, deps = defaultDeps) {
         if (params.stateFile) {
             await persistExternalRun(params.stateFile, params.repository, {
                 ...skippedRun,
+                files: [],
+                // No run config resolved, so there is no suite to identify — and a service
+                // name here would read as one. The reader never keys on it (the record is
+                // skipped, and skipped records are dropped first), so name it for what it is.
+                suite: NO_RUNNABLE_SUITE,
                 mode,
                 ranAt: deps.now(),
                 repository: params.repository,
@@ -518,6 +612,8 @@ export async function runExternalTests(params, deps = defaultDeps) {
             if (params.stateFile) {
                 await persistExternalRun(params.stateFile, params.repository, {
                     ...routedOut,
+                    files: [],
+                    suite: suiteIdOf(cfg),
                     mode,
                     ranAt: deps.now(),
                     repository: params.repository,
@@ -542,8 +638,13 @@ export async function runExternalTests(params, deps = defaultDeps) {
         };
         perSuite.push(suiteResult);
         if (params.stateFile) {
+            const files = summarizeFilesInRun(suiteResult.results, {
+                partialFiles: partialFilesFrom(routing.kept),
+            });
             await persistExternalRun(params.stateFile, params.repository, {
                 ...suiteResult,
+                files,
+                suite: suiteIdOf(cfg),
                 mode,
                 ranAt: deps.now(),
                 repository: params.repository,
@@ -575,8 +676,7 @@ export function registerRunExistingTestsTool(server) {
                 .describe("Spec files to run (paths relative to the run command's cwd), optionally `file::test title` to also filter by name. Omit to run the whole configured suite."),
             mode: z
                 .enum(["confirm", "verify"])
-                .optional()
-                .describe("confirm (default): scoped run to establish which tests a change breaks; disables the suite's fail-fast cap (e.g. Playwright's maxFailures) so ALL breakages are counted. verify: re-run after an edit."),
+                .describe("REQUIRED, and the only thing that marks a run as before or after the edit: confirm = the pre-edit baseline (a scoped run establishing which tests the change breaks; disables the suite's fail-fast cap, e.g. Playwright's maxFailures, so ALL breakages are counted), verify = the re-run after you edited. Passing confirm for a post-edit run overwrites the baseline with the fixed result, and the report then shows a repair as having never been broken."),
             project: z
                 .string()
                 .optional()
@@ -635,7 +735,7 @@ export function registerRunExistingTestsTool(server) {
             AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {
                 workspacePath: params.workspacePath,
                 service: params.service ?? "",
-                mode: params.mode ?? "confirm",
+                mode: params.mode,
             }).catch((err) => {
                 logger.warning("Analytics event failed", { error: String(err) });
             });

package/build/tools/submitReportTool.js

@@ -84,6 +84,96 @@ const MAINTENANCE_CHANGE_ACTIONS = new Set([
     DriftAction.Regenerate,
     DriftAction.Delete,
 ]);
+const EXTERNAL_TO_EXECUTION_STATUS = {
+    pass: TestExecutionStatus.Pass,
+    fail: TestExecutionStatus.Fail,
+    error: TestExecutionStatus.Error,
+};
+/** Does this run's file outcome describe `knownPath`?
+ *
+ *  `absoluteFile` is the framework's own path resolved against the root it reported,
+ *  so it names one file and is compared exactly. An outcome without one is refused:
+ *  a relative spelling is a suffix, and a suffix cannot separate
+ *  `packages/a/tests/x.spec.ts` from `packages/b/tests/x.spec.ts`. Guessing there and
+ *  guarding the guess is what this deliberately does not do — a missing status is a
+ *  row that reads Unknown, which is what it reads today; a wrong one is a false claim
+ *  about someone's tests.
+ *
+ *  The cost is real and accepted: a suite running inside a container reports the
+ *  container's root, which cannot equal the host path discovery recorded, so those
+ *  repos get no status until the run records a root the report can line up. */
+function outcomeMatches(outcome, knownPath) {
+    return (typeof outcome.absoluteFile === "string" &&
+        path.resolve(outcome.absoluteFile) === path.resolve(knownPath));
+}
+/** One line describing what the suite saw, for a row whose status came from the run
+ *  rather than from an execution the agent drafted a summary for. Without it the
+ *  rendered cell is a bare `Error` or `Pass` with nothing behind it. */
+function describeOutcome(o) {
+    // Defended like the record reads around it: a malformed outcome must not throw
+    // out of the handler and take the whole report submission with it.
+    const c = o.counts ?? { pass: 0, fail: 0, error: 0, skipped: 0 };
+    const parts = [
+        c.pass ? `${c.pass} passed` : "",
+        c.fail ? `${c.fail} failed` : "",
+        c.error ? `${c.error} errored` : "",
+        c.skipped ? `${c.skipped} skipped` : "",
+    ].filter(Boolean);
+    const head = `${parts.join(", ") || "no tests reported"} in the repo's own suite`;
+    // Adapter messages carry stack traces, so collapse whitespace before truncating:
+    // an embedded newline would break the maintenance row this lands in.
+    const first = (o.errors ?? [])[0]?.replace(/\s+/g, " ").trim();
+    return first ? `${head} — ${first.slice(0, 200)}` : head;
+}
+function externalStatusesFor(records, knownPath) {
+    // LATEST per suite, then WORST across suites. Both halves matter: a suite re-run
+    // after another edit supersedes its own earlier verdict, so an obsolete failure
+    // must not outlive the fix that repaired it; but two different suites can each
+    // own the same file, and a pass in one must not mask a failure in the other.
+    const RANK = { pass: 0, fail: 1, error: 2 };
+    const statusIn = (mode) => {
+        const latestPerSuite = new Map();
+        for (const record of records ?? []) {
+            if (record.mode !== mode || record.skipped || !record.environmentHealthy)
+                continue;
+            // A suite that stopped on --max-failures left tests unrun and does not say
+            // which, so no file it touched can stand as covered. The adapter reports this;
+            // it is not a guess. (The tool forces --max-failures=0 for confirm, so this is
+            // reachable on a verify re-run under the repo's own cap.)
+            if (record.diagnostics?.maxFailuresBail)
+                continue;
+            const hit = (record.files ?? []).find((o) => outcomeMatches(o, knownPath));
+            if (!hit)
+                continue;
+            const key = record.suite ?? "";
+            const prior = latestPerSuite.get(key);
+            // `ranAt` is an ISO timestamp, so a lexical compare is chronological. The
+            // newest run wins BEFORE it is judged: a later partial re-run supersedes an
+            // earlier complete one, and must leave the row unknown rather than let the
+            // stale complete outcome stand in for it.
+            if (!prior || record.ranAt >= prior.at)
+                latestPerSuite.set(key, { at: record.ranAt, outcome: hit });
+        }
+        let worst;
+        for (const { outcome } of latestPerSuite.values()) {
+            // `skipped` means every test in the file was skipped — it was not exercised,
+            // and TestExecutionStatus.Skipped means a deliberate decision not to run it.
+            if (!outcome.complete || outcome.status === "skipped")
+                continue;
+            if (!worst || RANK[outcome.status] > RANK[worst.status])
+                worst = outcome;
+        }
+        return worst;
+    };
+    const before = statusIn("confirm");
+    const after = statusIn("verify");
+    return {
+        before: before && EXTERNAL_TO_EXECUTION_STATUS[before.status],
+        after: after && EXTERNAL_TO_EXECUTION_STATUS[after.status],
+        beforeDetail: before && describeOutcome(before),
+        afterDetail: after && describeOutcome(after),
+    };
+}
 const TOOL_NAME = "skyramp_submit_report";
 const DEFAULT_COMMIT_MESSAGE = "Added recommendations by Skyramp Testbot.";
 // Per-repo attribution. In a multi-repo run, every report item carries the
@@ -553,14 +643,14 @@ const testMaintenanceSchema = z.object({
         "For passing runs: count and timing, e.g. '4 passed in 15.09s'. " +
         "For failing runs: failure name and one-line root cause, e.g. " +
         "'FAILED test_foo — assert 403 got 200, auth middleware not enforced'. " +
-        "Empty string for VERIFY/IGNORE entries where no before-execution was run."),
+        "Pass an empty string when you ran nothing yourself — the server then fills this from the repo suite's own run — and for VERIFY/IGNORE entries where nothing ran at all. Omit the whole entry, never this field."),
     afterDetails: z
         .string()
         .describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
         "For passing runs: count and timing, e.g. '5 passed in 10.96s'. " +
         "For failing runs: failure name and one-line root cause, e.g. " +
         "'FAILED test_foo — check_schema fails, order_id=1 has discount from prior PATCH test'. " +
-        "Empty string for VERIFY/IGNORE/DELETE entries where no after-execution was run."),
+        "Pass an empty string when you ran nothing yourself — the server then fills this from the repo suite's own run — and for VERIFY/IGNORE/DELETE entries where nothing ran at all. Omit the whole entry, never this field."),
     // Server-populated from stateFile execution records — never supplied by the LLM.
     beforeStatus: z.nativeEnum(TestExecutionStatus),
     afterStatus: z.nativeEnum(TestExecutionStatus),
@@ -1011,12 +1101,42 @@ export function registerSubmitReportTool(server) {
                     : MAINTENANCE_CHANGE_ACTIONS.has(m.action)
                         ? TestExecutionStatus.Unknown
                         : TestExecutionStatus.Skipped;
-                const beforeStatus = recorded?.executionBefore?.status ?? defaultBeforeStatus;
-                const afterStatus = recorded?.executionAfter?.status ?? defaultAfterStatus;
+                // A suite run answers only for a row whose default is Unknown. VERIFY and
+                // IGNORE are Skipped by decision, not for want of a result, and testbot
+                // drops both from the rendered table.
+                const external = externalStatusesFor(stateData.externalTestResults, m.testFilePath);
+                const beforeStatus = recorded?.executionBefore?.status ??
+                    (defaultBeforeStatus === TestExecutionStatus.Unknown
+                        ? external.before
+                        : undefined) ??
+                    defaultBeforeStatus;
+                const afterStatus = recorded?.executionAfter?.status ??
+                    (defaultAfterStatus === TestExecutionStatus.Unknown
+                        ? external.after
+                        : undefined) ??
+                    defaultAfterStatus;
                 // Trim before checking — a whitespace-only string is semantically blank and
                 // must not satisfy the "drafted a summary" requirement.
-                const beforeDetails = detail?.beforeDetails?.trim() ?? "";
-                const afterDetails = detail?.afterDetails?.trim() ?? "";
+                // The agent's drafted summary wins; otherwise a status that came from
+                // the suite run carries the run's own line, so the rendered cell is
+                // never a bare verdict with nothing behind it.
+                //
+                // Only when there is NO recorded execution. A file the agent ran itself
+                // owes a drafted summary, and filling it here would satisfy the check
+                // below on the agent's behalf whenever the two statuses happened to
+                // agree — silently dropping the requirement.
+                //
+                // And only where the STATUS came from the suite too. A VERIFY or DELETE
+                // row keeps its Skipped default, so a line saying "1 passed in the repo's
+                // own suite" beside it contradicts the very cell it annotates.
+                const beforeFromSuite = defaultBeforeStatus === TestExecutionStatus.Unknown &&
+                    !recorded?.executionBefore;
+                const afterFromSuite = defaultAfterStatus === TestExecutionStatus.Unknown &&
+                    !recorded?.executionAfter;
+                const beforeDetails = detail?.beforeDetails?.trim() ||
+                    (beforeFromSuite ? (external.beforeDetail ?? "") : "");
+                const afterDetails = detail?.afterDetails?.trim() ||
+                    (afterFromSuite ? (external.afterDetail ?? "") : "");
                 if (recorded?.executionBefore && !beforeDetails)
                     missingDetails.push(`${displayName} (beforeDetails)`);
                 if (recorded?.executionAfter && !afterDetails)

package/build/types/ExternalTestExecution.d.ts

@@ -11,8 +11,15 @@ export type ExternalTestStatus = "pass" | "fail" | "error" | "skipped";
 export interface ExternalTestResult {
     /** Human-readable identity, e.g. `<file> › <describe…> › <title>`. */
     testId: string;
-    /** Spec file the test lives in. */
+    /** Spec file the test lives in, in the framework's own spelling — usually absolute
+     *  for jest and mocha, relative to the framework's root for Playwright and pytest.
+     *  No framework guarantees which, so never assume: use `absoluteFile` when present. */
     file: string;
+    /** `file` resolved against the root the framework reported, when it reported one.
+     *  A relative spelling alone cannot identify a file: `tests/Button.test.tsx` fits
+     *  every package that has one. Consumers matching a result to a known path must
+     *  prefer this and compare exactly. */
+    absoluteFile?: string;
     status: ExternalTestStatus;
     /** Failure/error text (ANSI-stripped). Present only for fail/error. */
     message?: string;
@@ -55,10 +62,69 @@ export interface ParsedExternalRun {
     /** Why the run was skipped (present only when `skipped`). */
     skipReason?: string;
 }
+/**
+ * What one run established about ONE spec file. Computed by the run itself, which
+ * knows what it asked the framework for; the report then reads this instead of
+ * re-deriving coverage from the flat result list, where the facts below are no
+ * longer recoverable.
+ */
+export interface ExternalFileOutcome {
+    /** The framework's own spelling, kept for diagnostics. */
+    file: string;
+    /** `file` resolved against the root the framework reported, when it reported one.
+     *  Present here exactly when it was present on the results. */
+    absoluteFile?: string;
+    /** Worst outcome across the file's tests: error > fail > pass. `skipped` means
+     *  every test in it was skipped, so the file was not exercised at all. */
+    status: ExternalTestStatus;
+    /** How many tests produced each outcome. Rendered into the maintenance row's
+     *  before/after detail when the agent drafted none. */
+    counts: {
+        pass: number;
+        fail: number;
+        error: number;
+        skipped: number;
+    };
+    /**
+     * Did this run exercise every TEST in the file? False when a `file::test title`
+     * selector narrowed it — the one narrowing this tool applies per file, and so the
+     * only one it can state with certainty. A partial run says nothing about the tests
+     * it did not reach, so it cannot stand as the file's status.
+     *
+     * Two narrowings are deliberately NOT reflected here, and the reader compensates
+     * for one of them:
+     * - A `file::title` selector also becomes a run-WIDE --grep / -t on every
+     *   framework except pytest, so a co-selected file ran only its matching tests
+     *   while this says complete. Known and accepted.
+     * - A suite that bailed on --max-failures abandoned tests it cannot name. That one
+     *   IS reported, in `diagnostics.maxFailuresBail`, and the report skips such a
+     *   record outright rather than reading any file from it.
+     *
+     * A Playwright `project` scope does NOT make a file incomplete: every test in it
+     * still ran, in one project rather than all of them. That is a different axis, and
+     * discarding the status over it throws away the run's whole verdict — measured on
+     * eval run 33280302496, where it turned a real Error -> Pass into Unknown.
+     *
+     * Known gap: a repo whose runner bails on its own (a jest or mocha `bail` in
+     * config rather than on the command line) is not detectable from the report,
+     * and only Playwright reports a --max-failures bail back to us.
+     */
+    complete: boolean;
+    /** Failure/error text for the tests that did not pass. */
+    errors: string[];
+    durationMs: number;
+}
 /** One persisted run of the external suite, appended to
  *  UnifiedAnalysisState.externalTestResults so drift analysis / the report can
  *  fold in CONFIRMED failures instead of guesses. */
 export interface ExternalTestRunRecord extends ParsedExternalRun {
+    /** Per-file outcome, one entry per spec file this run produced results for.
+     *  Empty for a skipped run. */
+    files: ExternalFileOutcome[];
+    /** Which configured suite produced this record. A repo can declare several, and
+     *  without this a re-run of ONE suite cannot be told from a second suite's first
+     *  run — so a superseded failure would outlive the fix that repaired it. */
+    suite: string;
     mode: "confirm" | "verify";
     /** ISO timestamp when the run completed. */
     ranAt: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/mcp",
-  "version": "0.3.6-rc.1",
+  "version": "0.3.6-rc.2.ac20",
   "main": "build/index.js",
   "exports": {
     ".": "./build/index.js",