@skyramp/mcp 0.3.8 → 0.4.0-rc.1

package/build/recommendation/verifierContracts.js

@@ -0,0 +1,263 @@
+/** What each plan-time check SAYS to the agent: the objection it emits, and the
+ *  one-line question that closes it. Each suggestion also carries the judgment
+ *  the check itself cannot make — the agent reads it at the moment it matters,
+ *  when the objection arrives, and the skill does not repeat it. What to put in
+ *  a field, and what the check reads, is stated once in that field's own
+ *  `describe` in the plan tool's schema. A check compares a change the plan states
+ *  against something the server already holds, needs no new parser or threshold,
+ *  and can be answered by the agent. */
+// Evidence is deliberately absent from every contract. Each message below is
+// emitted beside an evidence line built from the planned test's own values, so the
+// evidence stays computed in the verifier.
+export const CHANGED_FILE_CONTRACT = {
+    id: "changedFile",
+    declarationFields: ["declarations.changedFile", "declarations.screenEvidence.file"],
+    objections: {
+        notStated: {
+            message: "This test does not state which changed file it targets.",
+            suggestion: "Name the repository-relative file this test is about in `changedFile`, spelled as the diff spells it.",
+        },
+        unrelated: {
+            message: "The file this test names as its changed file is not one this pull request changed.",
+            suggestion: "Name a file from this PR's own diff in `changedFile`, repository-relative and spelled as the diff spells it — or say why this test belongs in the plan without one.",
+        },
+        unknownRepository: {
+            message: "This planned test names a repository this run did not analyze.",
+            suggestion: "Name one of the repositories listed in the evidence, or leave `repository` out on a single-repository run. A name nothing analyzed leaves the planned test with no diff to be checked against.",
+        },
+        screenEvidenceUnrelated: {
+            message: "The file this UI test cites for its screen is not one this pull request changed.",
+            suggestion: "Which changed file renders the screen this test opens — or which changed screen should it open instead? Name that file in `screenEvidence.file`, repository-relative, spelled as the diff spells it.",
+        },
+    },
+    suggestion: "Which file that this pull request changed does this test target?",
+};
+export const SCREEN_ROUTE_CONTRACT = {
+    id: "screenRoute",
+    declarationFields: ["declarations.elements.pageUrl", "declarations.screenEvidence.file", "declarations.changedFile"],
+    objections: {
+        mismatch: {
+            message: "The page this UI test opens is not a page that renders the changed file it is about.",
+            suggestion: "Open one of the routes in the evidence, or name the changed file that does render this page in `screenEvidence.file`. `skyramp_resolve_screen` gives the url to open, with the route parameters it can fill. A trace recorded on a screen the diff did not change proves nothing about the change.",
+        },
+        unmounted: {
+            message: "No route this run found renders the file this UI test is about.",
+            suggestion: "If a route in this PR mounts it, name that route in `elements.pageUrl`. If nothing mounts it, the component is unintegrated: say so in one line and keep the planned test out of the browser.",
+        },
+    },
+    suggestion: "Does the page this test opens render the changed file the test is about?",
+};
+export const ENDPOINT_GROUNDED_CONTRACT = {
+    id: "endpointGrounded",
+    declarationFields: ["steps[]", "testType", "declarations.routes"],
+    objections: {
+        uncited: {
+            message: "A call this test makes cites no file that declares it.",
+            suggestion: "Add a `routes` entry naming the repository file that declares it — the route or operation need not appear verbatim in that file. A route the endpoint catalog missed is evidence of nothing: cite the file and keep the planned test.",
+        },
+        disqualifiedCitation: {
+            message: "The file this test cites for a call cannot be the file that declares it.",
+            suggestion: "Cite the source file that declares it — not a test, not built output, not documentation. Any language or format is accepted, and the route or operation need not appear verbatim in the file.",
+        },
+        uncitedAndDisqualified: {
+            message: "This test has a call with no citation, and a call whose cited file cannot be the file that declares it.",
+            suggestion: "Add `routes` for the call that has none, and point the other citation at the source file that declares its route or operation. Neither need appear verbatim in the file.",
+        },
+    },
+    suggestion: "Which repository file declares this route or operation, and does an entry name that step's `order`?",
+};
+export const STATED_DIFFERENCE_CONTRACT = {
+    id: "statedDifference",
+    declarationFields: ["steps[]", "testType", "scenarioName", "declarations.differsFrom"],
+    objections: {
+        unexplainedPair: {
+            message: "Another planned test in this plan tests the same endpoint and neither says how they differ.",
+            suggestion: "Two tests on one endpoint is a legitimate plan. This asks HOW they differ, not whether one should go. Add a `differsFrom` entry naming the other planned test and what this one asserts that it does not — or, if one really is redundant, drop it.",
+        },
+        unknownPartner: {
+            message: "A `differsFrom` entry names a partner that is not a planned test in this plan.",
+            suggestion: "Spell the partner's `scenarioName` exactly as it appears in your own planned test list — the match is exact apart from surrounding space, so a difference in case names nothing.",
+        },
+        subjectNotStated: {
+            message: "This test makes more than one call and does not say which one it is about.",
+            suggestion: "Set `declarations.stepUnderTest` to the `order` of the step under test. The other calls are setup, and nothing on the server ranks them for you.",
+        },
+    },
+    suggestion: "Which call is this test about, and what does it assert that the other planned test on the same endpoint does not?",
+};
+export const EXISTING_COVERAGE_CONTRACT = {
+    id: "existingCoverage",
+    declarationFields: ["declarations.existingTests"],
+    objections: {
+        citedTestMissing: {
+            message: "This planned test names an existing test file that is not in the checkout.",
+            suggestion: "Name the file repository-relative, spelled as the analysis lists it. Drop the entry if it names a test you did not read — naming none is an answer too.",
+        },
+    },
+    suggestion: "Which existing tests did you read on this change, and how does this planned test differ from them?",
+};
+export const EXPECTED_OUTCOME_CONTRACT = {
+    id: "expectedOutcome",
+    declarationFields: ["category", "declarations.expected.outcome", "declarations.expected.why"],
+    objections: {
+        // `{category}` is the planned test's own `category` value, quoted back to it so
+        // the agent recognises the value it sent.
+        passInShouldFailCategory: {
+            message: 'This planned test is categorised "{category}" but expects to pass on the app as it stands.',
+            suggestion: "If it catches a defect, it should be red until the defect is fixed — say so. If it pins behaviour that is already correct, change the category.",
+        },
+        // Two branches because the old single one told an agent to fix its endpoint
+        // path when the unmounted route WAS the critical defect it had just reported
+        // — four objections for being honest about `reachedBehavior` (run
+        // 33988304641, P11), which the run that wrote `true` never drew.
+        executed: {
+            message: "This test failed, but it did not reach the behavior it was written for — it fell over earlier. A failure like this is not a caught bug.",
+            suggestion: "If the blocker is the test's own setup — sign-in, token, a wrong path — fix it and run the test again. If the blocker is the defect this run reports (a route that is not mounted, a server error before the behaviour), answer this objection naming that issue: the failure then stands as evidence of that issue, not as a caught bug.",
+        },
+    },
+    suggestion: "Should this test fail until the defect is fixed, or is the category wrong for a test that pins behaviour the app already has?",
+};
+export const UI_ELEMENT_GROUNDED_CONTRACT = {
+    id: "uiElementGrounded",
+    declarationFields: [
+        "testType",
+        "steps[]",
+        "declarations.elements.items",
+        "declarations.elements.pageUrl",
+        "declarations.asserts",
+        "scenarioName",
+        "description",
+        "declarations.expected.why",
+    ],
+    objections: {
+        nogrounding: {
+            message: "This UI test names no `elements.items`, so nothing says the elements it targets are on the page.",
+            suggestion: "Copy the elements this test targets out of the capture of the page it runs on into `elements.items`, and set `elements.pageUrl` to that page. If no captured page holds this test's elements — the changed component mounts nowhere the run reached, or the screen is behind a gate this run could not pass — answer this objection saying which, and leave `elements.items` null.",
+        },
+        duplicate: {
+            message: "`elements.items` names the same element more than once.",
+            suggestion: "List each element once. Two entries that agree on role, name and identifiers describe one element, and the repeat states nothing the first entry did not.",
+        },
+        pagecontext: {
+            message: "This UI test names `elements.items` but no page they came from.",
+            suggestion: "Set `elements.pageUrl` to the `url` of the capture the elements were lifted from. Without it there is no page to look them up on.",
+        },
+        ungrounded: {
+            message: "An element this UI test targets is in no blueprint captured for that page.",
+            suggestion: "Cite what a capture shows you, never what the screen ought to have. A capture shows the page as it stood when it was taken, so an element behind a modal, a dropdown or an accordion is legitimately absent — answer this objection naming the action that reveals it. Otherwise copy the element from the capture as it is spelled there, or point `elements.pageUrl` at the page you actually captured.",
+        },
+    },
+    // Reconciled after the fix round, which is what the TO-RECONCILE marker here
+    // was waiting on. It phrases the question and names no phrase that closes the
+    // check — Ruling B removed exactly such a phrase from verifier 8's suggestion.
+    suggestion: "Which capture of the page does each declared element come from, and for one that is in no capture, which action on that page reveals it?",
+};
+export const COVERAGE_CONTRACT = {
+    id: "coverage",
+    declarationFields: [
+        "changes",
+        "declarations.changes",
+        "declarations.changedFile",
+        "declarations.routes",
+        "declarations.screenEvidence.file",
+        "declarations.startState",
+        "changes[].cases",
+        "changes[].surfaces",
+    ],
+    objections: {
+        change: {
+            message: "This declared change has no test in the plan.",
+            suggestion: "Plan a test that exercises it. If this run cannot write one, answer it and set `blocker` to what stopped the run — a service that is not running, a paired branch that no longer exists, the one credential the run holds. An answer with no blocker leaves this open: a change nothing prevented a test from reaching takes the test. Coverage per file is the basic minimum; coverage per change is what the plan is judged by, so a file covered by a shape test still leaves the change it makes untested.",
+        },
+        noChanges: {
+            message: "The plan declares no changes, so nothing says what the pull request must make different.",
+            suggestion: "List the behaviours the pull request description promises, one sentence each, and cite them from the planned tests that test them — or answer in one line that it promises none, which a docs-only or cosmetic change legitimately does.",
+        },
+        unknownChange: {
+            message: "A planned test cites a change id that this plan does not declare.",
+            suggestion: "Spell the change's `id` exactly as your own `changes` list gives it — the match is exact apart from surrounding space. Declare the change if it is missing.",
+        },
+        file: {
+            message: "This changed file has no test in the plan.",
+            suggestion: "Add a planned test that cites it — as `changedFile`, as a `routes` file, or as `screenEvidence.file` — or answer in ONE line why it needs none: config only, covered by the test on the route it serves, or style-only and covered by the UI test on the page it belongs to. This check is a floor, never a cap: it never asks you to plan fewer tests.",
+        },
+        // THE ONE OBJECTION NO ANSWER CLOSES, and `registerPlan` is what holds that.
+        // Twelve runs over 2026-09-04 and 05 missed the same kind of planted bug: a
+        // rule about a STORED value the new code reads and never wrote, which only a
+        // record already in another state reaches. Prose reached the agent twice and
+        // was skipped twice,
+        // so an answer here would close the objection and leave the bug.
+        // `{route}` is the mutated route, as `METHOD path`.
+        stateTest: {
+            message: "No test in this plan names the state the record is in when {route} mutates it.",
+            suggestion: "Plan one test that puts a record in a stated state — through the API, the UI, or data the app already holds; a record with no rows yet counts — then runs the mutation on it, and name that state as `startState`. If the route can meet a record that is not freshly created, use that state. Only a planned test closes this: there is no answer for it.",
+        },
+        // `{missing}` is the declared cases no step sends, as `param=value (expect)`.
+        // The check reads the values the citing tests send, so a case the agent
+        // declares and never sends is now visible: four runs on one fixture declared
+        // a 1-100 range and sent the upper end once, and one report claimed both ends
+        // anyway. What a case IS stays the agent's judgment — the field takes any
+        // parameter and any value, so nothing here has to know a kind of input in
+        // advance.
+        cases: {
+            message: "No test for this change sends the declared case {missing}.",
+            suggestion: "Add the missing values to a test that cites this change, or answer in one line why they need no test.",
+        },
+        // `{param}` is the parameter and `{value}` the placeholder it accepts. Read off
+        // the plan alone: a placeholder value is a record an earlier step created, so a
+        // route that takes it by id can also be given one that is not there. A DELETE on
+        // a collection member answered 200 for a user that was never a member, and no
+        // plan of six runs listed that rejection.
+        reject: {
+            message: "`{param}` accepts `{value}`, a record the plan creates, and no case rejects a record that is not there or belongs to another parent.",
+            suggestion: "Add a reject case on `{param}` with a value that is not that record and a step that sends it, or answer why the route cannot refuse it.",
+        },
+        // `{kind}` is the test types that reach the surface and `{surface}` is the
+        // surface itself. Run 34065056830 cited a page change — a colour filter the
+        // pull request added to a page — from an API test, and no test in the plan
+        // ever clicked the filter.
+        surface: {
+            message: "No {kind} test cites this change, which is on the {surface}.",
+            suggestion: "Add a test on that surface that cites this change, or answer in one line why that surface needs no test.",
+        },
+    },
+    suggestion: "Which planned test tests each behaviour this change promises and each file it touches, and for one nothing covers, why does it need none?",
+};
+export const REMOVED_ELEMENT_GUARDED_CONTRACT = {
+    id: "removedElementGuarded",
+    declarationFields: ["declarations.elements.items", "declarations.asserts"],
+    objections: {
+        // v1 resolved a removed element to a page in the server, and fell back to the
+        // root pages when the import graph found none, so that an element whose page
+        // could not be resolved still got a guard (SKYR-4339). v2 does not attribute
+        // pages, so the same protection reads the plan: the run picks the page, and
+        // has to pick one.
+        missing: {
+            message: "The diff removes `{element}` and no planned test says it is gone.",
+            suggestion: "Plan a UI test on the page that still renders, asserting this element is absent, and name it in that test's elements. Removing it is what this pull request ships, so the removal is what gets tested. If the diff shows it still rendering under a new identifier it is a rename, not a removal — say so as the answer.",
+        },
+    },
+    suggestion: "Does every element this diff removes have a planned test that asserts it is gone?",
+};
+/** In the order of `PLAN_TIME_VERIFIERS`, which decides the order objections are
+ *  reported in and carries no priority. References, not copies: iterating this
+ *  array, or indexing it with a variable, widens the element to the union of them
+ *  all and then no `objections` key resolves. */
+export const VERIFIER_CONTRACTS = [
+    CHANGED_FILE_CONTRACT,
+    ENDPOINT_GROUNDED_CONTRACT,
+    STATED_DIFFERENCE_CONTRACT,
+    EXISTING_COVERAGE_CONTRACT,
+    EXPECTED_OUTCOME_CONTRACT,
+    UI_ELEMENT_GROUNDED_CONTRACT,
+    SCREEN_ROUTE_CONTRACT,
+    COVERAGE_CONTRACT,
+    REMOVED_ELEMENT_GUARDED_CONTRACT,
+];
+/** Fills a contract sentence's `{placeholder}` spans from the run's or the
+ *  planned test's own values. A placeholder with no value is left as written rather
+ *  than blanked, so a missed key shows up as `{knob}` instead of reading as a gap
+ *  in an otherwise ordinary sentence. */
+export function fillPlaceholders(text, values) {
+    return text.replace(/\{(\w+)\}/g, (whole, key) => (key in values ? String(values[key]) : whole));
+}

package/build/recommendation/verifiers/changedFile.d.ts

@@ -0,0 +1,2 @@
+import { Verifier } from "../types.js";
+export declare const changedFile: Verifier;

package/build/recommendation/verifiers/changedFile.js

@@ -0,0 +1,82 @@
+import { allChangedFiles, changedFilesFor } from "../types.js";
+import { CHANGED_FILE_CONTRACT } from "../verifierContracts.js";
+import { changedPathSet, normalizeCitedPath, whyNotAChangedFile } from "./citedPath.js";
+/** Verifier 1. The files a planned test cites for the change it targets are files
+ *  this pull request really changed. NAMES ONLY — no diff content is read, so an
+ *  irrelevant file inside the diff cannot be caught. SILENT on a run with no
+ *  diff; where one repository has a diff and another does not, the reason says so.
+ *
+ *  The planned test used to declare the changed LINE here as well. It was recorded
+ *  and never compared against anything, and the reasoning a reviewer wants from
+ *  it belongs in the planned test's own description. */
+/** How a malformed declaration reads back to the agent. */
+function describeType(value) {
+    if (value === undefined)
+        return "missing";
+    if (value === null)
+        return "null";
+    if (Array.isArray(value))
+        return "an array";
+    return `a ${typeof value}`;
+}
+/** One objection, so every branch below reads the same way. */
+function raise(plannedTest, key, objection, evidence) {
+    return {
+        objectionId: `changedFile:${key}:${plannedTest?.plannedTestId}`,
+        verifier: "changedFile",
+        plannedTestId: plannedTest?.plannedTestId,
+        message: objection.message,
+        evidence,
+        suggestion: objection.suggestion,
+    };
+}
+export const changedFile = {
+    name: "changedFile",
+    run(registration, ctx) {
+        const objections = [];
+        // No diff anywhere in the run: no changed file to cite, so nothing to say.
+        if (allChangedFiles(ctx).length === 0)
+            return objections;
+        for (const plannedTest of registration.plannedTests) {
+            // Per planned test, from the planned test's OWN repository where it names one.
+            // Reading every repository's instead let a planned test for one repository
+            // cite a file only the other one changed and pass a check named for its
+            // repository.
+            const own = changedFilesFor(ctx, plannedTest);
+            if (own === undefined) {
+                objections.push(raise(plannedTest, "unknownRepository", CHANGED_FILE_CONTRACT.objections.unknownRepository, `this run analyzed ${Object.keys(ctx.changedFiles ?? {}).join(", ") || "no repository"}, and this planned test names "${String(plannedTest?.repository ?? "").trim()}"`));
+                continue;
+            }
+            const surfaces = changedPathSet(own, true);
+            // A deleted file is a changed surface but not a page: a screen that no
+            // longer exists cannot be opened, so the screen citation is compared
+            // against the files that survive the change.
+            const pages = changedPathSet(own, false);
+            // `screenEvidence.file` has to be a file the diff changed. Independent of
+            // the changed file's own result — a planned test can get both wrong, and each
+            // names its own fix.
+            const citedScreen = String(plannedTest?.declarations?.screenEvidence?.file ?? "").trim();
+            if (citedScreen && !pages.has(normalizeCitedPath(citedScreen))) {
+                const page = String(plannedTest?.declarations?.elements?.pageUrl ?? "").trim();
+                objections.push(raise(plannedTest, "screenEvidenceUnrelated", CHANGED_FILE_CONTRACT.objections.screenEvidenceUnrelated, `${pages.size} changed file(s) still in the tree. ${whyNotAChangedFile(citedScreen)}, for ${page ? `the test that opens ${page}` : "a test that names no page"}.`));
+            }
+            // The declared type holds only on the schema-validated path. A crash here
+            // ends the run; an objection lets the agent fix the field.
+            const raw = plannedTest?.declarations?.changedFile;
+            const cited = typeof raw === "string" ? raw.trim() : "";
+            if (!cited) {
+                objections.push(raise(plannedTest, "notStated", CHANGED_FILE_CONTRACT.objections.notStated, `changedFile is ${describeType(raw)}; ${surfaces.size} file(s) changed in this run and the plan does not say which one this test targets.`));
+                continue;
+            }
+            if (!surfaces.has(normalizeCitedPath(cited))) {
+                // A repository of the run has no diff, so "not among the changed files"
+                // is not a fact this check can establish for a planned test from it.
+                const unverifiable = ctx.reposWithoutDiff.length > 0
+                    ? ` No branch diff exists for ${ctx.reposWithoutDiff.join(", ")}, so a file from those changes could not be checked.`
+                    : "";
+                objections.push(raise(plannedTest, "unrelated", CHANGED_FILE_CONTRACT.objections.unrelated, `${surfaces.size} changed file(s). ${whyNotAChangedFile(cited)}.${unverifiable}`));
+            }
+        }
+        return objections;
+    },
+};

package/build/recommendation/verifiers/citedPath.d.ts

@@ -0,0 +1,12 @@
+import { ChangedFileName } from "../../utils/branchDiff.js";
+/** A cited path reduced to the spelling `git diff --name-status` produces: `./`,
+ *  backslashes, doubled and trailing slashes fold, and so do git's quoting of a
+ *  non-ASCII path and both unicode compositions. Case does NOT fold, and a path
+ *  leaving the repository normalises to empty, by `escapesRepo`. */
+export declare function normalizeCitedPath(raw: unknown): string;
+/** The changed paths a citation can name, in the same normalised spelling.
+ *  `includeDeleted` is the caller's decision: a deleted file is still a changed
+ *  surface a test can cover, and it is NOT a page a UI test can open. */
+export declare function changedPathSet(files: ChangedFileName[], includeDeleted: boolean): Set<string>;
+/** Why a citation is not one of the changed files. */
+export declare function whyNotAChangedFile(file: string): string;

package/build/recommendation/verifiers/citedPath.js

@@ -0,0 +1,35 @@
+import { unquoteIfQuoted } from "../../utils/branchDiff.js";
+import { escapesRepo } from "./endpointGrounded.js";
+/** A cited path reduced to the spelling `git diff --name-status` produces: `./`,
+ *  backslashes, doubled and trailing slashes fold, and so do git's quoting of a
+ *  non-ASCII path and both unicode compositions. Case does NOT fold, and a path
+ *  leaving the repository normalises to empty, by `escapesRepo`. */
+export function normalizeCitedPath(raw) {
+    if (typeof raw !== "string")
+        return "";
+    const slashed = unquoteIfQuoted(raw.trim()).normalize("NFC").replace(/\\/g, "/");
+    if (!slashed || escapesRepo(slashed))
+        return "";
+    const segments = slashed.split("/").filter((segment) => segment !== "" && segment !== ".");
+    return segments.join("/");
+}
+/** The changed paths a citation can name, in the same normalised spelling.
+ *  `includeDeleted` is the caller's decision: a deleted file is still a changed
+ *  surface a test can cover, and it is NOT a page a UI test can open. */
+export function changedPathSet(files, includeDeleted) {
+    const paths = new Set();
+    for (const file of Array.isArray(files) ? files : []) {
+        if (!includeDeleted && file?.deleted)
+            continue;
+        const path = normalizeCitedPath(file?.path);
+        if (path)
+            paths.add(path);
+    }
+    return paths;
+}
+/** Why a citation is not one of the changed files. */
+export function whyNotAChangedFile(file) {
+    return escapesRepo(file)
+        ? `cited path "${file}" points outside the repository`
+        : `cited file ${file} is not among the files this pull request changed`;
+}

package/build/recommendation/verifiers/coverage.d.ts

@@ -0,0 +1,7 @@
+import { Verifier } from "../types.js";
+/** The completeness check, over three surfaces: every declared behaviour, every
+ *  mutated route, and every changed file needs a test in the plan. More is
+ *  welcome, fewer is the objection. ONE RULE for the file half, no classification
+ *  — every changed file is a surface unless it cannot be product code, including a
+ *  removed one. A FLOOR, never a cap. Only the file half needs a diff. */
+export declare const coverage: Verifier;