@skyramp/mcp 0.4.0 → 0.4.1-rc.2

package/build/recommendation/verifiers/requirementSourced.d.ts

@@ -0,0 +1,2 @@
+import { Verifier } from "../types.js";
+export declare const requirementSourced: Verifier;

package/build/recommendation/verifiers/requirementSourced.js

@@ -0,0 +1,168 @@
+import { normalizeCategory } from "../../types/TestRecommendation.js";
+import { REQUIREMENT_SOURCED_CONTRACT, fillPlaceholders } from "../verifierContracts.js";
+import { normalizeCitedPath } from "./citedPath.js";
+import { appearsIn, searchable } from "../pullRequestText.js";
+/** Verifier 10. A change names where the agent read it, and the server reads the
+ *  name back.
+ *
+ *  `source` is the record of where a change came from, and the server holds the
+ *  pull request's own words, so it reads the claim back against them: a `spec:` path
+ *  has to name a file the checkout holds AND a file the pull request names, and a
+ *  change read from the pull request has to quote words the pull request carries.
+ *
+ *  A conflict is a disagreement between a requirement and the code. A change read
+ *  off the diff IS the code, so there is no second side for it to disagree with.
+ *
+ *  The plan tool always supplies the pull request: the testbot fetches its prompt
+ *  from this same server, so the text is in the process. Both fields blank means
+ *  the run had no title and no description. */
+const { unsourced: UNSOURCED, unreadable: UNREADABLE, fromDiff: FROM_DIFF, noChange: NO_CHANGE, notNamed: NOT_NAMED, emptyPullRequest: EMPTY_PULL_REQUEST, notQuoted: NOT_QUOTED, unquoted: UNQUOTED, } = REQUIREMENT_SOURCED_CONTRACT.objections;
+const CONFLICT_CATEGORY = "requirement_conflict";
+function readSource(raw) {
+    const none = (kind) => ({ kind, file: "", citedAs: "" });
+    const source = typeof raw === "string" ? raw.trim() : "";
+    if (source.length === 0)
+        return none("unknown");
+    if (/^(pr-title|pr-description)$/i.test(source))
+        return none("pullRequest");
+    if (/^diff$/i.test(source))
+        return none("diff");
+    const named = /^spec:(.*)$/i.exec(source);
+    if (!named)
+        return none("unknown");
+    const citedAs = named[1]
+        .split(/[#§]/)[0]
+        .replace(/(?::\d+)+(?:-\d+)?$/, "")
+        .trim();
+    const file = normalizeCitedPath(citedAs);
+    // `spec:` with nothing after it names no file, so there is nothing to check and
+    // nothing to trust either.
+    return file ? { kind: "spec", file, citedAs } : none("unknown");
+}
+/** A source that stands for a requirement: the pull request, or a requirements file. */
+const statesARequirement = (source) => source.kind === "pullRequest" || source.kind === "spec";
+const text = (value) => (typeof value === "string" ? value.trim() : "");
+const list = (value) => (Array.isArray(value) ? value : []);
+export const requirementSourced = {
+    name: "requirementSourced",
+    run(registration, ctx) {
+        const changes = list(registration?.changes);
+        const plannedTests = list(registration?.plannedTests);
+        const objections = [];
+        const sourceById = new Map();
+        const quoteById = new Map();
+        const pullRequestText = searchable(ctx.pullRequest);
+        for (const change of changes) {
+            const changeId = text(change?.id) || "(unnamed)";
+            const written = text(change?.source);
+            const source = readSource(written);
+            if (text(change?.id)) {
+                sourceById.set(text(change.id), source);
+                quoteById.set(text(change.id), typeof change?.quote === "string" ? change.quote : undefined);
+            }
+            if (source.kind === "unknown") {
+                objections.push({
+                    objectionId: `requirementSourced:unsourced:${changeId}`,
+                    verifier: "requirementSourced",
+                    message: fillPlaceholders(UNSOURCED.message, { change: changeId }),
+                    evidence: `change ${changeId} states source \`${written || "(blank)"}\``,
+                    suggestion: UNSOURCED.suggestion,
+                });
+                continue;
+            }
+            if (source.kind === "pullRequest") {
+                if (pullRequestText.length > 0)
+                    continue;
+                objections.push({
+                    objectionId: `requirementSourced:emptyPullRequest:${changeId}`,
+                    verifier: "requirementSourced",
+                    message: fillPlaceholders(EMPTY_PULL_REQUEST.message, { change: changeId }),
+                    evidence: `change ${changeId} states source \`${written}\`; this run rendered its prompt with no title and no description`,
+                    suggestion: EMPTY_PULL_REQUEST.suggestion,
+                });
+                continue;
+            }
+            if (source.kind !== "spec")
+                continue;
+            if (typeof ctx?.citedFileExists === "function" && !ctx.citedFileExists(source.file)) {
+                objections.push({
+                    objectionId: `requirementSourced:unreadable:${changeId}`,
+                    verifier: "requirementSourced",
+                    message: fillPlaceholders(UNREADABLE.message, { change: changeId, source: written }),
+                    evidence: `change ${changeId} states source \`${written}\`; ${source.file} is not a file in the run's repositories`,
+                    suggestion: UNREADABLE.suggestion,
+                });
+                continue;
+            }
+            // A requirements file the pull request never names is a file the agent found by
+            // looking through the repository, which is not reading a requirement.
+            if (appearsIn(pullRequestText, source.file) || appearsIn(pullRequestText, source.citedAs))
+                continue;
+            objections.push({
+                objectionId: `requirementSourced:notNamed:${changeId}`,
+                verifier: "requirementSourced",
+                message: fillPlaceholders(NOT_NAMED.message, { change: changeId, source: written }),
+                evidence: `change ${changeId} states source \`${written}\`; neither the title nor the description names ${source.file}`,
+                suggestion: NOT_NAMED.suggestion,
+            });
+        }
+        for (const plannedTest of plannedTests) {
+            if (normalizeCategory(plannedTest?.scenario?.category) !== CONFLICT_CATEGORY)
+                continue;
+            const plannedTestId = text(plannedTest?.plannedTestId);
+            const named = plannedTestId ? { plannedTestId } : {};
+            // An id no change carries names nothing, so it says no more than citing
+            // nothing does. Both leave the requirement with no stated origin.
+            const cited = list(plannedTest?.declarations?.changes)
+                .map((changeId) => text(changeId))
+                .filter((changeId) => sourceById.has(changeId));
+            if (cited.length === 0) {
+                objections.push({
+                    objectionId: `requirementSourced:noChange:${plannedTestId || "plan"}`,
+                    verifier: "requirementSourced",
+                    ...named,
+                    message: NO_CHANGE.message,
+                    evidence: `planned test ${plannedTestId || "(unnamed)"} reports a requirement conflict and cites no change this plan declares`,
+                    suggestion: NO_CHANGE.suggestion,
+                });
+                continue;
+            }
+            if (cited.some((changeId) => statesARequirement(sourceById.get(changeId)))) {
+                // The quote is what makes the claim checkable. One objection per test, on the
+                // first cited change that does not carry one the pull request holds.
+                const offending = cited.find((changeId) => {
+                    if (sourceById.get(changeId)?.kind !== "pullRequest")
+                        return false;
+                    const quote = quoteById.get(changeId);
+                    return quote === undefined || !appearsIn(pullRequestText, quote);
+                });
+                if (offending === undefined)
+                    continue;
+                const quote = quoteById.get(offending);
+                const objection = quote === undefined ? UNQUOTED : NOT_QUOTED;
+                objections.push({
+                    objectionId: `requirementSourced:notQuoted:${plannedTestId || "plan"}`,
+                    verifier: "requirementSourced",
+                    ...named,
+                    message: fillPlaceholders(objection.message, { change: offending }),
+                    evidence: quote === undefined
+                        ? `planned test ${plannedTestId || "(unnamed)"} cites ${offending}, which states no \`quote\``
+                        : `planned test ${plannedTestId || "(unnamed)"} cites ${offending}, whose quote \`${quote}\` is in neither the title nor the description`,
+                    suggestion: objection.suggestion,
+                });
+                continue;
+            }
+            objections.push({
+                objectionId: `requirementSourced:fromDiff:${plannedTestId || "plan"}`,
+                verifier: "requirementSourced",
+                ...named,
+                message: FROM_DIFF.message,
+                evidence: `planned test ${plannedTestId || "(unnamed)"} cites ${cited
+                    .map((changeId) => `${changeId} (kind ${sourceById.get(changeId)?.kind})`)
+                    .join(", ")}`,
+                suggestion: FROM_DIFF.suggestion,
+            });
+        }
+        return objections;
+    },
+};

package/build/services/TestGenerationService.js

@@ -438,7 +438,9 @@ The generated test file remains unchanged and ready to use as-is.
             if (generateOptions.authHeader === undefined) {
                 try {
                     const repoPath = generateOptions.outputDir || process.cwd();
-                    const resolved = await resolveAuthFromWorkspace(repoPath, generateOptions.authHeader, generateOptions.authScheme);
+                    const resolved = await resolveAuthFromWorkspace(repoPath, generateOptions.authHeader, generateOptions.authScheme, 
+                    // The directory the test is written into names the service it is for.
+                    generateOptions.outputDir);
                     if (resolved) {
                         generateOptions.authHeader = resolved.authHeader;
                         if (resolved.authScheme !== undefined) {

package/build/tools/code-refactor/codeReuseTool.js

@@ -31,7 +31,7 @@ const codeReuseSchema = z.object({
     verify: z
         .boolean()
         .default(false)
-        .describe("Verify a previously refactored test instead of returning the reuse prompt. POM path (browser tests): checks the test's POM calls against source; requires the discovery pass — this tool called for the same testFile without `verify` — to have run first. SkyrampUtils path (every other test type): stages the shared utils file for the output commit, checks its invariants, and — when skyramp_modularization recorded a baseline for this spec — compares the delivered assertion count and page.on('pageerror') guard against that hand-out, rejecting a net loss (API helpers: one helper per method+path, status-code-only assertions, method+resource names; browser helpers: actions and structural waits only, intent names). Call it after the reuse edits are written; skyramp_enhance_assertions and skyramp_execute_test refuse until it has passed."),
+        .describe("Verify a previously refactored test instead of returning the reuse prompt. POM path (browser tests): checks the test's POM calls against source; requires the discovery pass — this tool called for the same testFile without `verify` — to have run first. SkyrampUtils path (every other test type): stages the shared utils file for the output commit, checks its invariants, and — when skyramp_modularization recorded a baseline for this spec — compares the delivered assertion count and page.on('pageerror') guard against that hand-out, rejecting a net loss (API helpers: one helper per method+path, status-code-only assertions, method+resource names; browser helpers: actions and structural waits only, intent names, one helper per action sequence; every family: one name is defined once per module, with one parameter count across modules). Call it after the reuse edits are written; skyramp_enhance_assertions and skyramp_execute_test refuse until it has passed."),
     testType: z
         .nativeEnum(TestType)
         .optional()

package/build/tools/code-refactor/reuse-outcome.d.ts

@@ -21,7 +21,7 @@ export interface FlaggedMember {
  * moment the spec changes — and it does change: the execution fix-up may restore
  * `<testFile>.raw.bak` over the spec by plain `cp`, invisible to this tool. The
  * counts and the decline list are therefore derived at report time (see
- * `rederiveReuseOutcome`), never read back from here.
+ * `rederiveReuse`), never read back from here.
  *
  * And keeping the two types disjoint makes the report boundary opt-IN. When the
  * record extended the wire type, every field added here shipped to the

package/build/tools/code-refactor/reuse-state.d.ts

@@ -1,13 +1,14 @@
 import { type VerifyResult } from "../../utils/pom-verify/index.js";
 import { type ReuseOutcome, type ReuseRecord } from "./reuse-outcome.js";
 import { type UtilsVerifyResult } from "../../utils/utils-verify/index.js";
+import type { ReuseVerificationFailure } from "../../types/ReuseOutcome.js";
 /**
  * Record the tier-1 candidate count STEP 1 detected, plus the identity needed to
  * re-derive the counts from the delivered spec later.
  *
  * The identity is written here and not only by {@link recordVerifyOutcome} because
  * the verify pass is a call the agent decides whether to make. When it skips it,
- * a record holding the candidate count alone makes `rederiveReuseOutcome` bail out,
+ * a record holding the candidate count alone makes `rederiveReuse` bail out,
  * and the report states "N candidate POM files detected" with no reuse numbers —
  * indistinguishable from a run where reuse was measured and simply not summarized.
  * Recording it unconditionally is what makes the zero-reuse case reportable without
@@ -55,9 +56,11 @@ export declare function recordVerifyOutcome(testFile: string, r: VerifyResult, g
  * NOT terminal. A PASS is always reachable in one edit plus one call (substitute the
  * members, or document each candidate with a `// kept inline:` marker, which the gate
  * accepts), and an agent that resolves neither can still walk past these tools to
- * `skyramp_submit_report`, which does not block and reports the honest zero. So the
- * worst case degrades to the reporting half rather than to a run that produces
- * nothing — which is what a bound would have been protecting against.
+ * `skyramp_submit_report`. That tool refuses a blocking verdict on the delivered
+ * files too, but its refusal is BOUNDED (`REUSE_SUBMIT_MAX_REFUSALS`): past the bound
+ * it accepts the report with the fault recorded in the row. So the worst case still
+ * degrades to the reporting half rather than to a run that produces nothing — which
+ * is what a bound here would have been protecting against.
  */
 export declare function pendingReuseVerification(testFile: string, explicitStateFile?: string): Promise<string | undefined>;
 /**
@@ -81,7 +84,17 @@ export declare function recordUtilsVerifyError(testFile: string, explicitStateFi
 export declare function samePath(a: string, b: string): boolean;
 /** Exported for retrofit-state: the same canonicalisation the records use. */
 export declare function canonPath(p: string): string;
-/** The basename label a report row shows for a set of utils files. */
+/**
+ * The label a report row shows for a set of utils files: each file's path relative
+ * to its repository root, posix-separated, joined with `, `.
+ *
+ * Repository-relative, never absolute — an absolute path carries the runner's
+ * temporary directory and means nothing to a customer. Not a basename either: a
+ * fullstack delivery writes one module per test directory under the same
+ * conventional name, and two rows labelled `skyrampUtils.ts` cannot be told apart
+ * (a renderer keyed on the label then merges them into one aggregate). Outside a
+ * repository, or for a file the root does not contain, the basename stands in.
+ */
 export declare function utilsFileLabel(files: string[]): string;
 /** Verdict of one utils verify pass. The verdict alone is stored — counts and the
  *  file list are re-derived from the delivered files at report time. */
@@ -148,6 +161,64 @@ export declare function reuseChainSkipped(fileName: string, testType: string, re
     testType: string;
     language: string;
 }> | undefined): Promise<true | undefined>;
+/**
+ * A blocking reuse verdict measured on the DELIVERED files at report time — the
+ * same predicate the live `verify: true` pass refuses on, re-run on what ships.
+ *
+ * Verification is a one-time checkpoint on a file that keeps changing: the execution
+ * fix loop deletes a failing assertion after the gate recorded `passed`, and a
+ * post-verification full-file rewrite erases verified page-object reuse. Until this
+ * existed, the report re-derived the verdict, found the fault, wrote it into a field
+ * and shipped the work — the run committed the fault, detected it, disclosed it and
+ * delivered it. `skyramp_submit_report` refuses on this instead, and the agent
+ * repairs (or documents the decline) and resubmits.
+ *
+ * Only a BLOCKING verdict lands here. Advisories (`scenario-name`, untyped
+ * parameters, sibling inline call sites, single importers) never do: they are
+ * report-time signals whose number means something only on the final file, and
+ * they must stay free to be computed and reported at submit.
+ */
+export interface ReuseBlockingVerdict {
+    /** ABSOLUTE PATH of the spec the verdict is about — the refusal is agent-facing,
+     *  and two specs sharing a basename (one per repository) must stay two verdicts. */
+    file: string;
+    /** The blocking kinds found, each with its count and detail — what the row ships
+     *  with once the refusal bound is reached. */
+    failures: ReuseVerificationFailure[];
+    /** What failed and how to repair it — the text the live verify pass would have
+     *  returned for the delivered file, so the remediation is the one the agent knows. */
+    detail: string;
+    /** The exact `skyramp_reuse_code` verify call to make after the repair, so the
+     *  repaired files are re-checked. */
+    verifyCall: string;
+}
+/**
+ * How many times skyramp_submit_report refuses a report over one spec's blocking
+ * verdict before accepting it with the fault recorded in the row.
+ *
+ * The refusal is a repair loop, and a repair loop at the last step of a run needs a
+ * terminal state (SKYR-4059: an unbounded loop destroyed verified work). Without one,
+ * an agent that cannot reach a pass — out of turns, or a `guard-removed` it cannot
+ * restore, which no marker documents away — delivers no report at all, and a
+ * customer with no pull request is a worse outcome than one with a disclosed fault.
+ * Two refusals cover the common case, where the repair is one edit and one verify
+ * call away; the third call ships the row with `verificationFailures` set.
+ */
+export declare const REUSE_SUBMIT_MAX_REFUSALS = 2;
+/** What one report row's re-derivation established: the outcome to publish, and a
+ *  blocking verdict when the delivered files fail the live check. */
+export interface ReuseRederivation {
+    outcome: ReuseOutcome | undefined;
+    blocking?: ReuseBlockingVerdict;
+}
+/**
+ * The reason behind a failed shared-helper verdict, one entry per blocking kind in
+ * the order the verifier found them. The verify pass had these in hand and dropped
+ * them after computing `ok`; without them a `failed` row is a verdict nobody can act
+ * on. The detail text is the verifier's own — for an assertion loss it carries the
+ * counts (baseline, delivered, in the test, in the helpers) that decided the verdict.
+ */
+export declare function verificationFailures(r: UtilsVerifyResult): ReuseVerificationFailure[];
 /**
  * Re-derive a recorded outcome from the spec as it stands NOW, so the report
  * describes the delivered artifact rather than the state at verify time.
@@ -169,8 +240,15 @@ export declare function reuseChainSkipped(fileName: string, testType: string, re
  *
  * Returns the recorded outcome minus internals when re-derivation is impossible
  * (no path recorded — now only the no-POM-layer path, which has no spec to measure
- * and no candidates to contrast against), and `undefined` when it fails outright.
+ * and no candidates to contrast against), and no outcome when it fails outright.
  * Failing closed matters: falling back to the recorded counts is exactly the false
  * claim this exists to prevent.
+ *
+ * Beside the outcome comes the blocking verdict the delivered files earn under the
+ * live check — see {@link ReuseBlockingVerdict}. One measurement feeds both: the row
+ * the report publishes and the refusal `skyramp_submit_report` returns. That half
+ * fails OPEN like every reuse check: a re-derivation that throws yields no outcome
+ * and no verdict. A verifier that cannot run is not evidence against the file, and
+ * refusing a report on a malfunction would deliver nothing to the customer.
  */
-export declare function rederiveReuseOutcome(record: ReuseRecord): Promise<ReuseOutcome | undefined>;
+export declare function rederiveReuse(record: ReuseRecord): Promise<ReuseRederivation>;