@skyramp/mcp 0.3.4 → 0.3.6-rc.1

package/build/tools/test-management/registerTestPlanTool.js

@@ -10,6 +10,10 @@ import { SCENARIO_CATEGORIES, CATEGORY_PRIORITY, Novelty, PriorityTier } from ".
 import { HttpMethod, TestType } from "../../types/TestTypes.js";
 import { CandidateSource, computeCandidateId, scenarioMergeKey, DiscriminatorKind } from "../../types/Recommendation.js";
 import { selectPlan } from "../../recommendation/planRanker.js";
+import { resolveSubjectEndpoints } from "../../utils/subjectEndpoints.js";
+import { collectChangedRouteLines } from "../../utils/changedRoutes.js";
+import { reservedUISlots } from "../../recommendation/budgeters/shared.js";
+import { inferScenarioType } from "../../recommendation/diversity.js";
 import { validateDiscriminator } from "../../recommendation/discriminators.js";
 import { isAttackSurfaceSecurityBoundary } from "../../prompts/test-recommendation/recommendationShared.js";
 import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "../../prompts/test-recommendation/recommendationSections.js";
@@ -47,7 +51,10 @@ const registerStepSchema = z.object({
         .record(z.any())
         .optional()
         .describe("Query parameters with concrete values. Also scanned for boundary_equality and negative_match verification."),
-    responseBody: z.record(z.any()).optional().describe("Expected response body shape/values, when asserted."),
+    responseBody: z
+        .union([z.record(z.any()), z.array(z.any())])
+        .optional()
+        .describe("Expected response body shape/values, when asserted. An array for a collection/list endpoint that returns a JSON array."),
     expectedStatusCode: z.number().int().describe("HTTP status code this step asserts (e.g. 201, 400, 404)."),
     expectedResponseFields: z
         .array(z.string())
@@ -84,10 +91,12 @@ const registerTestPlanSchema = {
         .string()
         .refine((p) => path.isAbsolute(p), { message: "stateFile must be an absolute path" })
         .describe("Path to state file from skyramp_analyze_changes"),
-    repository: z
-        .string()
-        .optional()
-        .describe("The owner/repo whose analysis section to register against (e.g. 'letsramp/api-insight'). In a multi-repo run, ALWAYS set it. Omit only for a single-repo run."),
+    // No `repository` argument, deliberately (SKYR-4204 reopen). The approved
+    // plan is run-wide — one pooled registration covers every repository in the
+    // run — so unlike the per-repo tools (analyze_test_health, actions,
+    // execute_test) there is no section to select: the tool reads the root and
+    // derives run-wide facts (diff union, budget flags, scenario lookup) itself.
+    // A legacy caller still passing `repository` has it stripped harmlessly.
     candidates: z
         .array(registerCandidateSchema)
         .default([])
@@ -115,8 +124,8 @@ function toScenarioStep(step) {
         chainsFrom: step.chainsFrom,
     };
 }
-function toDraftedScenario(input) {
-    return {
+function toDraftedScenario(input, changedRoutes) {
+    const scenario = {
         scenarioName: input.scenarioName,
         description: input.description,
         category: input.category,
@@ -127,13 +136,20 @@ function toDraftedScenario(input) {
         estimatedComplexity: "moderate",
         testType: input.testType,
     };
+    // Resolved here, at the one boundary that has both the steps and the diff, so
+    // every later dedup stage reads one recorded value (SKYR-4214).
+    return { ...scenario, subjectEndpoints: resolveSubjectEndpoints(scenario, { changedRoutes }) };
 }
 /** Build the agent-submitted candidates. Discriminator claims are verified
  *  later, by {@link applyDiscriminatorClaims}, once the merge has settled which
- *  scenario each name resolves to. */
-function buildAgentCandidates(candidates) {
+ *  scenario each name resolves to. Exported for the direct unit test in
+ *  registerTestPlanTool.test.ts — a candidate that does not survive selection
+ *  never reaches the persisted plan, so its subjectEndpoints fill is not
+ *  observable through the tool's output, and testing here avoids coupling the
+ *  fill to what selection happens to keep. */
+export function buildAgentCandidates(candidates, changedRoutes) {
     return candidates.map((input) => {
-        const scenario = toDraftedScenario(input);
+        const scenario = toDraftedScenario(input, changedRoutes);
         return {
             scenario,
             priority: derivePriorityTier(scenario),
@@ -226,6 +242,13 @@ function resolveBudgetContext(stateData, fullState) {
         isUIOnlyPR: pbc?.isUIOnlyPR ?? false,
         hasFrontendChanges: pbc?.hasFrontendChanges ?? false,
         externalCoverage: new Set(pbc?.externalCoverageKeys ?? []),
+        // `true` for the same reason `computeDiffChangesTestFiles` answers `true` with
+        // no diff: an unknown must shut the reserve, not open it. State written by an
+        // older MCP build carries no such field, and if that PR did add its own tests
+        // the reserve would recommend duplicates of them — the failure this flag
+        // exists to stop, reached by version skew alone. It is also the pre-reserve
+        // behaviour: covered candidates were dropped.
+        diffChangesTestFiles: pbc?.diffChangesTestFiles ?? true,
     };
     const relatedSections = Object.values(fullState?.relatedRepos ?? {}).map((section) => section.data);
     if (relatedSections.length === 0)
@@ -245,12 +268,23 @@ function resolveBudgetContext(stateData, fullState) {
     const hasFrontendChanges = sections.some((s) => s?.planBudgetContext?.hasFrontendChanges ?? false);
     const hasApiChanges = sections.some(sectionHasApiChanges);
     const rootPbc = fullState?.planBudgetContext;
+    // SKYR-4211: external-test coverage is also a per-section fact — a backend
+    // candidate already covered by an external test recorded in a RELATED repo's
+    // section must dedup the same as one covered in the primary. Union the keys
+    // across all sections, like the flags above.
+    const externalCoverage = new Set(sections.flatMap((s) => s?.planBudgetContext?.externalCoverageKeys ?? []));
     return {
         maxGenerate: rootPbc?.maxGenerate ?? base.maxGenerate,
         maxTotal: rootPbc?.maxTotal ?? base.maxTotal,
         isUIOnlyPR: hasFrontendChanges && !hasApiChanges,
         hasFrontendChanges,
-        externalCoverage: base.externalCoverage,
+        externalCoverage,
+        // Unioned across sections like the flags above: a pull request that adds
+        // its own tests in ANY repo of the run must shut the reserve, or that repo's
+        // own coverage becomes grounds to recommend duplicates of it. A section with
+        // no budget context is an unknown, and an unknown shuts the reserve — the
+        // same direction `computeDiffChangesTestFiles` takes with no diff.
+        diffChangesTestFiles: sections.some((s) => s?.planBudgetContext?.diffChangesTestFiles ?? true),
     };
 }
 function describeGenerationCall(item) {
@@ -306,7 +340,7 @@ function renderGenerationDirective(plan) {
             (nonUICount > 0 ? ` Generate the ${tests(nonUICount)} of other types as well.` : ""),
     ];
 }
-function renderPlanText(plan) {
+function renderPlanText(plan, dropped) {
     const lines = [];
     lines.push(`## Approved Test Plan (${plan.planId})`);
     lines.push("");
@@ -329,10 +363,41 @@ function renderPlanText(plan) {
         plan.additional.forEach((item, i) => lines.push(renderPlanItem(item, plan.generate.length + i + 1)));
     }
     if (plan.demotions.length > 0) {
+        // A demotion (failed discriminator claim) and a drop (removed by the
+        // budgeter for an unrelated reason — coverage or budget) are independent:
+        // a candidate can suffer both. Split so the report never claims a
+        // dropped candidate is "still registered".
+        const dropReasonByCandidateId = new Map(dropped.map((d) => [d.candidateId, d.reason]));
+        const registeredDemotions = plan.demotions.filter((d) => !dropReasonByCandidateId.has(d.candidateId));
+        const removedDemotions = plan.demotions.filter((d) => dropReasonByCandidateId.has(d.candidateId));
+        if (registeredDemotions.length > 0) {
+            lines.push("");
+            lines.push(`### Demotions (${registeredDemotions.length}) — discriminator claims that did NOT verify (candidate still registered, claim dropped)`);
+            for (const demotion of registeredDemotions) {
+                lines.push(`- ${demotion.candidateId}: ${demotion.reason}`);
+            }
+        }
+        if (removedDemotions.length > 0) {
+            lines.push("");
+            lines.push(`### Demoted AND removed (${removedDemotions.length}) — the claim failed to verify AND the candidate left the plan. It is NOT registered.`);
+            for (const demotion of removedDemotions) {
+                lines.push(`- ${demotion.candidateId}: claim failed — ${demotion.reason}; removed — ${dropReasonByCandidateId.get(demotion.candidateId)}`);
+            }
+        }
+    }
+    // Every other removal — budget or coverage, with no failed claim attached.
+    // This section is independent of `demotions`: the two channels are unrelated,
+    // and the motivating case has drops and NO demotions. Nesting it inside the
+    // demotion block returned empty GENERATE/ADDITIONAL lists with no candidate id
+    // and no reason, which is the state item 5 exists to make visible. Measured on
+    // eval run 32586610615, fixture repo15-P14: 10 drops, 0 demotions.
+    const demotedIds = new Set(plan.demotions.map((d) => d.candidateId));
+    const removedWithoutDemotion = dropped.filter((d) => !demotedIds.has(d.candidateId));
+    if (removedWithoutDemotion.length > 0) {
         lines.push("");
-        lines.push(`### Demotions (${plan.demotions.length}) — discriminator claims that did NOT verify (candidate still registered, claim dropped)`);
-        for (const demotion of plan.demotions) {
-            lines.push(`- ${demotion.candidateId}: ${demotion.reason}`);
+        lines.push(`### Removed (${removedWithoutDemotion.length}) — candidates the selection stage dropped. They are NOT registered.`);
+        for (const drop of removedWithoutDemotion) {
+            lines.push(`- ${drop.candidateId}: ${drop.reason}`);
         }
     }
     lines.push("");
@@ -359,12 +424,16 @@ export function registerRegisterTestPlanTool(server) {
     }, async (params) => {
         let errorResult;
         try {
-            logger.info(`Registering test plan${params.repository ? ` for repository ${params.repository}` : ""}`, {
+            logger.info("Registering run-wide test plan", {
                 candidateCount: params.candidates?.length ?? 0,
             });
             const stateManager = StateManager.fromStatePath(params.stateFile);
-            const stateData = await stateManager.readRepoData(params.repository);
-            const repositoryPath = (await stateManager.getRepoRepositoryPath(params.repository)) || "";
+            // The plan is run-wide, so register always reads the ROOT (primary)
+            // section — there is no section selector (routing by one is what let a
+            // related-repo registration run with that section's context —
+            // SKYR-4204 reopen). Run-wide facts (diff union, budget flags,
+            // scenario lookup) come from fullState below.
+            const stateData = await stateManager.readData();
             if (!stateData) {
                 errorResult = toolError(`State file is empty or invalid: ${params.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
                 return errorResult;
@@ -388,9 +457,20 @@ export function registerRegisterTestPlanTool(server) {
             ];
             const diffText = diffSections.map((s) => s.diff).filter(Boolean).join("\n");
             const reposWithoutDiff = diffSections.filter((s) => !s.diff).map((s) => s.repo);
-            const allScenarios = stateData.repositoryAnalysis?.scenarios ?? [];
-            const agentCandidates = buildAgentCandidates(params.candidates ?? []);
-            const serverCandidates = recoverServerCandidates(stateData.approvedPlan, allScenarios);
+            // Scenario lookup for server-candidate recovery spans ALL sections —
+            // the primary's drafted scenarios must resolve even when `repository`
+            // targets a related repo (SKYR-4204 reopen: the same single-section
+            // shape as the diff union above).
+            const allScenarios = [
+                ...(fullState?.repositoryAnalysis?.scenarios ?? []),
+                ...Object.values(fullState?.relatedRepos ?? {}).flatMap((section) => section.data?.repositoryAnalysis?.scenarios ?? []),
+            ];
+            const changedRoutes = collectChangedRouteLines(diffText);
+            const agentCandidates = buildAgentCandidates(params.candidates ?? [], changedRoutes);
+            // The approved plan is run-wide and persists at the ROOT (see the
+            // persist step below), so prior-plan recovery reads the root first;
+            // the section fallback covers state written by older builds.
+            const serverCandidates = recoverServerCandidates(fullState?.approvedPlan ?? stateData.approvedPlan, allScenarios);
             // Merge by scenario-name identity, NOT the full content-hashed candidateId:
             // the server (analyze_changes) and the agent frequently draft their own
             // independent steps[] for "the same" scenario (identical scenarioName),
@@ -433,7 +513,44 @@ export function registerRegisterTestPlanTool(server) {
                     "Submit your complete candidate list (every test you would generate or recommend) in `candidates` — an empty plan would block all generation.");
                 return errorResult;
             }
+            // Review (mcp#774): enforce ONE run-wide plan at the tool boundary. A
+            // re-registration whose pool shares no scenario with the active plan
+            // is the per-repo-partial signature (the demoshop rig's second,
+            // backend-only call) — accepting it would silently REPLACE the run's
+            // plan with a subset and drop every earlier agent candidate. Genuine
+            // full-list updates overlap the active plan and replace it as before.
+            // The overlap test spans GENERATE ∪ ADDITIONAL: with maxGenerate=0 the
+            // active plan's GENERATE is empty while ADDITIONAL carries every
+            // candidate, and a disjoint partial could otherwise still replace it
+            // silently (Copilot review on #774).
+            const priorItems = [
+                ...(fullState?.approvedPlan?.generate ?? []),
+                ...(fullState?.approvedPlan?.additional ?? []),
+            ];
+            if (priorItems.length > 0) {
+                const poolKeys = new Set(allCandidates.map(mergeKey));
+                const overlaps = priorItems.some((item) => poolKeys.has(scenarioMergeKey(item.scenarioName) || item.candidateId));
+                if (!overlaps) {
+                    errorResult = toolError(`A run-wide plan is already active (${fullState.approvedPlan.planId}: ` +
+                        `${priorItems.map((i) => i.scenarioName).join(", ")}) and this submission shares none of its ` +
+                        "scenarios. skyramp_register_test_plan REPLACES the whole run's plan — resubmit ONE pooled candidate " +
+                        "list for the entire run (your previous candidates plus these changes), not a per-repo subset.");
+                    return errorResult;
+                }
+            }
             const budgetContext = resolveBudgetContext(stateData, fullState);
+            // SKYR-4204 reopen: a reserved UI slot is only fillable from the
+            // candidate pool. A pool with no UI candidate on a run with frontend
+            // changes would silently plan zero UI tests (runs 32419025266,
+            // 32431042133, 32431106864) — reject so the agent self-corrects in one
+            // step, exactly like the empty-candidates rejection above.
+            if (reservedUISlots(budgetContext) > 0 &&
+                !allCandidates.some((c) => inferScenarioType(c.scenario) === TestType.UI)) {
+                errorResult = toolError("This run has frontend changes, so the plan reserves a UI slot — but no submitted or recovered " +
+                    "candidate has testType 'ui'. Resubmit your full candidate list including at least one UI candidate " +
+                    "for the changed frontend (see uiContext.changedFrontendFiles).");
+                return errorResult;
+            }
             const result = selectPlan(allCandidates, { ...budgetContext, demotions, diffText });
             const approvedPlan = {
                 planId: crypto.randomUUID(),
@@ -441,16 +558,25 @@ export function registerRegisterTestPlanTool(server) {
                 generate: result.generate.map(buildApprovedPlanItem),
                 additional: result.additional.map(buildApprovedPlanItem),
                 demotions: result.demotions,
+                dropped: result.dropped,
             };
+            // The approved plan is the ONE run-wide authority (the prompt mandates
+            // a single pooled registration on multi-repo runs), so it persists at
+            // the state-file ROOT regardless of `repository`. Per-section plans
+            // blinded every root-reading consumer — planGuard let generation run
+            // unguarded, submitReportTool's reconciliation was inert, and the eval
+            // extractor saw "no plan" (SKYR-4204 reopen). writeData preserves the
+            // relatedRepos sections and the root metadata.
             try {
-                await stateManager.writeRepoData({ ...stateData, approvedPlan }, { repo: params.repository, repositoryPath, step: TOOL_NAME });
+                const { metadata: _meta, relatedRepos: _sections, ...rootData } = fullState ?? {};
+                await stateManager.writeData({ ...rootData, approvedPlan }, { step: TOOL_NAME });
             }
             catch (error) {
                 errorResult = toolError(`Failed to persist approved test plan to state file: ${error.message}. Retry this call.`);
                 return errorResult;
             }
             return {
-                content: [{ type: "text", text: renderPlanText(approvedPlan) }],
+                content: [{ type: "text", text: renderPlanText(approvedPlan, result.dropped) }],
             };
         }
         catch (error) {

package/build/types/Recommendation.d.ts

@@ -1,4 +1,4 @@
-import { DraftedScenario } from "../types/RepositoryAnalysis.js";
+import { DraftedScenario, SubjectEndpoint } from "../types/RepositoryAnalysis.js";
 import { Novelty, PriorityTier } from "../types/TestRecommendation.js";
 /**
  * The structural bug-shape a candidate claims to probe. Verified against the
@@ -53,21 +53,42 @@ export interface BudgetContext {
     hasFrontendChanges: boolean;
     /** Method-aware coverage keys of external tests; matching candidates are dropped. */
     externalCoverage: Set<string>;
+    /**
+     * Whether the PR's own diff changes any test file. External coverage is
+     * discovered from the working tree, so a test the PR itself adds is
+     * indistinguishable from pre-existing coverage; when this is true the
+     * covered-candidate reserve stays shut rather than forcing duplicates of a
+     * test the author already wrote.
+     */
+    diffChangesTestFiles: boolean;
 }
 /** A demoted claim: which candidate, and the human-readable reason returned to
- *  the agent so it can fix the claim (never a silent drop). */
+ *  the agent so it can fix the claim (never a silent drop of the claim itself).
+ *  A demotion records that a declared discriminator claim failed to verify.
+ *  It does not say whether the candidate is still in the plan. Budgeting
+ *  decides that separately. A candidate can be both demoted and dropped. */
 export interface Demotion {
     candidateId: string;
     reason: string;
 }
+/** A candidate the selection stage removed, and why. A demotion annotates a
+ *  claim. A drop removes a candidate from `generate` and `additional`. The
+ *  two are independent. The same candidate can carry both. */
+export interface Dropped {
+    candidateId: string;
+    reason: string;
+}
 /** Result of budgeting: the GENERATE set, the ADDITIONAL set, how many UI
- *  placeholder slots the render layer should add (mixed/UI-only PRs), and the
- *  demoted-claim channel (the missing rejection surface for register-plan). */
+ *  placeholder slots the render layer should add (mixed/UI-only PRs), the
+ *  demoted-claim channel (the missing rejection surface for register-plan),
+ *  and the dropped-candidate channel (SKYR-4214: every candidate the
+ *  selection stage removed outright, with a human-actionable reason). */
 export interface SelectionResult {
     generate: Candidate[];
     additional: Candidate[];
     reservedUISlots: number;
     demotions: Demotion[];
+    dropped: Dropped[];
 }
 /** A pluggable budgeting algorithm. Deterministic and pure. */
 export interface Budgeter {
@@ -105,7 +126,14 @@ export declare function scenarioMergeKey(scenarioName: string | undefined): stri
 export declare function computeCandidateId(scenario: DraftedScenario): string;
 /** One approved plan item as persisted in `UnifiedAnalysisState.approvedPlan`.
  *  Deliberately slim (no steps/requestBody/etc.) — `matchKeys` already carries
- *  everything a downstream matcher needs. */
+ *  everything a downstream matcher needs.
+ *
+ *  `subjectEndpoints` is the one exception. `matchKeys` holds one key per
+ *  step and is deliberately broad, so a generated test can be matched back
+ *  to its plan item. It does not say which step is the subject. Dedup
+ *  judges the subject, not the whole step list, so the plan item must
+ *  record it too. See `DraftedScenario.subjectEndpoints` for the
+ *  absent-vs-empty contract this field carries over unchanged. */
 export interface ApprovedPlanItem {
     candidateId: string;
     scenarioName: string;
@@ -113,5 +141,6 @@ export interface ApprovedPlanItem {
     category: string;
     source: CandidateSource;
     verifiedDiscriminator?: string;
+    subjectEndpoints?: SubjectEndpoint[];
     matchKeys: string[];
 }