@skyramp/mcp 0.3.1 → 0.3.2-rc.pom-1

package/build/prompts/testbot/testbot-prompts.test.js

@@ -378,3 +378,47 @@ describe("parseRelatedRepositories", () => {
         expect(parsed).toEqual([{ repo: "o/ok", repositoryPath: "/ok", baseBranch: undefined }]);
     });
 });
+describe("plan-only eval mode (via getTestbotPrompt)", () => {
+    function callWithPlanOnly(planOnly) {
+        return getTestbotPrompt(baseArgs.prTitle, baseArgs.prDescription, baseArgs.summaryOutputFile, baseArgs.repositoryPath, undefined, // baseBranch
+        undefined, // maxRecommendations
+        undefined, // maxGenerate
+        undefined, // maxCritical
+        undefined, // prNumber
+        undefined, // userPrompt
+        undefined, // services
+        undefined, // uiCredentials
+        undefined, // testsRepoDir
+        undefined, // relatedRepositories
+        undefined, // primaryRepo
+        planOnly);
+    }
+    it("branches Task 2 to the plan-only variant when planOnly is true", () => {
+        const prompt = callWithPlanOnly(true);
+        expect(prompt).toContain("## Task 2: Commit the Test Plan (plan-only run)");
+        expect(prompt).not.toContain("## Task 2: Generate New Tests");
+        // The standard generation/execution mandates must be absent, not overridden.
+        expect(prompt).not.toContain("skyramp_integration_test_generation");
+        expect(prompt).not.toContain('phase: "before"');
+        // Task 3 declares the plan instead of counting generated files.
+        expect(prompt).toContain("DECLARES your final GENERATE selection");
+    });
+    it("renders the standard prompt untouched when planOnly is false or omitted", () => {
+        const prompt = callWithPlanOnly(false);
+        expect(prompt).toContain("## Task 2: Generate New Tests");
+        expect(prompt).not.toContain("plan-only");
+        expect(prompt).toBe(callWithServices([]));
+    });
+});
+describe("UI code reuse wiring", () => {
+    it("instructs codeReuse: true for TS/JS Playwright UI generation", () => {
+        const p = callWithServices([]);
+        expect(p).toContain("codeReuse: true");
+    });
+    it("teaches both reuse outcomes and the raw.bak insurance rule", () => {
+        const p = callWithServices([]);
+        expect(p).toContain("No reusable POM layer detected");
+        expect(p).toContain("verification: passed");
+        expect(p).toContain(".raw.bak");
+    });
+});

package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts

@@ -0,0 +1,7 @@
+import { Budgeter } from "../../types/Recommendation.js";
+/**
+ * Budgeter that guarantees cross-test-type coverage in the GENERATE set.
+ * `maxGenerate` acts as an upper cap (not a hard N). Fixes the single-type skew
+ * that starves contract/error-path tests. Opt-in via strategy selection.
+ */
+export declare const diversityBalancedBudgeter: Budgeter;

package/build/recommendation/budgeters/diversityBalancedBudgeter.js

@@ -0,0 +1,71 @@
+import { runBudget } from "./shared.js";
+import { bucketByType, inferScenarioType, isProtectedCandidate, roundRobinFill, } from "../diversity.js";
+const typeOf = (c) => inferScenarioType(c.scenario);
+/**
+ * Pick `count` items guaranteeing each present test type at least one slot.
+ *
+ * Unlike roundRobinByType (which fills ALL protected items first and can
+ * starve other types at small N — the SKYR-3879 "0 contract executed" skew),
+ * this caps protected occupancy to `count - (numTypes - 1)` so at least one slot
+ * per remaining type is always reachable, then:
+ *   Phase 1 — one item for each still-uncovered present type (by rank), then
+ *   Phase 2 — round-robin the remainder across types until `count` is reached.
+ * Rank order is preserved within each type; degenerate cases match a rank slice.
+ */
+function floorBalancedPick(items, count) {
+    if (count <= 0)
+        return [];
+    if (count >= items.length)
+        return items.slice(0, count);
+    // Present types in first-appearance (rank) order.
+    const typesOrder = [];
+    const seen = new Set();
+    for (const it of items) {
+        const t = typeOf(it);
+        if (!seen.has(t)) {
+            seen.add(t);
+            typesOrder.push(t);
+        }
+    }
+    const maxProtected = Math.max(0, count - (typesOrder.length - 1));
+    const selected = [];
+    const pool = [];
+    let protectedTaken = 0;
+    for (const it of items) {
+        if (isProtectedCandidate(it.priority, it.scenario) &&
+            protectedTaken < maxProtected &&
+            selected.length < count) {
+            selected.push(it);
+            protectedTaken++;
+        }
+        else {
+            pool.push(it);
+        }
+    }
+    const { buckets } = bucketByType(pool, typeOf);
+    const covered = new Set(selected.map(typeOf));
+    // Phase 1 — floor: one item per still-uncovered present type.
+    for (const t of typesOrder) {
+        if (selected.length >= count)
+            break;
+        if (covered.has(t))
+            continue;
+        const bucket = buckets.get(t);
+        if (bucket && bucket.length > 0) {
+            selected.push(bucket.shift());
+            covered.add(t);
+        }
+    }
+    // Phase 2 — round-robin the remainder across types until full.
+    roundRobinFill(selected, typesOrder, buckets, count);
+    return selected;
+}
+/**
+ * Budgeter that guarantees cross-test-type coverage in the GENERATE set.
+ * `maxGenerate` acts as an upper cap (not a hard N). Fixes the single-type skew
+ * that starves contract/error-path tests. Opt-in via strategy selection.
+ */
+export const diversityBalancedBudgeter = {
+    name: "diversity-balanced",
+    select: (ranked, ctx) => runBudget(ranked, ctx, floorBalancedPick),
+};

package/build/recommendation/budgeters/diversityBalancedBudgeter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/recommendation/budgeters/diversityBalancedBudgeter.test.js

@@ -0,0 +1,75 @@
+import { diversityBalancedBudgeter } from "./diversityBalancedBudgeter.js";
+import { PriorityTier } from "../../types/TestRecommendation.js";
+import { fixedNBudgeter } from "./fixedNBudgeter.js";
+import { mkCandidate } from "../testFixtures.js";
+function ctx(over = {}) {
+    return {
+        maxGenerate: 3,
+        maxTotal: 20,
+        isUIOnlyPR: false,
+        hasFrontendChanges: false,
+        externalCoverage: new Set(),
+        ...over,
+    };
+}
+const countType = (r, t) => r.generate.filter((c) => c.scenario.testType === t).length;
+const typesIn = (r) => new Set(r.generate.map((c) => c.scenario.testType));
+describe("diversityBalancedBudgeter", () => {
+    it("guarantees a contract slot where fixed-n starves it (the SKYR-3879 fix)", () => {
+        // Protected (CRITICAL) integration scenarios out-rank contract ones and, at N=3,
+        // fixed-n's protected-first fill consumes every slot with integration → 0 contract.
+        const ranked = [
+            mkCandidate("i1", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i2", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i3", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+        ];
+        const c = ctx({ maxGenerate: 3 });
+        // Baseline: fixed-n reproduces the skew.
+        expect(countType(fixedNBudgeter.select(ranked, c), "contract")).toBe(0);
+        // Fix: diversity-balanced reserves a per-type floor.
+        const div = diversityBalancedBudgeter.select(ranked, c);
+        expect(div.generate.length).toBe(3);
+        expect(countType(div, "contract")).toBeGreaterThanOrEqual(1);
+        expect(countType(div, "integration")).toBeGreaterThanOrEqual(1);
+    });
+    it("gives every present test type at least one slot before any type gets a second", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("i3", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+            mkCandidate("u1", { testType: "ui" }),
+        ];
+        const div = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 3 }));
+        expect(typesIn(div)).toEqual(new Set(["integration", "contract", "ui"]));
+    });
+    it("raising maxGenerate increases per-type coverage while keeping the floor", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i2", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i3", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i4", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("i5", { testType: "integration", priority: PriorityTier.CRITICAL }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+            mkCandidate("u1", { testType: "ui" }),
+        ];
+        const atThree = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 3 }));
+        const atSix = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 6 }));
+        expect(countType(atThree, "contract")).toBeGreaterThanOrEqual(1);
+        expect(countType(atThree, "ui")).toBeGreaterThanOrEqual(1);
+        expect(countType(atSix, "contract")).toBeGreaterThanOrEqual(1);
+        expect(countType(atSix, "ui")).toBeGreaterThanOrEqual(1);
+        // more budget → more of the highest-ranked (integration) coverage
+        expect(countType(atSix, "integration")).toBeGreaterThan(countType(atThree, "integration"));
+        expect(atSix.generate.length).toBe(6);
+    });
+    it("returns everything (rank order) when maxGenerate >= candidate count", () => {
+        const ranked = [mkCandidate("i1", { testType: "integration" }), mkCandidate("c1", { testType: "contract" })];
+        const div = diversityBalancedBudgeter.select(ranked, ctx({ maxGenerate: 5 }));
+        expect(div.generate.map((c) => c.scenario.scenarioName)).toEqual(["i1", "c1"]);
+    });
+});

package/build/recommendation/budgeters/fixedNBudgeter.d.ts

@@ -0,0 +1,7 @@
+import { Budgeter } from "../../types/Recommendation.js";
+/**
+ * Default budgeter — reproduces the pre-refactor behavior exactly: fill up to
+ * `maxGenerate` GENERATE slots by round-robin across the present test types
+ * (protected items first), the rest become ADDITIONAL.
+ */
+export declare const fixedNBudgeter: Budgeter;

package/build/recommendation/budgeters/fixedNBudgeter.js

@@ -0,0 +1,11 @@
+import { roundRobinByType } from "../diversity.js";
+import { runBudget } from "./shared.js";
+/**
+ * Default budgeter — reproduces the pre-refactor behavior exactly: fill up to
+ * `maxGenerate` GENERATE slots by round-robin across the present test types
+ * (protected items first), the rest become ADDITIONAL.
+ */
+export const fixedNBudgeter = {
+    name: "fixed-n",
+    select: (ranked, ctx) => runBudget(ranked, ctx, roundRobinByType),
+};

package/build/recommendation/budgeters/fixedNBudgeter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/build/recommendation/budgeters/fixedNBudgeter.test.js

@@ -0,0 +1,66 @@
+import { fixedNBudgeter } from "./fixedNBudgeter.js";
+import { mkCandidate } from "../testFixtures.js";
+import { externalDedupKey } from "../../prompts/test-recommendation/recommendationShared.js";
+function ctx(over = {}) {
+    return {
+        maxGenerate: 3,
+        maxTotal: 20,
+        isUIOnlyPR: false,
+        hasFrontendChanges: false,
+        externalCoverage: new Set(),
+        ...over,
+    };
+}
+const gen = (r) => r.generate.map((c) => c.scenario.scenarioName);
+describe("fixedNBudgeter", () => {
+    it("backend-only PR fills maxGenerate slots via round-robin across test types", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("i3", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("c2", { testType: "contract" }),
+        ];
+        const res = fixedNBudgeter.select(ranked, ctx());
+        expect(res.generate.length).toBe(3);
+        expect(res.reservedUISlots).toBe(0);
+        // buckets [integration, contract]; round 1 → i1, c1; round 2 → i2
+        expect(gen(res)).toEqual(["i1", "c1", "i2"]);
+    });
+    it("mixed (frontend) PR reserves one UI slot → backend count = maxGenerate - 1", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+        ];
+        const res = fixedNBudgeter.select(ranked, ctx({ hasFrontendChanges: true }));
+        expect(res.generate.length).toBe(2);
+        expect(res.reservedUISlots).toBe(1);
+    });
+    it("UI-only PR selects no backend generate items", () => {
+        const res = fixedNBudgeter.select([mkCandidate("i1", { testType: "integration" })], ctx({ isUIOnlyPR: true }));
+        expect(res.generate.length).toBe(0);
+    });
+    it("additional = leftover candidates (rank order), excluding anything generated", () => {
+        const ranked = [
+            mkCandidate("i1", { testType: "integration" }),
+            mkCandidate("c1", { testType: "contract" }),
+            mkCandidate("i2", { testType: "integration" }),
+            mkCandidate("c2", { testType: "contract" }),
+        ];
+        const res = fixedNBudgeter.select(ranked, ctx());
+        const generated = new Set(gen(res));
+        for (const a of res.additional) {
+            expect(generated.has(a.scenario.scenarioName)).toBe(false);
+        }
+        expect(res.additional.length).toBe(1);
+    });
+    it("drops an ordinary candidate already covered by an external test", () => {
+        const covered = mkCandidate("covered", { testType: "contract" });
+        const fresh = mkCandidate("fresh", { testType: "integration" });
+        const res = fixedNBudgeter.select([covered, fresh], ctx({ maxGenerate: 5, externalCoverage: new Set([externalDedupKey(covered.scenario)]) }));
+        const all = [...res.generate, ...res.additional].map((c) => c.scenario.scenarioName);
+        expect(all).toContain("fresh");
+        expect(all).not.toContain("covered");
+    });
+});

package/build/recommendation/budgeters/shared.d.ts

@@ -0,0 +1,19 @@
+import { Candidate, BudgetContext, SelectionResult } from "../../types/Recommendation.js";
+/**
+ * Backend GENERATE slot count:
+ *  - UI-only PR: 0 (all slots are UI placeholders)
+ *  - Mixed PR:   maxGenerate - 1 (last slot reserved for a UI placeholder)
+ *  - Backend-only PR: maxGenerate
+ */
+export declare function backendGenerateCount(ctx: BudgetContext): number;
+/** UI placeholder slots the render layer fills (UI-only → all; mixed → one). */
+export declare function reservedUISlots(ctx: BudgetContext): number;
+/**
+ * Shared budgeting pipeline. All Budgeters run the same external-dedup,
+ * attack-surface prioritization, and ADDITIONAL set-difference; they differ
+ * ONLY in how they pick the GENERATE items from the slot-ordered list (`pick`).
+ *
+ * With `pick = roundRobinByType` this reproduces the pre-refactor selection in
+ * diffExecutionPlan.ts exactly.
+ */
+export declare function runBudget(ranked: Candidate[], ctx: BudgetContext, pick: (items: Candidate[], count: number) => Candidate[]): SelectionResult;

package/build/recommendation/budgeters/shared.js

@@ -0,0 +1,66 @@
+import { prioritizeAttackSurfaceBundles } from "../diversity.js";
+import { externalDedupKey, scenarioCoverageKey, isAttackSurfaceSecurityBoundary, } from "../../prompts/test-recommendation/recommendationShared.js";
+import { logger } from "../../utils/logger.js";
+/**
+ * Backend GENERATE slot count:
+ *  - UI-only PR: 0 (all slots are UI placeholders)
+ *  - Mixed PR:   maxGenerate - 1 (last slot reserved for a UI placeholder)
+ *  - Backend-only PR: maxGenerate
+ */
+export function backendGenerateCount(ctx) {
+    if (ctx.isUIOnlyPR)
+        return 0;
+    return ctx.hasFrontendChanges ? Math.max(0, ctx.maxGenerate - 1) : ctx.maxGenerate;
+}
+/** UI placeholder slots the render layer fills (UI-only → all; mixed → one). */
+export function reservedUISlots(ctx) {
+    if (ctx.isUIOnlyPR)
+        return ctx.maxGenerate;
+    return ctx.hasFrontendChanges && ctx.maxGenerate > 0 ? 1 : 0;
+}
+/**
+ * Drop candidates whose method-aware resource+type is already covered by an
+ * external test — except protected `bug_caught` / attack-surface scenarios,
+ * which require semantic flaw coverage the external test may not provide.
+ */
+function applyExternalDedup(ranked, externalCoverage) {
+    if (externalCoverage.size === 0)
+        return ranked;
+    return ranked.filter((item) => {
+        const key = externalDedupKey(item.scenario);
+        if (externalCoverage.has(key)) {
+            if (item.scenario.category === "bug_caught" || isAttackSurfaceSecurityBoundary(item.scenario)) {
+                logger.info(`External dedup: preserving "${item.scenario.scenarioName}" (${key}) — protected bug/attack-surface scenario requires semantic flaw coverage`);
+                return true;
+            }
+            logger.info(`External dedup: skipping "${item.scenario.scenarioName}" (${key}) — covered by external test`);
+            return false;
+        }
+        return true;
+    });
+}
+/**
+ * Shared budgeting pipeline. All Budgeters run the same external-dedup,
+ * attack-surface prioritization, and ADDITIONAL set-difference; they differ
+ * ONLY in how they pick the GENERATE items from the slot-ordered list (`pick`).
+ *
+ * With `pick = roundRobinByType` this reproduces the pre-refactor selection in
+ * diffExecutionPlan.ts exactly.
+ */
+export function runBudget(ranked, ctx, pick) {
+    const backend = backendGenerateCount(ctx);
+    const deduped = applyExternalDedup(ranked, ctx.externalCoverage);
+    const slotOrdered = prioritizeAttackSurfaceBundles(deduped);
+    const generate = pick(slotOrdered, Math.min(backend, slotOrdered.length));
+    // ADDITIONAL = everything not chosen for GENERATE, in rank order, capped at the
+    // remaining budget, minus any whose resource+type is already covered by GENERATE.
+    const generateSet = new Set(generate);
+    const generatedCoverage = new Set(generate.map((item) => scenarioCoverageKey(item.scenario)));
+    const additional = slotOrdered
+        .filter((it) => !generateSet.has(it))
+        .slice(0, Math.max(0, ctx.maxTotal - backend))
+        .filter((item) => !generatedCoverage.has(scenarioCoverageKey(item.scenario)));
+    // Budgeting itself produces no demotions; the register-plan selection stage
+    // (planRanker.selectPlan) fills this channel from discriminator verification.
+    return { generate, additional, reservedUISlots: reservedUISlots(ctx), demotions: [] };
+}

package/build/recommendation/discriminators.d.ts

@@ -0,0 +1,31 @@
+import { DraftedScenario } from "../types/RepositoryAnalysis.js";
+import { DiscriminatorKind } from "../types/Recommendation.js";
+/**
+ * A discriminator claim the agent (or server) attaches to a candidate: the
+ * structural bug-shape (`kind`) plus the verbatim diff snippet the test probes.
+ */
+export interface DiscriminatorClaim {
+    kind: DiscriminatorKind;
+    changedCodeAnchor: string;
+}
+/** Result of verifying a claim against a scenario's steps and the PR diff. */
+export interface DiscriminatorValidation {
+    verified: boolean;
+    /** One-sentence, human-readable reason present ONLY when `verified` is false;
+     *  it is returned to the LLM to help it fix the claim. */
+    reason?: string;
+}
+/**
+ * Verify a declared discriminator against the candidate's `steps[]` and the PR
+ * diff. Mirrors the classification precedent (`recommendationShared.ts`): the
+ * claim is checked STRUCTURALLY, never by fuzzy keyword scans of prose. A failed
+ * check returns `verified: false` with a one-sentence reason (the claim is
+ * demoted, never used to reject the candidate). NEVER throws — malformed or
+ * partial input is treated as unverified.
+ *
+ * Two gates, both required to verify:
+ *  1. `changedCodeAnchor` must be a non-trivial string (>= 8 chars after trim)
+ *     occurring verbatim in `diffText` — grounds the claim in the real change.
+ *  2. The `kind`-specific structural predicate must hold over `steps[]`.
+ */
+export declare function validateDiscriminator(scenario: DraftedScenario, declared: DiscriminatorClaim, diffText: string): DiscriminatorValidation;