@skyramp/mcp 0.0.63 → 0.0.64-rc.2

package/build/prompts/test-recommendation/test-recommendation-prompt.js

@@ -1,5 +1,6 @@
+import * as crypto from "crypto";
 import { WorkspaceAuthType } from "../../utils/workspaceAuth.js";
-import { buildPrioritizationDimensions, buildTestExamples, buildTestPatternGuidelines, buildTestQualityCriteria, buildGenerationRules, buildToolWorkflows, buildCoverageChecklist, getAuthSnippets, MAX_CRITICAL_TESTS, } from "./recommendationSections.js";
+import { buildToolWorkflows, buildTestPatternGuidelines, buildTestQualityCriteria, buildTestExamples, buildGenerationRules, getAuthSnippets, MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, } from "./recommendationSections.js";
 function formatTestLocations(locs) {
     const entries = Object.entries(locs || {});
     if (entries.length === 0)
@@ -7,7 +8,221 @@ function formatTestLocations(locs) {
     return "\n**Existing test files (do NOT duplicate these):**\n" +
         entries.map(([type, files]) => "  - [" + type + "] " + files).join("\n");
 }
-export function buildRecommendationPrompt(analysis, analysisScope = "full_repo", topN = 10, prContext, workspaceAuthHeader, workspaceAuthType) {
+const CATEGORY_PRIORITY = {
+    new_endpoint: "CRITICAL", // diff-direct scenarios always fill GENERATE slots first
+    security_boundary: "HIGH",
+    business_rule: "HIGH",
+    data_integrity: "HIGH",
+    breaking_change: "HIGH",
+    auth: "HIGH",
+    workflow: "MEDIUM",
+    "error-handling": "MEDIUM",
+    "data-validation": "MEDIUM",
+    crud: "LOW",
+};
+const PRIORITY_ORDER = { CRITICAL: 4, HIGH: 3, MEDIUM: 2, LOW: 1 };
+const NOVELTY_ORDER = { new: 3, modified: 2, existing: 1 };
+function classifyNovelty(scenario, diffContext) {
+    if (!diffContext)
+        return "existing";
+    const paths = scenario.steps.map(s => s.path);
+    const newPaths = new Set((diffContext.newEndpoints || []).map(ep => ep.path));
+    const modPaths = new Set((diffContext.modifiedEndpoints || []).map(ep => ep.path));
+    if (paths.some(p => newPaths.has(p)))
+        return "new";
+    if (paths.some(p => modPaths.has(p)))
+        return "modified";
+    return "existing";
+}
+function prioritiseCandidate(scenario, diffContext) {
+    const priority = CATEGORY_PRIORITY[scenario.category] ?? "LOW";
+    const novelty = classifyNovelty(scenario, diffContext);
+    return { priority, novelty };
+}
+function computeTiebreakerSeed(endpoints, diffFiles) {
+    const canonical = [...endpoints].sort().join("|") + "::" + [...diffFiles].sort().join("|");
+    return crypto.createHash("sha256").update(canonical).digest("hex").slice(0, 8);
+}
+// ── Execution Plan (replaces pre-ranked + scenarios + heuristic sections) ──
+function buildExecutionPlan(scored, maxGen, topN, baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, seed, endpointCount, isUIOnlyPR, hasFrontendChanges = false, hasTraces = false) {
+    const generateItems = scored.slice(0, Math.min(maxGen, scored.length));
+    const additionalItems = scored.slice(maxGen, topN);
+    const authRef = authHeaderValue
+        ? `, authHeader: "${authHeaderValue}"${authSchemeSnippet}`
+        : `, authHeader: <check OpenAPI securitySchemes or auth middleware; "" if confirmed unauthenticated>`;
+    const hasWorkspaceAuthType = !!authTypeValue && authTypeValue !== "none";
+    // For skyramp_scenario_test_generation: always embed full auth so the generated test file
+    // includes the correct Authorization header at execution time.
+    // The workspace authType conflict only applies to skyramp_integration_test_generation (not scenario gen).
+    const scenarioAuthRef = authRef;
+    // For skyramp_integration_test_generation with scenarioFile:
+    // - If workspace has authType set: omit auth entirely — workspace handles Bearer prefix.
+    // - If no authType: pass authHeader only (no authScheme).
+    const authHeaderOnlyRef = hasWorkspaceAuthType
+        ? ""
+        : authHeaderValue
+            ? `, authHeader: "${authHeaderValue}"`
+            : `, authHeader: <check OpenAPI securitySchemes or auth middleware; "" if confirmed unauthenticated>`;
+    const generateBlocks = generateItems.map((item, i) => {
+        const rank = i + 1;
+        const s = item.scenario;
+        const testType = s.testType ?? (s.steps.length === 1 ? "contract" : "integration");
+        if (testType === "contract") {
+            const step = s.steps[0];
+            const endpointURL = `${baseUrl}${step.path}`;
+            const isBodyMethod = ["POST", "PUT", "PATCH"].includes(step.method);
+            const dataParam = isBodyMethod
+                ? `, requestData: <${step.method} ${step.path} body from source code schemas>`
+                : "";
+            return (`**#${rank} — GENERATE** | ${testType} | ${s.category} | priority=${item.priority} | ${item.novelty}\n` +
+                `${step.method} ${step.path} → ${step.expectedStatusCode}\n` +
+                `Tool: skyramp_contract_test_generation({ endpointURL: "${endpointURL}", method: "${step.method}"${authRef}${dataParam} })\n` + // contract tests always use full authRef
+                `From source: authScheme (OpenAPI securitySchemes or auth middleware)${isBodyMethod ? "; requestData field shapes" : ""}`);
+        }
+        else {
+            // integration / e2e / ui — multi-step scenario pipeline
+            const stepLines = s.steps.map((st) => {
+                const chains = st.chainsFrom
+                    ? ` (chains: ${Array.isArray(st.chainsFrom)
+                        ? st.chainsFrom.map(c => `${c.sourceField} from step ${c.sourceStep}`).join(", ")
+                        : `${st.chainsFrom.sourceField} from step ${st.chainsFrom.sourceStep}`})`
+                    : "";
+                return `  ${st.order}. ${st.method} ${st.path} → ${st.expectedStatusCode}: ${st.description}${chains}`;
+            }).join("\n");
+            const toolCalls = s.steps.map((st) => {
+                const isBodyMethod = ["POST", "PUT", "PATCH"].includes(st.method);
+                const dataParam = isBodyMethod
+                    ? `, requestBody: <${st.method} ${st.path} body from source code schemas>`
+                    : "";
+                return `  skyramp_scenario_test_generation({ scenarioName: "${s.scenarioName}", destination: "${s.scenarioName}", baseURL: "${baseUrl}", method: "${st.method}", path: "${st.path}", statusCode: ${st.expectedStatusCode}${scenarioAuthRef}${dataParam} })`;
+            }).join("\n");
+            const prereqNote = s.category === "new_endpoint"
+                ? `\nPrerequisite discovery (MANDATORY for new_endpoint): Before executing these tool calls, read the source code for the new endpoint's request body. Look for FK fields (e.g. \`product_id\`, \`user_id\`, \`order_id\`). For each FK field found, prepend one \`skyramp_scenario_test_generation\` call to create that prerequisite resource first, then chain its \`id\` into the dependent step. If no FK fields exist, proceed with the steps above as-is.`
+                : "";
+            return (`**#${rank} — GENERATE** | ${testType} | ${s.category} | priority=${item.priority} | ${item.novelty}\n` +
+                `Scenario: ${s.scenarioName} (${s.steps.length} steps)\n` +
+                `${stepLines}\n` +
+                `Tool calls:\n` +
+                `${toolCalls}\n` +
+                `  skyramp_integration_test_generation({ scenarioFile: "scenario_${s.scenarioName}.json"${authHeaderOnlyRef} })\n` +
+                `From source: requestBody shapes for POST/PUT/PATCH steps; responseBody shapes; authScheme` +
+                prereqNote);
+        }
+    }).join("\n\n");
+    // For mixed PRs, always reserve slots for UI and E2E recommendations regardless of whether
+    // traces already exist — the user can record them later or the bot can record during the run.
+    const needsE2ESlot = hasFrontendChanges && !isUIOnlyPR;
+    const needsUISlot = hasFrontendChanges && !isUIOnlyPR;
+    const frontendSlots = (needsE2ESlot ? 1 : 0) + (needsUISlot ? 1 : 0);
+    const backendAdditionalItems = frontendSlots > 0
+        ? additionalItems.slice(0, Math.max(additionalItems.length - frontendSlots, 0))
+        : additionalItems;
+    // Additional items — compact listing, no tool calls needed
+    const additionalLines = backendAdditionalItems.map((item, i) => {
+        const rank = maxGen + i + 1;
+        const s = item.scenario;
+        const testType = s.testType ?? (s.steps.length === 1 ? "contract" : "integration");
+        const target = s.steps.length === 1
+            ? `${s.steps[0].method} ${s.steps[0].path} → ${s.steps[0].expectedStatusCode}`
+            : `Scenario: ${s.scenarioName} (${s.steps.map(st => `${st.method} ${st.path}`).join(" → ")})`;
+        return `#${rank} [ADDITIONAL] | ${testType} | ${s.category} | priority=${item.priority} | ${item.novelty}\n  ${target}\n  Validates: ${s.description}`;
+    }).join("\n\n");
+    const uiSlotLine = needsUISlot ? (() => {
+        const rank = maxGen + backendAdditionalItems.length + 1;
+        const traceNote = hasTraces
+            ? "Use an existing Playwright `.zip` trace from the repo."
+            : "Record a trace using `browser_navigate` + `browser_snapshot` + `skyramp_export_zip`, then call `skyramp_ui_test_generation`.";
+        return `\n\n#${rank} [ADDITIONAL] | UI | workflow | priority=HIGH | new\n  Scenario: ui-interaction-for-changed-components (frontend files changed in this diff)\n  Validates: Component-level interaction flow for the changed UI — derive the scenario name and steps from the actual changed frontend files. ${traceNote}`;
+    })() : "";
+    const e2eSlotLine = needsE2ESlot ? (() => {
+        const rank = maxGen + backendAdditionalItems.length + (needsUISlot ? 1 : 0) + 1;
+        const traceNote = hasTraces
+            ? "Call `skyramp_e2e_test_generation` with the discovered trace/recording files."
+            : "No traces exist yet — record a backend trace via `skyramp_start_trace_collection` + `skyramp_stop_trace_collection` and a UI trace via Playwright browser tools, then call `skyramp_e2e_test_generation`.";
+        return `\n\n#${rank} [ADDITIONAL] | E2E | workflow | priority=HIGH | new\n  Scenario: e2e-flow-for-changed-feature (frontend + backend files changed in this diff)\n  Validates: Full browser-level flow for the changed UI components end-to-end — derive the scenario name and steps from the actual changed frontend files. ${traceNote}`;
+    })() : "";
+    const supplementCount = topN - generateItems.length - backendAdditionalItems.length - frontendSlots;
+    const supplementNote = supplementCount > 0
+        ? `\n**REQUIRED — You MUST add ${supplementCount} more to reach the total of ${topN}.** Draft them from endpoint interactions and source code patterns not yet covered. Use the same 5-dimension rubric and quality gate to assign priority (HIGH/MEDIUM/LOW), testType, and category.${hasFrontendChanges && !isUIOnlyPR ? " Since this PR has frontend changes, at least 1 of these should be a UI or E2E test targeting the changed components." : ""} Do NOT produce fewer than ${topN} total.`
+        : "";
+    return `## Execution Plan
+Seed: ${seed} | Endpoints: ${endpointCount} | Budget: ${generateItems.length} generate + ${Math.max(topN - generateItems.length, 0)} additional = ${topN} total
+
+**Step 1 — Source-Code Enrichment (MANDATORY before executing anything)**
+Read the source code for ALL changed files. Look for:
+- **Auth middleware** (passport, jwt.verify, authMiddleware, @requires_auth, Depends(get_current_user), @UseGuards, EnsureSessionDep, session middleware) — if found, override \`authHeader\` and \`authScheme\` in scenario and contract tool calls even if workspace.yml says authType: none. Exception: for \`skyramp_integration_test_generation\` with \`scenarioFile\`, omit auth params entirely if workspace has \`api.authType\` set (workspace handles it); if workspace has no \`authType\`, pass \`authHeader\` only.
+- Business rules and formulas (e.g. total_cost = compute * rate + memory * rate)
+- State transitions and domain constraints (e.g. budget cannot drop below current spend)
+- Validation logic (field constraints, cross-field dependencies)
+- Security boundaries not covered by the structural candidates below
+
+For each one found, evaluate it against these 5 dimensions and assign priority:
+  | Dimension | What to assess |
+  | Production Safety | Guards a critical boundary (auth, unique constraint, cascade delete, data integrity, breaking migration)? → HIGH |
+  | Bug-Finding Potential | Targets a known failure mode (race condition, data consistency, state transition, cascade effect)? → HIGH |
+  | User Journey Relevance | Reflects how real users interact (from traces, business flows, critical paths)? → HIGH or MEDIUM |
+  | Coverage Gap | Addresses an area with zero existing test coverage? → bump up one tier |
+  | Code Insight | Derived from actual implementation (spotted middleware pattern, N+1 risk, unique constraint)? → bump up one tier |
+
+Quality gate — ask both questions:
+  1. "Would this test prevent a production incident?" → YES = HIGH priority regardless of other dimensions
+  2. "Does this test exercise a real workflow or catch a real bug?" → YES = at least MEDIUM
+
+Assign category: security_boundary | business_rule | data_integrity | breaking_change | auth | workflow | error-handling | data-validation | crud
+
+${buildTestPatternGuidelines()}
+
+${buildTestExamples()}
+
+INSERT a source-code-derived candidate into the ranked list **only if ALL three conditions are met**:
+1. Priority is HIGH (it guards a critical boundary or would prevent a production incident)
+2. It is specific to THIS codebase — derived from a concrete business rule, formula, or constraint found in the changed files (not a general pattern that applies to any API)
+3. It is not already covered by a structural candidate in the list below
+
+If these conditions are not met, add it to ADDITIONAL only — do NOT displace a pre-ranked GENERATE item.
+
+When a qualifying candidate is inserted: place it HIGH before MEDIUM before LOW; within the same priority, source-code-derived candidates go BEFORE structural ones. Re-number ranks after insertion. The top ${generateItems.length} become GENERATE items.
+
+**Cascade vs referential integrity:** If both a \`cascade-delete\` and a \`delete-blocked\` scenario appear for the same resource pair, keep only the one that matches the source code's FK delete policy (e.g. \`ON DELETE CASCADE\`, \`cascade=True\`, or \`onDelete: 'CASCADE'\` → keep cascade-delete; \`RESTRICT\`/\`PROTECT\`/no cascade → keep delete-blocked). Remove the inapplicable variant before executing.
+
+**Unique constraints:** Unique-constraint scenarios (duplicate POST → expect 409) are pre-drafted for all resources. Before keeping them, check whether the storage backend actually enforces uniqueness — look for SQL \`UNIQUE\` indexes, Mongoose \`unique: true\`, Prisma \`@unique\`, or explicit duplicate-check logic in the source. If the backend is Redis, an in-memory store, or a schema-less DB with no explicit unique constraint in the changed files, move the unique-constraint scenario to ADDITIONAL with a note that enforcement is unconfirmed — do NOT generate it as a GENERATE item.
+
+**Step 2 — Execute merged plan in rank order**
+Replace any scenario that pairs unrelated resources with one reflecting actual FK relationships in the codebase.
+Use realistic request bodies from source code schemas; verify response data (not just status codes).
+
+${buildTestQualityCriteria()}
+
+${buildGenerationRules(isUIOnlyPR)}
+
+**Per-recommendation format** (apply to all items — GENERATE and ADDITIONAL):
+1. Test type  2. Category  3. Priority (HIGH/MEDIUM/LOW)  4. Target endpoint/scenario
+5. What it validates (business logic — not just "tests the endpoint")
+6. Exact tool call with key params for zero-editing execution
+7. For integration/E2E: reference draftedScenario by scenarioName
+8. Reasoning — the specific production risk prevented or business rule enforced
+
+**Never mark a recommendation "blocked":** No OpenAPI spec → use source code for shapes. No traces → provide \`skyramp_start_trace_collection\` instructions. No backend trace → use the scenario pipeline.
+
+**Critical-category minimum:** At least ${Math.min(MAX_CRITICAL_TESTS, maxGen)} of the ${maxGen} GENERATE items MUST be from HIGH-priority categories (security_boundary, business_rule, data_integrity, breaking_change). The pre-ranked plan below already prioritises this — only override if source-code enrichment reveals a higher-value candidate.
+
+### GENERATE (execute these in order after Step 1 insertion, one retry on failure then skip)
+
+${generateBlocks || "  (no pre-ranked generate items — draft your own based on endpoint analysis)"}
+
+### ADDITIONAL (list in additionalRecommendations in this order after Step 1 insertion)
+
+${additionalLines || "  (none pre-ranked)"}${uiSlotLine}${e2eSlotLine}
+${supplementNote}
+
+**You MUST produce EXACTLY ${topN} total recommendations: ${generateItems.length} to generate + ${Math.max(topN - generateItems.length, 0)} as additionalRecommendations. Do NOT produce fewer. Generate recommendations now.**
+
+## Recommendation Stability
+- **Carry forward** previous additionalRecommendations that still apply — match by scenarioName (multi-step) or endpoint (single-endpoint). Re-derive category and priority from test content.
+- **Only drop** a previous recommendation if its target endpoint was removed, its business logic changed, or it is now covered by a generated test.
+- **Only add** new recommendations for code paths introduced since the last run.`;
+}
+export function buildRecommendationPrompt(analysis, analysisScope = "full_repo", topN = MAX_RECOMMENDATIONS, prContext, workspaceAuthHeader, workspaceAuthType) {
     const isDiffScope = analysisScope === "current_branch_diff";
     const diffContext = analysis.branchDiffContext;
     const openApiSpec = analysis.artifacts?.openApiSpecs?.[0];
@@ -30,12 +245,14 @@ export function buildRecommendationPrompt(analysis, analysisScope = "full_repo",
         ? (diffContext.newEndpoints.length > 0 || diffContext.modifiedEndpoints.length > 0)
         : false;
     const isUIOnlyPR = hasFrontendChanges && !hasApiChanges;
+    const hasTraces = (analysis.artifacts?.traceFiles?.length ?? 0) > 0 ||
+        (analysis.artifacts?.playwrightRecordings?.length ?? 0) > 0;
     // ── Mode preamble ──
     const modePreamble = isDiffScope
         ? `You are in **PR mode**. Maximize test coverage for the branch changes.
 Focus on tests that validate the changed fields, endpoints, and their interactions.
 ${isUIOnlyPR ? `\n**UI-only PR** — no backend changes. UI and E2E tests are most relevant.`
-            : hasFrontendChanges ? `\n**Mixed PR** — both frontend and backend changes detected.`
+            : hasFrontendChanges ? `\n**Mixed PR** — both frontend and backend changes detected. Backend and E2E/UI tests are both required.`
                 : ``}
 Output should be concise and immediately actionable.`
         : `You are in **Repo mode**. Comprehensive test strategy across all endpoints.`;
@@ -58,7 +275,7 @@ Output should be concise and immediately actionable.`
             : /api[_-]?key/i.test(authMethod) ? "X-API-Key"
                 : "Authorization";
     }
-    const { authSchemeSnippet, authTokenSnippet } = getAuthSnippets(authHeaderValue);
+    const { authSchemeSnippet } = getAuthSnippets(authHeaderValue, authTypeValue);
     const sourcePriority = `
 ## Source Priority
 When information conflicts, prefer: **Traces** (actual behavior) > **Code** (implemented behavior) > **Spec/Docs** (documented behavior).
@@ -123,80 +340,120 @@ ${isDiffScope ? "Changed endpoints only. " : ""}Use source code schemas (Zod/Pyd
 ${detailBlocks}
 `;
     }
-    // ── Scenarios ──
-    let scenarioSection = "";
-    {
-        const scenarios = analysis.businessContext.draftedScenarios;
-        if (scenarios.length > 0) {
-            const baseUrl = analysis.apiEndpoints.baseUrl;
-            const scenarioBlocks = scenarios
-                .map((s) => {
-                const stepLines = s.steps.map((st) => `      ${st.order ?? ""}. **${st.method} ${st.path}** → ${st.expectedStatusCode ?? 200}: ${st.description || ""}`).join("\n");
-                const toolCalls = s.steps.map((st) => {
-                    const isBodyMethod = ["POST", "PUT", "PATCH"].includes(st.method);
-                    const hasQueryParams = st.queryParams && Object.keys(st.queryParams).length > 0;
-                    const isIdPath = /\{\w+\}$/.test(st.path);
-                    let dataParam;
-                    if (isBodyMethod) {
-                        dataParam = `requestBody: <from source schemas for ${st.method} ${st.path}>`;
-                    }
-                    else if (hasQueryParams) {
-                        dataParam = `queryParams: '${JSON.stringify(st.queryParams)}'`;
-                    }
-                    else if (isIdPath) {
-                        dataParam = "";
-                    }
-                    else {
-                        dataParam = `queryParams: <derive from source code for ${st.method} ${st.path} as a JSON string of URL query params>`;
-                    }
-                    const dataSnippet = dataParam ? `, ${dataParam}` : "";
-                    const authSnippet = authHeaderValue
-                        ? `, authHeader: "${authHeaderValue}"${authSchemeSnippet}${authTokenSnippet}`
-                        : `, authHeader: <determine from OpenAPI spec securitySchemes or source code auth middleware — use "" ONLY if confirmed unauthenticated>`;
-                    return `      skyramp_scenario_test_generation({ scenarioName: "${s.scenarioName}", destination: "${s.scenarioName}", baseURL: "${baseUrl}", method: "${st.method}", path: "${st.path}", statusCode: ${st.expectedStatusCode ?? 200}${authSnippet}${dataSnippet} })`;
-                }).join("\n");
-                return (`  ### ${s.scenarioName} (${s.category}, ${s.priority})\n` +
-                    `  ${s.description}\n` +
-                    `  **Steps:**\n${stepLines}\n` +
-                    `  **Chaining keys:** ${s.chainingKeys.join(", ") || "none"}\n` +
-                    `  **Tool calls:**\n${toolCalls}\n` +
-                    `  Then: skyramp_integration_test_generation({ scenarioFile: "scenario_${s.scenarioName}.json"${authHeaderValue ? `, authHeader: "${authHeaderValue}"${authSchemeSnippet}${authTokenSnippet}` : `, authHeader: <same auth as scenario calls above>`} })`);
-            })
-                .join("\n\n");
-            const authWarning = !authHeaderValue
-                ? `\n**WARNING — No auth configured.** Before generating scenarios without auth, verify the API does not require authentication by checking the OpenAPI spec \`securitySchemes\` and source code auth middleware. If auth is needed, include \`authHeader\` (and \`authScheme\` if applicable) in every \`skyramp_scenario_test_generation\` call.\n`
-                : "";
-            scenarioSection = `
-## Drafted Scenarios
-**Base URL:** \`${baseUrl}\` | **Auth:** \`${authHeaderValue || "none (verify independently)"}\`${authTypeValue ? ` | **Auth type:** \`${authTypeValue}\`` : ""}${authWarning}
+    // ── Scoring ──
+    const endpointCount = allEndpoints.reduce((acc, ep) => acc + (ep.methods ?? []).length, 0);
+    const maxGen = isUIOnlyPR ? (hasTraces ? MAX_TESTS_TO_GENERATE : 0) : MAX_TESTS_TO_GENERATE;
+    const scenarios = analysis.businessContext.draftedScenarios;
+    let scored = [];
+    let seed = "";
+    if (!isUIOnlyPR && scenarios.length > 0) {
+        const diffFiles = filteredChangedFiles; // use filtered list so bot-committed test files don't shift the seed
+        const endpointPaths = allEndpoints.map(ep => ep.path);
+        seed = computeTiebreakerSeed(endpointPaths, diffFiles);
+        scored = scenarios.map(s => {
+            const result = prioritiseCandidate(s, diffContext ?? undefined);
+            return { scenario: s, ...result };
+        });
+        scored.sort((a, b) => {
+            const pa = PRIORITY_ORDER[a.priority], pb = PRIORITY_ORDER[b.priority];
+            if (pb !== pa)
+                return pb - pa;
+            const na = NOVELTY_ORDER[a.novelty], nb = NOVELTY_ORDER[b.novelty];
+            if (nb !== na)
+                return nb - na;
+            const crossA = a.scenario.steps.length > 2 ? 1 : 0;
+            const crossB = b.scenario.steps.length > 2 ? 1 : 0;
+            if (crossB !== crossA)
+                return crossB - crossA;
+            if (b.scenario.steps.length !== a.scenario.steps.length)
+                return b.scenario.steps.length - a.scenario.steps.length;
+            const errorA = a.scenario.steps.some(s => s.interactionType === "error" || s.interactionType === "edge-case") ? 1 : 0;
+            const errorB = b.scenario.steps.some(s => s.interactionType === "error" || s.interactionType === "edge-case") ? 1 : 0;
+            if (errorB !== errorA)
+                return errorB - errorA;
+            // Use locale-independent comparison to avoid runtime-locale non-determinism
+            const nameA = a.scenario.scenarioName;
+            const nameB = b.scenario.scenarioName;
+            if (nameA < nameB)
+                return -1;
+            if (nameA > nameB)
+                return 1;
+            const hashA = parseInt(crypto.createHash("sha256").update(seed + a.scenario.scenarioName).digest("hex").slice(0, 8), 16);
+            const hashB = parseInt(crypto.createHash("sha256").update(seed + b.scenario.scenarioName).digest("hex").slice(0, 8), 16);
+            return hashA - hashB;
+        });
+    }
+    // ── Main section: execution plan, UI-only guidance, or draft-your-own ──
+    let mainSection;
+    if (isUIOnlyPR) {
+        mainSection = hasTraces ? `
+## Test Recommendations (UI-only PR — traces available)
+
+**Budget: ${maxGen} generate + ${Math.max(topN - maxGen, 0)} additional = ${topN} total**
+
+No backend API changes detected. Generate UI/E2E tests from the available traces. Do NOT generate integration or contract tests.
+
+**Generate from traces:**
+1. Call \`skyramp_ui_test_generation\` or \`skyramp_e2e_test_generation\` using the trace files
+2. Generate exactly ${maxGen} tests targeting the changed UI flows
+
+**You MUST produce EXACTLY ${topN} total recommendations: ${maxGen} to generate + ${topN - maxGen} as additionalRecommendations. Do NOT produce fewer. Generate recommendations now.**
 
-Only use scenarios where resources are ACTUALLY related in the codebase. Replace any
-scenario that pairs unrelated resources with one reflecting real foreign key relationships.
+Do not churn recommendations without cause.
+` : `
+## Test Recommendations (UI-only PR)
 
-**Quality bar:** Realistic request bodies, actual foreign keys for chaining, response data
-verification (not just status codes), realistic test data (not "test product").
-**Query vs Body:** For GET/DELETE requests, use \`queryParams\` (not \`requestBody\`) for search terms,
-filters, pagination. Only POST/PUT/PATCH use \`requestBody\`.
-**Path verification:** Cross-reference paths against Router Mounting context — use correct
-nested paths. **Request bodies:** Replace placeholders with actual schemas from source code.
+**Budget: ${maxGen} generate + ${topN - maxGen} additional = ${topN} total**
 
-${scenarioBlocks}
+No backend API changes detected and no traces available. Do NOT generate or execute any tests.
+All ${topN} recommendations go into additionalRecommendations only.
+
+Draft ${topN} UI and E2E test recommendations covering:
+- Component rendering (correct copy, classes, layout for changed components)
+- User interactions (clicks, form submissions, navigation triggered by changed UI)
+- Empty/error states (if the PR touches empty state or error UI)
+- Cross-browser or responsive behavior (if applicable)
+- Full E2E user journeys that exercise the changed UI flows end-to-end
+
+**To generate UI/E2E tests** (when ready):
+1. Call \`skyramp_start_trace_collection\` (playwright: true)
+2. Exercise the changed UI flows in the browser
+3. Call \`skyramp_stop_trace_collection\`
+4. Generate with \`skyramp_ui_test_generation\` or \`skyramp_e2e_test_generation\`
+
+**You MUST produce EXACTLY ${topN} total recommendations in additionalRecommendations. Do NOT produce fewer. Generate recommendations now.**
+
+Do not churn recommendations without cause.
 `;
-        }
-        else {
-            scenarioSection = `
-## Scenarios — Draft From Your Analysis
+    }
+    else if (scored.length > 0) {
+        mainSection = buildExecutionPlan(scored, maxGen, topN, analysis.apiEndpoints.baseUrl, authHeaderValue, authSchemeSnippet, authTypeValue, seed, endpointCount, isUIOnlyPR, hasFrontendChanges, hasTraces);
+    }
+    else {
+        mainSection = `
+## Draft Your Execution Plan
+
+**Budget: ${maxGen} generate + ${topN - maxGen} additional = ${topN} total**
+
+No pre-drafted scenarios available. Draft ${maxGen} tests from your analysis of the endpoint interactions and source code above, then supplement with your own candidates to reach ${topN} total. Prioritize critical categories (security_boundary > data_integrity > business_rule > workflow > crud).
 
-Draft at least 2-3 MEANINGFUL scenarios based on your codebase analysis:
-1. **Cross-resource workflow** — resources referencing each other via foreign keys
-2. **Search/filter + verify** — create data, search, verify results
-3. **Error handling** — invalid cross-resource references → appropriate errors
+For each test: pick the highest-impact endpoint(s), draft a realistic scenario with actual request/response shapes from source code, and execute the same pipeline described in Tool Workflows below.
 
-Use base URL: \`${analysis.apiEndpoints.baseUrl}\` and auth: \`${authHeaderValue || "none — verify independently from OpenAPI spec and source code"}\`${authTypeValue ? ` (auth type: \`${authTypeValue}\`)` : ""}.
-${!authHeaderValue ? "**Verify auth requirements** from OpenAPI spec and source code before omitting auth headers." : ""}
+**You MUST produce EXACTLY ${topN} total recommendations: ${maxGen} to generate + ${topN - maxGen} as additionalRecommendations. Do NOT produce fewer. Generate recommendations now.**
+
+## Recommendation Stability
+- **Carry forward** previous additionalRecommendations that still apply — match by scenarioName (multi-step) or endpoint (single-endpoint). Re-derive category and priority from test content.
+- **Only drop** a previous recommendation if its target endpoint was removed, its business logic changed, or it is now covered by a generated test.
+- **Only add** new recommendations for code paths introduced since the last run.
+- Do not churn recommendations without cause.
 `;
-        }
     }
+    // ── OpenAPI spec note ──
+    const specNote = openApiSpec
+        ? `\n**OpenAPI Spec available**: \`${openApiSpec.path}\`
+Use it for contract tests (\`apiSchema: "${openApiSpec.path}"\`) and to extract request/response shapes.
+`
+        : "";
     // ── PR History ──
     let prHistorySection = "";
     if (prContext && prContext.previousRecommendations.length > 0) {
@@ -280,32 +537,26 @@ ${modePreamble}
 
 Scope: ${scopeNote}
 
-${buildTestQualityCriteria()}
-
 ${sourcePriority}
 
-${buildPrioritizationDimensions()}
-
-${buildTestExamples()}
-
-${buildTestPatternGuidelines()}
-
-${buildGenerationRules(isUIOnlyPR)}
-
-${diffSection}
 ## Repository Context
 
 ${repoContext}
-
+${specNote}
+${diffSection}
 ${interactionSection}
-${scenarioSection}
+${mainSection}
 ${prHistorySection}
 ## Existing Tests
 - Frameworks: ${analysis.existingTests.frameworks.join(", ") || "none"}
 ${formatTestLocations(analysis.existingTests.testLocations)}
 
-${buildToolWorkflows(authHeaderValue, authTypeValue)}
+${isUIOnlyPR
+        ? `## How to Generate Tests — Tool Workflows
 
-${buildCoverageChecklist(openApiSpec, isUIOnlyPR, topN, undefined, MAX_CRITICAL_TESTS)}
+**For UI tests:** \`skyramp_start_trace_collection\` (playwright: true) → perform browser steps → \`skyramp_stop_trace_collection\` → \`skyramp_ui_test_generation\` with the playwright zip.
+**For E2E tests:** Same trace flow, pass both trace file and playwright zip to \`skyramp_e2e_test_generation\`.
+Without traces, list as additionalRecommendations with instructions to record traces first.`
+        : buildToolWorkflows(authHeaderValue, authTypeValue)}
 `;
 }

package/build/prompts/test-recommendation/test-recommendation-prompt.test.js

@@ -2,7 +2,7 @@ jest.mock("@skyramp/skyramp", () => ({
     WorkspaceConfigManager: { create: jest.fn() },
 }));
 import { buildRecommendationPrompt } from "./test-recommendation-prompt.js";
-import { buildCoverageChecklist, PATH_PARAM_UUID_GUIDANCE, MAX_CRITICAL_TESTS, MAX_TESTS_TO_GENERATE } from "./recommendationSections.js";
+import { PATH_PARAM_UUID_GUIDANCE, MAX_TESTS_TO_GENERATE } from "./recommendationSections.js";
 // ---------------------------------------------------------------------------
 // Minimal fixtures
 // ---------------------------------------------------------------------------
@@ -261,30 +261,73 @@ describe("buildRecommendationPrompt — PR History section", () => {
     });
 });
 // ---------------------------------------------------------------------------
-// Tests — Coverage Checklist stability section
+// Tests — Stability and supplement in buildRecommendationPrompt
 // ---------------------------------------------------------------------------
-describe("buildCoverageChecklist — Recommendation Stability section", () => {
-    it("includes Recommendation Stability section in output", () => {
-        const checklist = buildCoverageChecklist(null, false, 10);
-        expect(checklist).toContain("## Recommendation Stability");
+function minimalScenario(overrides = {}) {
+    return {
+        scenarioName: "item-create-delete",
+        description: "Create and delete an item to verify cascade",
+        category: "data_integrity",
+        priority: "high",
+        steps: [
+            { order: 1, method: "POST", path: "/api/items", description: "Create item", interactionType: "success", expectedStatusCode: 201 },
+            { order: 2, method: "DELETE", path: "/api/items/{id}", description: "Delete item", interactionType: "success", expectedStatusCode: 204 },
+        ],
+        chainingKeys: ["id"],
+        requiresAuth: false,
+        estimatedComplexity: "simple",
+        testType: "integration",
+        ...overrides,
+    };
+}
+describe("buildRecommendationPrompt — Stability and supplement section", () => {
+    it("includes Recommendation Stability section in output when scenarios exist", () => {
+        const analysis = minimalAnalysis({
+            businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
+        });
+        const prompt = buildRecommendationPrompt(analysis, "full_repo", 10);
+        expect(prompt).toContain("## Recommendation Stability");
     });
     it("stability section uses scenarioName/endpoint matching strategy", () => {
-        const checklist = buildCoverageChecklist(null, false, 10);
-        const stabilityStart = checklist.indexOf("## Recommendation Stability");
-        const stabilityBlock = checklist.slice(stabilityStart, stabilityStart + 500);
+        const analysis = minimalAnalysis({
+            businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
+        });
+        const prompt = buildRecommendationPrompt(analysis, "full_repo", 10);
+        const stabilityStart = prompt.indexOf("## Recommendation Stability");
+        const stabilityBlock = prompt.slice(stabilityStart, stabilityStart + 500);
         expect(stabilityBlock).toContain("scenarioName");
         expect(stabilityBlock).toContain("endpoint");
         expect(stabilityBlock).toContain("Re-derive category and priority");
     });
     it("stability section specifies when to drop a recommendation", () => {
-        const checklist = buildCoverageChecklist(null, false, 10);
-        expect(checklist).toContain("target endpoint was removed");
-        expect(checklist).toContain("business logic changed");
-        expect(checklist).toContain("covered by a generated test");
+        const analysis = minimalAnalysis({
+            businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
+        });
+        const prompt = buildRecommendationPrompt(analysis, "full_repo", 10);
+        expect(prompt).toContain("target endpoint was removed");
+        expect(prompt).toContain("business logic changed");
+        expect(prompt).toContain("covered by a generated test");
+    });
+    it("instructs to supplement beyond pre-ranked list to reach topN", () => {
+        // 1 scenario, topN=10 → supplementCount=9 → supplement note should appear
+        const analysis = minimalAnalysis({
+            businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
+        });
+        const prompt = buildRecommendationPrompt(analysis, "full_repo", 10);
+        expect(prompt).toContain("REQUIRED — You MUST add");
+        expect(prompt).toContain("5-dimension rubric");
+    });
+    // Verify MAX_TESTS_TO_GENERATE is still exported and equals 3
+    it("MAX_TESTS_TO_GENERATE is 3", () => {
+        expect(MAX_TESTS_TO_GENERATE).toBe(3);
     });
     it("uses MAX_CRITICAL_TESTS in category-aware selection rules", () => {
-        const checklist = buildCoverageChecklist(null, false, 10, MAX_TESTS_TO_GENERATE, MAX_CRITICAL_TESTS);
-        expect(checklist).toContain(`up to ${Math.min(MAX_CRITICAL_TESTS, MAX_TESTS_TO_GENERATE)} of the ${MAX_TESTS_TO_GENERATE} generated tests MUST be from critical categories`);
+        const analysis = minimalAnalysis({
+            businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
+        });
+        const prompt = buildRecommendationPrompt(analysis, "full_repo", 10);
+        // The critical-category minimum line references MAX_CRITICAL_TESTS (= 3)
+        expect(prompt).toContain("GENERATE items MUST be from HIGH-priority categories");
     });
 });
 // ---------------------------------------------------------------------------

package/build/prompts/testGenerationPrompt.js

@@ -198,7 +198,7 @@ Trace File:
 
 **CRITICAL: FOR A SINGLE SCENARIO CREATE A SINGLE SCENARIO FILE WITH EACH STEP AS A JSON OBJECT IN THE ARRAY.**
 
-**CRITICAL: SHOW STEPS TO START/STOP PLAYWRIGHT TRACE COLLECTION FOR UI TESTS. NEVER SHOW THE CLI COMMANDS.**
+**CRITICAL: For UI test trace collection, use the browser_* tools (browser_navigate, browser_click, browser_type, etc.) to interact with the application, then call skyramp_export_zip to export the trace zip. Do NOT use skyramp_start_trace_collection/skyramp_stop_trace_collection.**
             `,
                 },
             },