@skyramp/mcp 0.2.1-rc.1 → 0.2.1

package/build/playwright/registerPlaywrightTools.js

@@ -45,6 +45,7 @@ export async function registerPlaywrightTools(server, options) {
         'browser_wait_for',
         'browser_take_screenshot',
         'browser_assert',
+        'browser_assert_api_request',
         'skyramp_export_zip',
         // DOM Analyzer tools (Phase C)
         'browser_blueprint',

package/build/prompts/test-maintenance/drift-analysis-prompt.js

@@ -1,91 +1,102 @@
-import { buildActionDecisionTree, buildCheckAdditiveFields, buildCheckEndpointExistence, buildCheckResponseShape, buildCheckAuthAndAuthorization, buildCheckBehavioralContract, buildCheckAssignAction, buildDriftOutputChecklist, buildUpdateExecutionRules, } from "./driftAnalysisSections.js";
+import { buildActionDecisionMatrix, buildBreakingChangePatterns, buildTestAssessmentGuidelines, buildAddRecommendationGuidelines, buildDriftOutputChecklist, buildUpdateExecutionRules, } from "./driftAnalysisSections.js";
+import { isTestbotEnabled } from "../../utils/featureFlags.js";
 import { readDiffFile } from "../../utils/utils.js";
-import { PromptPlan } from "../test-recommendation/promptPlan.js";
-// ── Private body helpers ──────────────────────────────────────────────────────
-// Each receives DriftAnalysisPromptParams and returns the step body string.
-// The "### Step N: Title" header is added by PromptPlan.render().
-function _assessBody(_p) {
-    return buildActionDecisionTree();
-}
-function _checkAdditiveFieldsBody(_p) {
-    return buildCheckAdditiveFields();
-}
-function _checkEndpointExistenceBody(_p) {
-    return buildCheckEndpointExistence();
-}
-function _checkResponseShapeBody(_p) {
-    return buildCheckResponseShape();
-}
-function _checkAuthAndAuthorizationBody(_p) {
-    return buildCheckAuthAndAuthorization();
-}
-function _checkBehavioralContractBody(_p) {
-    return buildCheckBehavioralContract();
-}
-function _checkAssignActionBody(_p) {
-    return buildCheckAssignAction();
-}
-function _applyBody(_p) {
-    return buildUpdateExecutionRules();
-}
-function _callToolBody(p) {
-    return buildDriftOutputChecklist(p.existingTests.length, p.newEndpointCount ?? 0, p.stateFile);
-}
-// ── PromptPlan declaration ────────────────────────────────────────────────────
-// All steps are unconditional — both MCP and testbot callers render the same
-// five steps. The only per-caller variation is skipContextHeader (context
-// section prepended by buildDriftAnalysisPrompt, not inside the plan).
-const _plan = new PromptPlan()
-    .addPhase("maintenance", "Test Maintenance Assessment", {
-    headerLevel: "##",
-    stepFormat: "hash",
-})
-    .step("ASSESS", "Action Decision Tree — assess each existing test against the diff", _assessBody)
-    .subStep("ENDPOINT_EXISTENCE", "Endpoint existence", _checkEndpointExistenceBody)
-    .subStep("RESPONSE_SHAPE", "Request/response shape (breaking changes)", _checkResponseShapeBody)
-    .subStep("ADDITIVE_FIELDS", "Additive response fields (coverage gaps)", _checkAdditiveFieldsBody)
-    .subStep("AUTH_AUTHZ", "Auth and authorization changes", _checkAuthAndAuthorizationBody)
-    .subStep("BEHAVIORAL_CONTRACT", "Behavioral and semantic contract changes", _checkBehavioralContractBody)
-    .subStep("ASSIGN_ACTION", "Assign action", _checkAssignActionBody)
-    .step("APPLY", "Apply update execution rules", _applyBody)
-    .step("CALL_TOOL", "Submit recommendations", _callToolBody)
-    .done();
-// ── Exported step label constants ─────────────────────────────────────────────
-// Static — safe to export at module load; renumber automatically on insertion.
-/** "1" — Assess each test against the diff */
-export const DRIFT_STEP_ASSESS = _plan.labels.ASSESS; // "1"
-/** "1.1" — Endpoint existence check */
-export const DRIFT_STEP_ENDPOINT_EXISTENCE = _plan.labels.ENDPOINT_EXISTENCE; // "1.1"
-/** "1.2" — Request/response shape check */
-export const DRIFT_STEP_RESPONSE_SHAPE = _plan.labels.RESPONSE_SHAPE; // "1.2"
-/** "1.3" — Additive response fields check */
-export const DRIFT_STEP_ADDITIVE_FIELDS = _plan.labels.ADDITIVE_FIELDS; // "1.3"
-/** "1.4" — Auth and authorization changes check */
-export const DRIFT_STEP_AUTH_AUTHZ = _plan.labels.AUTH_AUTHZ; // "1.4"
-/** "1.5" — Behavioral and semantic contract changes check */
-export const DRIFT_STEP_BEHAVIORAL_CONTRACT = _plan.labels.BEHAVIORAL_CONTRACT; // "1.5"
-/** "1.6" — Assign action */
-export const DRIFT_STEP_ASSIGN_ACTION = _plan.labels.ASSIGN_ACTION; // "1.6"
-/** "2" — Apply update execution rules */
-export const DRIFT_STEP_APPLY = _plan.labels.APPLY; // "2"
-/** "3" — Submit recommendations (calls skyramp_actions) */
-export const DRIFT_STEP_CALL_TOOL = _plan.labels.CALL_TOOL; // "3"
-// ── Public builder ────────────────────────────────────────────────────────────
 export function buildDriftAnalysisPrompt(params) {
-    // Pre-compute newEndpointCount from rawDiff only when caller did not supply it.
-    // Use strict undefined check — an explicit 0 means "no new endpoints" and must
-    // not trigger a diff read.
-    let newEndpointCount = params.newEndpointCount ?? 0;
-    if (params.newEndpointCount === undefined) {
-        const rawDiff = readDiffFile(params.diffFilePath);
-        if (rawDiff) {
-            const m = rawDiff.match(/\*\*New Endpoints\*\*\s+\((\d+)\)/);
-            if (m)
-                newEndpointCount = parseInt(m[1], 10);
-        }
+    const { existingTests, scannedEndpoints, repositoryPath, stateFile, routerMountContext, candidateRouteFiles, diffFilePath } = params;
+    // Read raw diff once — used for both the inline summary block and the per-line file reference.
+    const rawDiff = readDiffFile(diffFilePath);
+    let newEndpointCount = 0;
+    let diffSection = "";
+    if (rawDiff) {
+        const lines = rawDiff.split("\n");
+        const newEndpointMatch = rawDiff.match(/\*\*New Endpoints\*\*\s+\((\d+)\)/);
+        if (newEndpointMatch)
+            newEndpointCount = parseInt(newEndpointMatch[1], 10);
+        diffSection = `## Branch Diff
+\`\`\`
+${lines.slice(0, 200).join("\n")}
+\`\`\`
+`;
+    }
+    const testListSection = existingTests.length > 0
+        ? `## Existing Test Files (${existingTests.length})
+${existingTests.map((t) => `- ${t.testFile} (${t.testType})`).join("\n")}
+`
+        : `## Existing Test Files
+No existing Skyramp tests found in repository.
+`;
+    const scannedSection = scannedEndpoints.length > 0
+        ? `## Scanned Endpoints (${scannedEndpoints.length})
+Note: paths below come from static analysis and may be incomplete for nested resources or unsupported frameworks. Use the Routing entry-point files section below to verify and reconstruct full paths.
+${scannedEndpoints.map((ep) => {
+            let methods;
+            if (Array.isArray(ep.methods)) {
+                methods = ep.methods.map((m) => (typeof m === "string" ? m : m.method)).join("|");
+            }
+            else {
+                methods = ep.method;
+            }
+            return `- ${methods} ${ep.path}`;
+        }).join("\n")}
+`
+        : "";
+    const mountSection = routerMountContext?.length
+        ? `## Routing entry-point files
+Read these to trace the full router/module hierarchy when verifying endpoint paths:
+${routerMountContext.map(f => `- \`${f}\``).join("\n")}
+`
+        : "";
+    const hasJavaFiles = candidateRouteFiles?.some(f => /\.(java|kt)$/.test(f)) ?? false;
+    const candidateFilesSection = candidateRouteFiles && candidateRouteFiles.length > 0
+        ? `## Route Files (read these to find endpoints from any framework)
+${candidateRouteFiles.map(f => `- ${f}`).join("\n")}
+${hasJavaFiles ? "Note — Java Spring: full URL = class-level `@RequestMapping` prefix + method-level path. If the prefix is a constant reference (e.g. `@RequestMapping(Url.PAGE_URL)`), find the constant — same file, inner class, or a separate `Url.java` — and resolve it (including `+` concatenation)." : ""}
+`
+        : "";
+    const diffFileSection = diffFilePath
+        ? `## Raw Diff File
+Read \`${diffFilePath}\` to get the full line-by-line diff. Use it to detect:
+- Additive response fields: lines starting with \`+\` inside a view/serializer/controller (e.g. \`+ "newField":\`, \`+ newField =\`)
+- Renamed routes: \`-  @app.route("/old")\` / \`+  @app.route("/new")\` or similar framework patterns
+- Status code changes: \`- return 200\` / \`+ return 201\`, \`- res.status(200)\` / \`+ res.status(204)\`
+- Auth additions/removals: \`+ @require_auth\`, \`- @login_required\`, middleware changes
+Read the file once and cache its contents — it is the primary source for per-line breaking-change detection. Use it as evidence for Checks A–D below.
+`
+        : "";
+    // In inline mode (testbot), skip the context header — existing tests and diff
+    // are provided by skyramp_analyze_changes at runtime, not at prompt-build time.
+    const contextSection = isTestbotEnabled()
+        ? ""
+        : `# Test Health Analysis
+
+**Repository**: \`${repositoryPath}\`
+**Existing tests**: ${existingTests.length}
+**New endpoints in diff**: ${newEndpointCount}
+
+${diffSection}
+${diffFileSection}
+${testListSection}
+${scannedSection}
+${mountSection}
+${candidateFilesSection}`;
+    if (isTestbotEnabled()) {
+        // Testbot inline mode: all maintenance logic lives here so the testbot
+        // prompt only orchestrates steps without duplicating rules.
+        // No persona statement here — the outer testbot prompt already establishes
+        // the agent's context; a nested identity statement causes role confusion.
+        return `<drift_analysis_rules>
+${buildActionDecisionMatrix()}
+${buildUpdateExecutionRules()}
+${buildDriftOutputChecklist(existingTests.length, newEndpointCount, isTestbotEnabled())}
+</drift_analysis_rules>`;
     }
-    const resolvedParams = { ...params, newEndpointCount };
-    // Always emit the lean wrapped form — context is already in the conversation
-    // from skyramp_analyze_changes, which always runs before this tool.
-    return `<drift_analysis_rules>\n${_plan.render(resolvedParams)}\n</drift_analysis_rules>`;
+    return `You are acting as a Skyramp Integration Architect. Your responsibility is to assess each existing test against the branch diff and determine the correct maintenance action.
+
+${contextSection}
+${buildActionDecisionMatrix()}
+${buildBreakingChangePatterns()}
+${buildTestAssessmentGuidelines()}
+${buildUpdateExecutionRules()}
+${buildAddRecommendationGuidelines()}
+${buildDriftOutputChecklist(existingTests.length, newEndpointCount, isTestbotEnabled(), stateFile)}`;
 }

package/build/prompts/test-maintenance/drift-analysis-prompt.test.js

@@ -1,84 +1,116 @@
-import { buildDriftAnalysisPrompt, DRIFT_STEP_ASSESS, DRIFT_STEP_ENDPOINT_EXISTENCE, DRIFT_STEP_RESPONSE_SHAPE, DRIFT_STEP_ADDITIVE_FIELDS, DRIFT_STEP_AUTH_AUTHZ, DRIFT_STEP_BEHAVIORAL_CONTRACT, DRIFT_STEP_ASSIGN_ACTION, DRIFT_STEP_APPLY, DRIFT_STEP_CALL_TOOL, } from "./drift-analysis-prompt.js";
+import { buildDriftAnalysisPrompt } from "./drift-analysis-prompt.js";
 import { buildDriftOutputChecklist } from "./driftAnalysisSections.js";
-const STATE_FILE = "/tmp/skyramp-analysis-abc123.json";
-// ── Step label constants ──────────────────────────────────────────────────────
-describe("DRIFT_STEP_* label constants", () => {
-    it("main steps are sequentially numbered from 1", () => {
-        expect(DRIFT_STEP_ASSESS).toBe("1");
-        expect(DRIFT_STEP_APPLY).toBe("2");
-        expect(DRIFT_STEP_CALL_TOOL).toBe("3");
-    });
-    it("sub-steps are numbered within their parent", () => {
-        expect(DRIFT_STEP_ENDPOINT_EXISTENCE).toBe("1.1");
-        expect(DRIFT_STEP_RESPONSE_SHAPE).toBe("1.2");
-        expect(DRIFT_STEP_ADDITIVE_FIELDS).toBe("1.3");
-        expect(DRIFT_STEP_AUTH_AUTHZ).toBe("1.4");
-        expect(DRIFT_STEP_BEHAVIORAL_CONTRACT).toBe("1.5");
-        expect(DRIFT_STEP_ASSIGN_ACTION).toBe("1.6");
-    });
-});
-// ── buildDriftOutputChecklist ─────────────────────────────────────────────────
-describe("buildDriftOutputChecklist", () => {
-    it("includes recommendations, updateInstructions, and skyramp_actions CTA", () => {
-        const checklist = buildDriftOutputChecklist(3, 0, STATE_FILE);
+describe("buildDriftOutputChecklist — final-step recommendations guidance", () => {
+    const STATE_FILE = "/tmp/skyramp-analysis-abc123.json";
+    it("non-inline mode includes recommendations and updateInstructions in final step", () => {
+        const checklist = buildDriftOutputChecklist(3, 0, false, STATE_FILE);
+        // Must instruct the LLM to pass recommendations to skyramp_actions
         expect(checklist).toContain("recommendations");
+        // Must mention updateInstructions so the LLM knows to populate it
         expect(checklist).toContain("updateInstructions");
+        // Must reference the stateFile path
         expect(checklist).toContain(STATE_FILE);
+        // Must call skyramp_actions as the final action
         expect(checklist).toContain("skyramp_actions");
     });
-    it("does not contain JSON shape — schema is authoritative", () => {
-        const checklist = buildDriftOutputChecklist(3, 0, STATE_FILE);
+    it("non-inline mode does not contain JSON shape — schema is authoritative", () => {
+        const checklist = buildDriftOutputChecklist(3, 0, false, STATE_FILE);
+        // The JSON shape was moved to inputSchema — prompt must not duplicate it
         expect(checklist).not.toContain('"testFile":');
         expect(checklist).not.toContain('"action":');
     });
-    it("CTA appears exactly once", () => {
-        const checklist = buildDriftOutputChecklist(3, 0, STATE_FILE);
-        const ctaCount = (checklist.match(/call `skyramp_actions`/g) || []).length;
-        expect(ctaCount).toBe(1);
+    it("inline mode does not reference skyramp_actions or stateFile", () => {
+        const checklist = buildDriftOutputChecklist(3, 0, true, STATE_FILE);
+        // Inline mode applies changes directly — no skyramp_actions call
+        expect(checklist).not.toContain("skyramp_actions");
+        expect(checklist).not.toContain(STATE_FILE);
+    });
+    it("full prompt (non-inline) includes recommendations guidance", () => {
+        const prompt = buildDriftAnalysisPrompt({
+            existingTests: [],
+            scannedEndpoints: [],
+            repositoryPath: "/repo",
+            stateFile: STATE_FILE,
+        });
+        expect(prompt).toContain("recommendations");
+        expect(prompt).toContain("updateInstructions");
     });
 });
-// ── buildDriftAnalysisPrompt ──────────────────────────────────────────────────
-describe("buildDriftAnalysisPrompt", () => {
-    function prompt() {
+describe("buildDriftAnalysisPrompt - inline mode", () => {
+    beforeEach(() => { process.env.SKYRAMP_FEATURE_TESTBOT = "1"; });
+    afterEach(() => { delete process.env.SKYRAMP_FEATURE_TESTBOT; });
+    function inlinePrompt() {
         return buildDriftAnalysisPrompt({
             existingTests: [],
             scannedEndpoints: [],
             repositoryPath: "/repo",
-            stateFile: STATE_FILE,
+            // stateFile omitted → inline mode
         });
     }
-    it("wraps output in drift_analysis_rules XML tags", () => {
-        expect(prompt()).toContain("<drift_analysis_rules>");
-        expect(prompt()).toContain("</drift_analysis_rules>");
+    it("wraps inline rules in drift_analysis_rules XML tags", () => {
+        const prompt = inlinePrompt();
+        expect(prompt).toContain("<drift_analysis_rules>");
+        expect(prompt).toContain("</drift_analysis_rules>");
     });
-    it("does not contain the persona statement or context header", () => {
-        expect(prompt()).not.toContain("You are acting as a Skyramp Integration Architect");
-        expect(prompt()).not.toContain("# Test Health Analysis");
+    it("does not contain the persona statement", () => {
+        const prompt = inlinePrompt();
+        expect(prompt).not.toContain("You are acting as a Skyramp Integration Architect");
     });
-    it("includes recommendations guidance and updateInstructions", () => {
-        expect(prompt()).toContain("recommendations");
-        expect(prompt()).toContain("updateInstructions");
+    it("does not contain the standalone Test Health Analysis header", () => {
+        const prompt = inlinePrompt();
+        expect(prompt).not.toContain("# Test Health Analysis");
     });
-    it("includes all PromptPlan steps", () => {
-        const p = prompt();
-        expect(p).toContain(`### Step ${DRIFT_STEP_ASSESS}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_ENDPOINT_EXISTENCE}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_RESPONSE_SHAPE}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_ADDITIVE_FIELDS}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_AUTH_AUTHZ}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_BEHAVIORAL_CONTRACT}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_ASSIGN_ACTION}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_APPLY}:`);
-        expect(p).toContain(`### Step ${DRIFT_STEP_CALL_TOOL}:`);
+    it("does not contain the skyramp_actions CTA (that belongs to standalone mode)", () => {
+        const prompt = inlinePrompt();
+        // Inline mode final step directs applying changes directly, not calling skyramp_actions
+        expect(prompt).not.toContain("call `skyramp_actions`");
     });
-    it("skyramp_actions CTA appears exactly once", () => {
-        const ctaCount = (prompt().match(/call `skyramp_actions`/g) || []).length;
+});
+describe("buildDriftAnalysisPrompt - scanned endpoints rendering", () => {
+    // Reproduces the [object Object] bug: skeletonEndpoints from analyzeChangesTool
+    // stores methods as objects { method: string, ... }, not plain strings.
+    const skeletonMethodObjects = [
+        {
+            path: "/api/v1/",
+            methods: [{ method: "GET", description: "", queryParams: [], authRequired: true, sourceFile: "main.py", interactions: [] }],
+            resourceGroup: "v1",
+            pathParams: [],
+        },
+        {
+            path: "/api/v1/orders",
+            methods: [
+                { method: "GET", description: "", queryParams: [], authRequired: true, sourceFile: "orders.py", interactions: [] },
+                { method: "POST", description: "", queryParams: [], authRequired: true, sourceFile: "orders.py", interactions: [] },
+            ],
+            resourceGroup: "orders",
+            pathParams: [],
+        },
+    ];
+    it("renders HTTP methods as strings, not [object Object]", () => {
+        const prompt = buildDriftAnalysisPrompt({
+            existingTests: [],
+            scannedEndpoints: skeletonMethodObjects,
+            repositoryPath: "/repo",
+            stateFile: "/tmp/state.json",
+        });
+        expect(prompt).not.toContain("[object Object]");
+        expect(prompt).toContain("GET /api/v1/");
+        expect(prompt).toContain("GET|POST /api/v1/orders");
+        // CTA should appear exactly once (not duplicated)
+        const ctaCount = (prompt.match(/call `skyramp_actions`/g) || []).length;
         expect(ctaCount).toBe(1);
     });
+    it("also works with plain string methods (ScannedEndpoint format)", () => {
+        const stringMethods = [
+            { path: "/api/v1/products", methods: ["GET", "POST"], sourceFile: "products.py" },
+        ];
+        const prompt = buildDriftAnalysisPrompt({
+            existingTests: [],
+            scannedEndpoints: stringMethods,
+            repositoryPath: "/repo",
+            stateFile: "/tmp/state.json",
+        });
+        expect(prompt).not.toContain("[object Object]");
+        expect(prompt).toContain("GET|POST /api/v1/products");
+    });
 });
-// ── Scanned endpoints no longer in prompt output ─────────────────────────────
-// The context header (repo, diff, test list, scanned endpoints) was removed —
-// skyramp_analyze_changes already delivers that context to the conversation.
-// The scanned endpoints rendering tests were removed along with the header.
-// The [object Object] bug that was guarded against is no longer reachable via
-// this prompt path.