@skyramp/mcp 0.3.1 → 0.3.2-rc.pom-2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +2 -1
- package/build/prompts/code-reuse.d.ts +7 -1
- package/build/prompts/code-reuse.js +8 -5
- package/build/prompts/code-reuse.test.d.ts +1 -0
- package/build/prompts/code-reuse.test.js +62 -0
- package/build/prompts/pom-aware-code-reuse.d.ts +6 -1
- package/build/prompts/pom-aware-code-reuse.js +100 -53
- package/build/prompts/pom-aware-code-reuse.test.d.ts +1 -0
- package/build/prompts/pom-aware-code-reuse.test.js +11 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.d.ts +4 -2
- package/build/prompts/test-recommendation/diffExecutionPlan.js +11 -65
- package/build/prompts/test-recommendation/fullRepoCatalog.d.ts +2 -2
- package/build/prompts/test-recommendation/recommendationSections.js +5 -2
- package/build/prompts/test-recommendation/scopeAssessment.js +1 -1
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +26 -1
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +68 -56
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +48 -11
- package/build/prompts/testbot/testbot-prompts.d.ts +1 -1
- package/build/prompts/testbot/testbot-prompts.js +67 -21
- package/build/prompts/testbot/testbot-prompts.test.js +44 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.d.ts +7 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.js +71 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.test.d.ts +1 -0
- package/build/recommendation/budgeters/diversityBalancedBudgeter.test.js +75 -0
- package/build/recommendation/budgeters/fixedNBudgeter.d.ts +7 -0
- package/build/recommendation/budgeters/fixedNBudgeter.js +11 -0
- package/build/recommendation/budgeters/fixedNBudgeter.test.d.ts +1 -0
- package/build/recommendation/budgeters/fixedNBudgeter.test.js +66 -0
- package/build/recommendation/budgeters/shared.d.ts +19 -0
- package/build/recommendation/budgeters/shared.js +66 -0
- package/build/recommendation/discriminators.d.ts +31 -0
- package/build/recommendation/discriminators.js +355 -0
- package/build/recommendation/discriminators.test.d.ts +1 -0
- package/build/recommendation/discriminators.test.js +324 -0
- package/build/recommendation/diversity.d.ts +47 -0
- package/build/recommendation/diversity.js +101 -0
- package/build/recommendation/diversity.test.d.ts +1 -0
- package/build/recommendation/diversity.test.js +77 -0
- package/build/recommendation/planRanker.d.ts +50 -0
- package/build/recommendation/planRanker.js +67 -0
- package/build/recommendation/planRanker.test.d.ts +1 -0
- package/build/recommendation/planRanker.test.js +110 -0
- package/build/recommendation/testFixtures.d.ts +25 -0
- package/build/recommendation/testFixtures.js +45 -0
- package/build/resources/testbotResource.js +4 -1
- package/build/services/ScenarioGenerationService.d.ts +5 -0
- package/build/services/ScenarioGenerationService.js +16 -1
- package/build/services/ScenarioGenerationService.test.js +44 -0
- package/build/services/TestExecutionService.d.ts +15 -1
- package/build/services/TestExecutionService.js +210 -55
- package/build/services/TestExecutionService.test.js +397 -0
- package/build/services/TestGenerationService.js +19 -1
- package/build/services/TestGenerationService.test.js +58 -0
- package/build/tool-phases.js +1 -0
- package/build/toolNames.d.ts +19 -0
- package/build/toolNames.js +19 -0
- package/build/tools/code-refactor/codeReuseTool.d.ts +7 -0
- package/build/tools/code-refactor/codeReuseTool.js +130 -4
- package/build/tools/code-refactor/codeReuseTool.test.d.ts +1 -0
- package/build/tools/code-refactor/codeReuseTool.test.js +290 -0
- package/build/tools/executeSkyrampTestTool.js +8 -2
- package/build/tools/generate-tests/generateBatchScenarioRestTool.d.ts +6 -1
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +110 -17
- package/build/tools/generate-tests/generateBatchScenarioRestTool.test.js +147 -0
- package/build/tools/generate-tests/generateContractRestTool.js +11 -1
- package/build/tools/generate-tests/generateIntegrationRestTool.js +24 -1
- package/build/tools/generate-tests/generateIntegrationRestTool.test.d.ts +1 -0
- package/build/tools/generate-tests/generateIntegrationRestTool.test.js +159 -0
- package/build/tools/generate-tests/planGuard.d.ts +13 -0
- package/build/tools/generate-tests/planGuard.js +78 -0
- package/build/tools/generate-tests/planGuard.test.d.ts +1 -0
- package/build/tools/generate-tests/planGuard.test.js +185 -0
- package/build/tools/generate-tests/scenarioFileIdentity.d.ts +10 -0
- package/build/tools/generate-tests/scenarioFileIdentity.js +46 -0
- package/build/tools/generate-tests/scenarioLint.d.ts +30 -0
- package/build/tools/generate-tests/scenarioLint.js +150 -0
- package/build/tools/generate-tests/scenarioLint.test.d.ts +1 -0
- package/build/tools/generate-tests/scenarioLint.test.js +100 -0
- package/build/tools/submitReportTool.js +78 -0
- package/build/tools/submitReportTool.test.js +255 -0
- package/build/tools/test-management/analyzeChangesTool.js +55 -2
- package/build/tools/test-management/analyzeChangesTool.test.js +12 -0
- package/build/tools/test-management/index.d.ts +1 -0
- package/build/tools/test-management/index.js +1 -0
- package/build/tools/test-management/registerTestPlanTool.d.ts +2 -0
- package/build/tools/test-management/registerTestPlanTool.js +329 -0
- package/build/tools/test-management/registerTestPlanTool.test.d.ts +1 -0
- package/build/tools/test-management/registerTestPlanTool.test.js +296 -0
- package/build/types/Recommendation.d.ts +97 -0
- package/build/types/Recommendation.js +48 -0
- package/build/types/RepositoryAnalysis.d.ts +14 -14
- package/build/types/TestExecution.d.ts +2 -0
- package/build/types/TestRecommendation.d.ts +12 -1
- package/build/types/TestRecommendation.js +26 -11
- package/build/types/TestTypes.js +1 -1
- package/build/utils/AnalysisStateManager.d.ts +47 -0
- package/build/utils/docker.test.js +1 -1
- package/build/utils/planMatchKeys.d.ts +61 -0
- package/build/utils/planMatchKeys.js +125 -0
- package/build/utils/pom-scope/import-expansion.d.ts +5 -0
- package/build/utils/pom-scope/import-expansion.js +32 -0
- package/build/utils/pom-scope/index.d.ts +39 -0
- package/build/utils/pom-scope/index.js +120 -0
- package/build/utils/pom-scope/index.test.d.ts +1 -0
- package/build/utils/pom-scope/index.test.js +239 -0
- package/build/utils/pom-scope/pom-files.d.ts +3 -0
- package/build/utils/pom-scope/pom-files.js +48 -0
- package/build/utils/pom-scope/pom-files.test.d.ts +1 -0
- package/build/utils/pom-scope/pom-files.test.js +29 -0
- package/build/utils/pom-scope/scoring.d.ts +20 -0
- package/build/utils/pom-scope/scoring.js +45 -0
- package/build/utils/pom-scope/scoring.test.d.ts +1 -0
- package/build/utils/pom-scope/scoring.test.js +39 -0
- package/build/utils/pom-scope/selector-extractor.d.ts +7 -0
- package/build/utils/pom-scope/selector-extractor.js +57 -0
- package/build/utils/pom-scope/selector-extractor.test.d.ts +1 -0
- package/build/utils/pom-scope/selector-extractor.test.js +67 -0
- package/build/utils/pom-verify/__fixtures__/af-style/asset-list.page.d.ts +5 -0
- package/build/utils/pom-verify/__fixtures__/af-style/asset-list.page.js +5 -0
- package/build/utils/pom-verify/__fixtures__/af-style/pageobjects/asset-list-page.d.ts +5 -0
- package/build/utils/pom-verify/__fixtures__/af-style/pageobjects/asset-list-page.js +9 -0
- package/build/utils/pom-verify/__fixtures__/af-style/report.iframe.page.d.ts +4 -0
- package/build/utils/pom-verify/__fixtures__/af-style/report.iframe.page.js +4 -0
- package/build/utils/pom-verify/__fixtures__/af-style/workflow-footer.page.d.ts +4 -0
- package/build/utils/pom-verify/__fixtures__/af-style/workflow-footer.page.js +4 -0
- package/build/utils/pom-verify/bindings.d.ts +19 -0
- package/build/utils/pom-verify/bindings.js +161 -0
- package/build/utils/pom-verify/bindings.test.d.ts +1 -0
- package/build/utils/pom-verify/bindings.test.js +164 -0
- package/build/utils/pom-verify/calls.d.ts +16 -0
- package/build/utils/pom-verify/calls.js +42 -0
- package/build/utils/pom-verify/calls.test.d.ts +1 -0
- package/build/utils/pom-verify/calls.test.js +61 -0
- package/build/utils/pom-verify/index.d.ts +4 -0
- package/build/utils/pom-verify/index.js +4 -0
- package/build/utils/pom-verify/resolve.d.ts +7 -0
- package/build/utils/pom-verify/resolve.js +27 -0
- package/build/utils/pom-verify/resolve.test.d.ts +1 -0
- package/build/utils/pom-verify/resolve.test.js +68 -0
- package/build/utils/pom-verify/strip.d.ts +9 -0
- package/build/utils/pom-verify/strip.js +89 -0
- package/build/utils/pom-verify/verify.d.ts +14 -0
- package/build/utils/pom-verify/verify.js +158 -0
- package/build/utils/pom-verify/verify.test.d.ts +1 -0
- package/build/utils/pom-verify/verify.test.js +325 -0
- package/build/utils/reportVerification.d.ts +61 -0
- package/build/utils/reportVerification.js +104 -0
- package/build/utils/reportVerification.test.d.ts +1 -0
- package/build/utils/reportVerification.test.js +185 -0
- package/build/utils/scenarioDrafting.js +5 -5
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/utils/workspaceAuth.d.ts +9 -1
- package/build/utils/workspaceAuth.js +25 -5
- package/build/utils/workspaceAuth.test.js +48 -0
- package/build/workspace/workspace.d.ts +20 -0
- package/build/workspace/workspace.js +4 -0
- package/build/workspace/workspace.test.js +10 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +77 -8
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/pollingRecorderSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/utils/isomorphic/volatileDate.js +101 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/utils.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-aszq5EdG.js → codeMirrorModule-Bzd72-bG.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-BxS7Jm4s.js → defaultSettingsView-DzxTioTK.js} +101 -101
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.D4JTTy4R.js → index.BGc30U3S.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.DaRMQKOI.js → uiMode.IaDrb29A.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/pollingRecorderSource.ts +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/utils/isomorphic/volatileDate.ts +131 -0
- package/node_modules/playwright/node_modules/playwright-core/src/utils.ts +1 -0
- package/node_modules/playwright/package.json +1 -1
- package/package.json +3 -3
- package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
|
@@ -9,6 +9,23 @@ import { DriftAction } from "../types/TestAnalysis.js";
|
|
|
9
9
|
import { TestExecutionStatus } from "../types/TestExecution.js";
|
|
10
10
|
import { StateManager } from "../utils/AnalysisStateManager.js";
|
|
11
11
|
import { toolError, testFileMatches } from "../utils/utils.js";
|
|
12
|
+
import { matchesApprovedPlan } from "../utils/planMatchKeys.js";
|
|
13
|
+
import { isTestbotEnabled } from "../utils/featureFlags.js";
|
|
14
|
+
import { findUnbackedClaims, listChangedFiles } from "../utils/reportVerification.js";
|
|
15
|
+
// SKYR-3879 Path B: which testTypes the register-plan checkpoint gates. Mirrors
|
|
16
|
+
// the generation tools actually wired to planGuard (batch-scenario/integration,
|
|
17
|
+
// contract) — UI and E2E are on a separate blueprint-grounded pipeline and are
|
|
18
|
+
// NOT gated at generation time (see planGuard.ts wiring), so they are excluded
|
|
19
|
+
// here too rather than surprising the agent with a report-time-only gate.
|
|
20
|
+
const PLAN_GATED_TEST_TYPES = new Set([TestType.CONTRACT, TestType.INTEGRATION]);
|
|
21
|
+
/** Split an `endpoint` field ("GET /api/v1/products") into method + path. */
|
|
22
|
+
function parseEndpointField(endpoint) {
|
|
23
|
+
const trimmed = (endpoint ?? "").trim();
|
|
24
|
+
const spaceIdx = trimmed.indexOf(" ");
|
|
25
|
+
if (spaceIdx <= 0)
|
|
26
|
+
return { path: trimmed || undefined };
|
|
27
|
+
return { method: trimmed.slice(0, spaceIdx), path: trimmed.slice(spaceIdx + 1).trim() };
|
|
28
|
+
}
|
|
12
29
|
// Drift actions that actually modify a test file. VERIFY and IGNORE are
|
|
13
30
|
// no-ops (the test was assessed but left unchanged), so they must not count
|
|
14
31
|
// toward "tests maintained" telemetry.
|
|
@@ -430,6 +447,33 @@ export function registerSubmitReportTool(server) {
|
|
|
430
447
|
"Call skyramp_actions (with recommendations: [] if no existing tests needed action) before skyramp_submit_report.");
|
|
431
448
|
return errorResult;
|
|
432
449
|
}
|
|
450
|
+
// SKYR-3879 Path B: every gated newTestsCreated entry (contract/integration —
|
|
451
|
+
// the testTypes actually wired to planGuard) must be admissible under the
|
|
452
|
+
// approved plan from skyramp_register_test_plan — the same predicate the
|
|
453
|
+
// generation-tool gate uses (GENERATE, or ADDITIONAL backfill when GENERATE
|
|
454
|
+
// is non-empty), so a generation the gate permitted is never rejected at
|
|
455
|
+
// report time. Absent approvedPlan (register tool never ran / never
|
|
456
|
+
// pre-seeded anything) → no check, back-compat.
|
|
457
|
+
if (stateData.approvedPlan) {
|
|
458
|
+
const approvedPlan = stateData.approvedPlan;
|
|
459
|
+
const unapproved = dedupedNewTests.filter((t) => {
|
|
460
|
+
if (!PLAN_GATED_TEST_TYPES.has(t.testType))
|
|
461
|
+
return false;
|
|
462
|
+
const { method, path } = parseEndpointField(t.endpoint);
|
|
463
|
+
return !matchesApprovedPlan(approvedPlan, { testType: t.testType, method, path });
|
|
464
|
+
});
|
|
465
|
+
if (unapproved.length > 0) {
|
|
466
|
+
const approvedList = approvedPlan.generate.length > 0
|
|
467
|
+
? approvedPlan.generate.map((item) => `[${item.testType}] ${item.scenarioName}`).join(", ")
|
|
468
|
+
: "(none)";
|
|
469
|
+
errorResult = toolError(`${unapproved.length} newTestsCreated entr${unapproved.length === 1 ? "y" : "ies"} not in the approved plan from ` +
|
|
470
|
+
`skyramp_register_test_plan (plan ${approvedPlan.planId}) — neither its GENERATE list nor its ADDITIONAL backfill pool: ` +
|
|
471
|
+
`${unapproved.map((t) => `${t.testId} (${t.testType} ${t.endpoint})`).join(", ")}. ` +
|
|
472
|
+
`Approved GENERATE items: ${approvedList}. Call skyramp_register_test_plan to register this test before reporting it, ` +
|
|
473
|
+
`or remove it from newTestsCreated.`);
|
|
474
|
+
return errorResult;
|
|
475
|
+
}
|
|
476
|
+
}
|
|
433
477
|
// TestbotReport.testMaintenance has no repository field, so a related repo's
|
|
434
478
|
// reconciled verdicts have nowhere to go in this report. Fail loud instead of
|
|
435
479
|
// silently omitting real maintenance work skyramp_actions already did and persisted.
|
|
@@ -493,6 +537,40 @@ export function registerSubmitReportTool(server) {
|
|
|
493
537
|
"Add a testMaintenanceDetails entry with the missing field(s), drafted from the execution output you already saw.");
|
|
494
538
|
return errorResult;
|
|
495
539
|
}
|
|
540
|
+
// SKYR-3883: in a testbot run, refuse to ship a report that claims file work
|
|
541
|
+
// the working tree doesn't reflect. The delivery step can only commit what the
|
|
542
|
+
// agent actually created/edited, so a report claiming otherwise erodes trust
|
|
543
|
+
// (a reviewer believes maintenance happened that never landed). Fail loudly so
|
|
544
|
+
// the agent makes the edit — or drops the claim — instead of hallucinating it.
|
|
545
|
+
// Only the working tree exists at report time (delivery commits later), so we
|
|
546
|
+
// verify against created/modified/untracked paths, not the eventual commit.
|
|
547
|
+
if (isTestbotEnabled()) {
|
|
548
|
+
const fullState = await stateManager.readFullState();
|
|
549
|
+
const repoRoot = fullState?.metadata?.repositoryPath;
|
|
550
|
+
if (repoRoot && repoRoot !== "unknown") {
|
|
551
|
+
try {
|
|
552
|
+
const changedFiles = await listChangedFiles(repoRoot);
|
|
553
|
+
const unbacked = findUnbackedClaims({
|
|
554
|
+
repoRoot,
|
|
555
|
+
changedFiles,
|
|
556
|
+
newTests: dedupedNewTests,
|
|
557
|
+
verdicts: stateData.maintenanceVerdicts ?? [],
|
|
558
|
+
primaryRepository: fullState?.metadata?.repository,
|
|
559
|
+
});
|
|
560
|
+
if (unbacked.length > 0) {
|
|
561
|
+
errorResult = toolError(`${unbacked.length} report claim(s) are not backed by any change in the working tree — ` +
|
|
562
|
+
`Testbot will not report file work that hasn't actually been made. Make the edit (or remove the claim), then resubmit:\n` +
|
|
563
|
+
unbacked.map((u) => ` - ${u}`).join("\n"));
|
|
564
|
+
return errorResult;
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
catch (err) {
|
|
568
|
+
// Never block a valid report because verification itself failed (path not a
|
|
569
|
+
// git repo, git unavailable) — degrade to a warning instead of false-rejecting.
|
|
570
|
+
logger.warning(`Skipping report/working-tree verification (could not enumerate changes): ${err.message}`);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
}
|
|
496
574
|
// Strip generation-artifact fields from newTestsCreated before writing.
|
|
497
575
|
// scenarioFile, traceFile, frontendTrace are internal paths used during
|
|
498
576
|
// generation — downstream scoring scripts don't expect them and fail if
|
|
@@ -6,6 +6,7 @@ import { TestExecutionStatus } from "../types/TestExecution.js";
|
|
|
6
6
|
import * as fs from "fs/promises";
|
|
7
7
|
import * as path from "path";
|
|
8
8
|
import * as os from "os";
|
|
9
|
+
import { execFileSync } from "child_process";
|
|
9
10
|
import { z } from "zod";
|
|
10
11
|
jest.unstable_mockModule("../utils/logger.js", () => ({
|
|
11
12
|
logger: { info: jest.fn(), error: jest.fn(), warning: jest.fn(), debug: jest.fn() },
|
|
@@ -16,6 +17,7 @@ jest.unstable_mockModule("../services/AnalyticsService.js", () => ({
|
|
|
16
17
|
// @ts-ignore
|
|
17
18
|
const { registerSubmitReportTool, additionalRecommendationSchema, newTestSchema, targetElementSchema, pageContextSchema } = await import("./submitReportTool.js");
|
|
18
19
|
const { AnalyticsService } = await import("../services/AnalyticsService.js");
|
|
20
|
+
const { CandidateSource } = await import("../types/Recommendation.js");
|
|
19
21
|
const pushMCPToolEventMock = AnalyticsService.pushMCPToolEvent;
|
|
20
22
|
function captureToolHandler() {
|
|
21
23
|
let handler;
|
|
@@ -1170,4 +1172,257 @@ describe("registerSubmitReportTool", () => {
|
|
|
1170
1172
|
expect(result.content[0].text).not.toContain("beforeDetails");
|
|
1171
1173
|
});
|
|
1172
1174
|
});
|
|
1175
|
+
describe("approvedPlan cross-check (SKYR-3879 Path B)", () => {
|
|
1176
|
+
async function writeStateFileWithPlan(dir, approvedPlan) {
|
|
1177
|
+
const stateFile = path.join(dir, "analyze-changes-state.json");
|
|
1178
|
+
await fs.writeFile(stateFile, JSON.stringify({
|
|
1179
|
+
existingTests: [],
|
|
1180
|
+
maintenanceVerdicts: [],
|
|
1181
|
+
...(approvedPlan ? { approvedPlan } : {}),
|
|
1182
|
+
metadata: sampleMetadata(dir),
|
|
1183
|
+
}), "utf-8");
|
|
1184
|
+
return stateFile;
|
|
1185
|
+
}
|
|
1186
|
+
const APPROVED_PLAN = {
|
|
1187
|
+
planId: "plan-42",
|
|
1188
|
+
createdAt: "2026-01-01T00:00:00.000Z",
|
|
1189
|
+
generate: [
|
|
1190
|
+
{
|
|
1191
|
+
candidateId: "products-search-abcd1234",
|
|
1192
|
+
scenarioName: "products_search",
|
|
1193
|
+
testType: "contract",
|
|
1194
|
+
category: "business_rule",
|
|
1195
|
+
source: CandidateSource.AGENT,
|
|
1196
|
+
matchKeys: ["name:products-search", "ep:contract:GET:/api/v1/products/search"],
|
|
1197
|
+
},
|
|
1198
|
+
],
|
|
1199
|
+
additional: [],
|
|
1200
|
+
demotions: [],
|
|
1201
|
+
};
|
|
1202
|
+
function contractNewTest(overrides = {}) {
|
|
1203
|
+
return {
|
|
1204
|
+
testId: "contract-get-products-search",
|
|
1205
|
+
testType: TestType.CONTRACT,
|
|
1206
|
+
category: "business_rule",
|
|
1207
|
+
endpoint: "GET /api/v1/products/search",
|
|
1208
|
+
fileName: "products_search_contract_test.py",
|
|
1209
|
+
description: "Searches products and asserts the response schema.",
|
|
1210
|
+
reasoning: "Validates the contract for the new search endpoint.",
|
|
1211
|
+
...overrides,
|
|
1212
|
+
};
|
|
1213
|
+
}
|
|
1214
|
+
it("passes when every gated newTestsCreated entry matches an approved GENERATE item", async () => {
|
|
1215
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
1216
|
+
tmpDirs.push(tmpDir);
|
|
1217
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
1218
|
+
const stateFile = await writeStateFileWithPlan(tmpDir, APPROVED_PLAN);
|
|
1219
|
+
const result = await handler({
|
|
1220
|
+
summaryOutputFile: outputFile,
|
|
1221
|
+
stateFile,
|
|
1222
|
+
businessCaseAnalysis: "Adds product search.",
|
|
1223
|
+
newTestsCreated: [contractNewTest()],
|
|
1224
|
+
testResults: [],
|
|
1225
|
+
issuesFound: [],
|
|
1226
|
+
});
|
|
1227
|
+
expect(result.isError).toBeUndefined();
|
|
1228
|
+
});
|
|
1229
|
+
it("rejects an unapproved gated entry, naming the offending entry and the approved list", async () => {
|
|
1230
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
1231
|
+
tmpDirs.push(tmpDir);
|
|
1232
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
1233
|
+
const stateFile = await writeStateFileWithPlan(tmpDir, APPROVED_PLAN);
|
|
1234
|
+
const result = await handler({
|
|
1235
|
+
summaryOutputFile: outputFile,
|
|
1236
|
+
stateFile,
|
|
1237
|
+
businessCaseAnalysis: "Adds an unregistered contract test.",
|
|
1238
|
+
newTestsCreated: [
|
|
1239
|
+
contractNewTest({
|
|
1240
|
+
testId: "contract-get-products-unregistered",
|
|
1241
|
+
endpoint: "GET /api/v1/products/unregistered",
|
|
1242
|
+
}),
|
|
1243
|
+
],
|
|
1244
|
+
testResults: [],
|
|
1245
|
+
issuesFound: [],
|
|
1246
|
+
});
|
|
1247
|
+
expect(result.isError).toBe(true);
|
|
1248
|
+
expect(result.content[0].text).toContain("plan-42");
|
|
1249
|
+
expect(result.content[0].text).toContain("contract-get-products-unregistered");
|
|
1250
|
+
expect(result.content[0].text).toContain("products_search");
|
|
1251
|
+
});
|
|
1252
|
+
it("accepts a gated entry matching an approved ADDITIONAL item — backfill parity with planGuard", async () => {
|
|
1253
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
1254
|
+
tmpDirs.push(tmpDir);
|
|
1255
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
1256
|
+
const stateFile = await writeStateFileWithPlan(tmpDir, {
|
|
1257
|
+
...APPROVED_PLAN,
|
|
1258
|
+
additional: [
|
|
1259
|
+
{
|
|
1260
|
+
candidateId: "products-backfill-efgh5678",
|
|
1261
|
+
scenarioName: "products_backfill",
|
|
1262
|
+
testType: "contract",
|
|
1263
|
+
category: "business_rule",
|
|
1264
|
+
source: CandidateSource.AGENT,
|
|
1265
|
+
matchKeys: ["name:products-backfill", "ep:contract:GET:/api/v1/products/backfill"],
|
|
1266
|
+
},
|
|
1267
|
+
],
|
|
1268
|
+
});
|
|
1269
|
+
const result = await handler({
|
|
1270
|
+
summaryOutputFile: outputFile,
|
|
1271
|
+
stateFile,
|
|
1272
|
+
businessCaseAnalysis: "GENERATE item failed; backfilled from ADDITIONAL as the plan instructs.",
|
|
1273
|
+
newTestsCreated: [
|
|
1274
|
+
contractNewTest({
|
|
1275
|
+
testId: "contract-get-products-backfill",
|
|
1276
|
+
endpoint: "GET /api/v1/products/backfill",
|
|
1277
|
+
}),
|
|
1278
|
+
],
|
|
1279
|
+
testResults: [],
|
|
1280
|
+
issuesFound: [],
|
|
1281
|
+
});
|
|
1282
|
+
expect(result.isError).toBeUndefined();
|
|
1283
|
+
});
|
|
1284
|
+
it("rejects an ADDITIONAL-matching entry when the GENERATE list is empty — zero-budget plans have nothing to backfill", async () => {
|
|
1285
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
1286
|
+
tmpDirs.push(tmpDir);
|
|
1287
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
1288
|
+
const stateFile = await writeStateFileWithPlan(tmpDir, {
|
|
1289
|
+
...APPROVED_PLAN,
|
|
1290
|
+
generate: [],
|
|
1291
|
+
additional: APPROVED_PLAN.generate,
|
|
1292
|
+
});
|
|
1293
|
+
const result = await handler({
|
|
1294
|
+
summaryOutputFile: outputFile,
|
|
1295
|
+
stateFile,
|
|
1296
|
+
businessCaseAnalysis: "Generated a test on a zero-budget plan.",
|
|
1297
|
+
newTestsCreated: [contractNewTest()],
|
|
1298
|
+
testResults: [],
|
|
1299
|
+
issuesFound: [],
|
|
1300
|
+
});
|
|
1301
|
+
expect(result.isError).toBe(true);
|
|
1302
|
+
expect(result.content[0].text).toContain("plan-42");
|
|
1303
|
+
});
|
|
1304
|
+
it("does not check newTestsCreated when approvedPlan is absent (back-compat)", async () => {
|
|
1305
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
1306
|
+
tmpDirs.push(tmpDir);
|
|
1307
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
1308
|
+
const stateFile = await writeStateFileWithPlan(tmpDir, undefined);
|
|
1309
|
+
const result = await handler({
|
|
1310
|
+
summaryOutputFile: outputFile,
|
|
1311
|
+
stateFile,
|
|
1312
|
+
businessCaseAnalysis: "No plan was ever registered for this run.",
|
|
1313
|
+
newTestsCreated: [contractNewTest({ endpoint: "GET /api/v1/products/anything" })],
|
|
1314
|
+
testResults: [],
|
|
1315
|
+
issuesFound: [],
|
|
1316
|
+
});
|
|
1317
|
+
expect(result.isError).toBeUndefined();
|
|
1318
|
+
});
|
|
1319
|
+
it("does not gate UI/E2E entries even when unmatched — those tools are outside planGuard's scope", async () => {
|
|
1320
|
+
const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-test-"));
|
|
1321
|
+
tmpDirs.push(tmpDir);
|
|
1322
|
+
const outputFile = path.join(tmpDir, "report.json");
|
|
1323
|
+
const stateFile = await writeStateFileWithPlan(tmpDir, APPROVED_PLAN);
|
|
1324
|
+
const result = await handler({
|
|
1325
|
+
summaryOutputFile: outputFile,
|
|
1326
|
+
stateFile,
|
|
1327
|
+
businessCaseAnalysis: "Adds a UI test not present in the approved plan.",
|
|
1328
|
+
newTestsCreated: [
|
|
1329
|
+
{
|
|
1330
|
+
testId: "ui-checkout-flow",
|
|
1331
|
+
testType: TestType.UI,
|
|
1332
|
+
category: "workflow",
|
|
1333
|
+
endpoint: "N/A",
|
|
1334
|
+
fileName: "checkout_ui_test.spec.ts",
|
|
1335
|
+
description: "Walks through the checkout flow. [no-blueprint-data]",
|
|
1336
|
+
reasoning: "Covers the new checkout UI. [no-blueprint-data]",
|
|
1337
|
+
targetElements: null,
|
|
1338
|
+
},
|
|
1339
|
+
],
|
|
1340
|
+
testResults: [],
|
|
1341
|
+
issuesFound: [],
|
|
1342
|
+
});
|
|
1343
|
+
expect(result.isError).toBeUndefined();
|
|
1344
|
+
});
|
|
1345
|
+
});
|
|
1346
|
+
});
|
|
1347
|
+
// SKYR-3883: end-to-end coverage of the working-tree verification gate. The unit
|
|
1348
|
+
// behavior lives in reportVerification.test.ts; these exercise the wiring through
|
|
1349
|
+
// the real tool handler (feature-flag gating, git enumeration, error surfacing).
|
|
1350
|
+
describe("submit_report working-tree verification (SKYR-3883)", () => {
|
|
1351
|
+
let handler;
|
|
1352
|
+
let tmpDirs = [];
|
|
1353
|
+
const prevFlag = process.env.SKYRAMP_FEATURE_TESTBOT;
|
|
1354
|
+
beforeAll(() => {
|
|
1355
|
+
handler = captureToolHandler();
|
|
1356
|
+
});
|
|
1357
|
+
beforeEach(() => {
|
|
1358
|
+
process.env.SKYRAMP_FEATURE_TESTBOT = "1";
|
|
1359
|
+
});
|
|
1360
|
+
afterEach(async () => {
|
|
1361
|
+
if (prevFlag === undefined)
|
|
1362
|
+
delete process.env.SKYRAMP_FEATURE_TESTBOT;
|
|
1363
|
+
else
|
|
1364
|
+
process.env.SKYRAMP_FEATURE_TESTBOT = prevFlag;
|
|
1365
|
+
for (const dir of tmpDirs)
|
|
1366
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
1367
|
+
tmpDirs = [];
|
|
1368
|
+
});
|
|
1369
|
+
async function initGitRepo() {
|
|
1370
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-wt-"));
|
|
1371
|
+
tmpDirs.push(dir);
|
|
1372
|
+
execFileSync("git", ["init", "-q"], { cwd: dir });
|
|
1373
|
+
execFileSync("git", ["config", "user.email", "t@t.dev"], { cwd: dir });
|
|
1374
|
+
execFileSync("git", ["config", "user.name", "t"], { cwd: dir });
|
|
1375
|
+
return dir;
|
|
1376
|
+
}
|
|
1377
|
+
it("rejects a report whose newTestsCreated file was never written", async () => {
|
|
1378
|
+
const dir = await initGitRepo();
|
|
1379
|
+
const outputFile = path.join(dir, "report.json");
|
|
1380
|
+
const stateFile = await writeStateFile(dir, []);
|
|
1381
|
+
// search_smoke_test.py is claimed but never created → unbacked.
|
|
1382
|
+
const result = await handler(sampleReportParams(outputFile, stateFile));
|
|
1383
|
+
expect(result.isError).toBe(true);
|
|
1384
|
+
expect(result.content[0].text).toContain("not backed");
|
|
1385
|
+
expect(result.content[0].text).toContain("search_smoke_test.py");
|
|
1386
|
+
});
|
|
1387
|
+
it("accepts the report once the claimed file exists in the working tree", async () => {
|
|
1388
|
+
const dir = await initGitRepo();
|
|
1389
|
+
const outputFile = path.join(dir, "report.json");
|
|
1390
|
+
const stateFile = await writeStateFile(dir, []);
|
|
1391
|
+
await fs.writeFile(path.join(dir, "search_smoke_test.py"), "def test_search(): pass\n");
|
|
1392
|
+
const result = await handler(sampleReportParams(outputFile, stateFile));
|
|
1393
|
+
expect(result.isError).toBeUndefined();
|
|
1394
|
+
});
|
|
1395
|
+
it("rejects an UPDATE maintenance verdict with no working-tree change, exempting DELETE", async () => {
|
|
1396
|
+
const dir = await initGitRepo();
|
|
1397
|
+
const outputFile = path.join(dir, "report.json");
|
|
1398
|
+
const stateFile = await writeReconciledStateFile(dir, [], [
|
|
1399
|
+
{ testFilePath: path.join(dir, "dcim/tests/test_filtersets.py"), action: DriftAction.Update, rationale: "physical_address filter", testType: TestType.INTEGRATION, endpoint: "GET /api/dcim/sites/" },
|
|
1400
|
+
{ testFilePath: path.join(dir, "dcim/tests/gone.py"), action: DriftAction.Delete, rationale: "obsolete", testType: TestType.INTEGRATION, endpoint: "GET /api/dcim/sites/" },
|
|
1401
|
+
]);
|
|
1402
|
+
// Keep the newTestsCreated claim backed so the failure isolates to the verdict.
|
|
1403
|
+
await fs.writeFile(path.join(dir, "search_smoke_test.py"), "x\n");
|
|
1404
|
+
const result = await handler(sampleReportParams(outputFile, stateFile));
|
|
1405
|
+
expect(result.isError).toBe(true);
|
|
1406
|
+
expect(result.content[0].text).toContain("test_filtersets.py");
|
|
1407
|
+
// DELETE is applied at delivery, never a working-tree claim — must not be flagged.
|
|
1408
|
+
expect(result.content[0].text).not.toContain("gone.py");
|
|
1409
|
+
});
|
|
1410
|
+
it("skips verification entirely when the testbot feature flag is off", async () => {
|
|
1411
|
+
delete process.env.SKYRAMP_FEATURE_TESTBOT;
|
|
1412
|
+
const dir = await initGitRepo();
|
|
1413
|
+
const outputFile = path.join(dir, "report.json");
|
|
1414
|
+
const stateFile = await writeStateFile(dir, []);
|
|
1415
|
+
// Claimed file absent, but the gate is off → report goes through.
|
|
1416
|
+
const result = await handler(sampleReportParams(outputFile, stateFile));
|
|
1417
|
+
expect(result.isError).toBeUndefined();
|
|
1418
|
+
});
|
|
1419
|
+
it("degrades to a pass when repositoryPath is not a git repo (verification can't run)", async () => {
|
|
1420
|
+
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "submit-report-nogit-"));
|
|
1421
|
+
tmpDirs.push(dir);
|
|
1422
|
+
const outputFile = path.join(dir, "report.json");
|
|
1423
|
+
const stateFile = await writeStateFile(dir, []); // repositoryPath=dir, but no git init
|
|
1424
|
+
const result = await handler(sampleReportParams(outputFile, stateFile));
|
|
1425
|
+
// git status throws → caught → warning → report not falsely rejected.
|
|
1426
|
+
expect(result.isError).toBeUndefined();
|
|
1427
|
+
});
|
|
1173
1428
|
});
|
|
@@ -10,8 +10,12 @@ import { parseWorkspaceAuthType, getDefaultAuthHeader, WorkspaceAuthType, readWo
|
|
|
10
10
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
11
11
|
import { makeProgressReporter } from "../../utils/progress.js";
|
|
12
12
|
import { StateManager, registerSession, storeSessionData, setTestsRepoDir, getActiveRunStatePath, setActiveRunStatePath, } from "../../utils/AnalysisStateManager.js";
|
|
13
|
-
import { buildRecommendationPrompt } from "../../prompts/test-recommendation/test-recommendation-prompt.js";
|
|
13
|
+
import { buildRecommendationPrompt, computeScoredCandidates } from "../../prompts/test-recommendation/test-recommendation-prompt.js";
|
|
14
14
|
import { hasFlutterSdkDep, isFrontendFile, isTestFile } from "../../prompts/test-recommendation/scopeAssessment.js";
|
|
15
|
+
import { buildExternalCoverageSet } from "../../prompts/test-recommendation/recommendationShared.js";
|
|
16
|
+
import { CandidateSource, computeCandidateId } from "../../types/Recommendation.js";
|
|
17
|
+
import { selectPlan } from "../../recommendation/planRanker.js";
|
|
18
|
+
import { buildApprovedPlanItem } from "../../utils/planMatchKeys.js";
|
|
15
19
|
import { enumerateCandidateUiPages } from "../../utils/uiPageEnumerator.js";
|
|
16
20
|
import { checkFrontendFileIntegration } from "../../utils/frontendIntegration.js";
|
|
17
21
|
import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "../../prompts/test-recommendation/recommendationSections.js";
|
|
@@ -1278,6 +1282,12 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1278
1282
|
diffFilePath = path.join(os.tmpdir(), `skyramp-diff-${sessionId}.diff`);
|
|
1279
1283
|
await fs.promises.writeFile(diffFilePath, diffData.diffContent, { encoding: "utf-8", mode: 0o600 });
|
|
1280
1284
|
}
|
|
1285
|
+
// SKYR-3879 Path B: size-capped raw diff text, persisted (not just the
|
|
1286
|
+
// file path above) so skyramp_register_test_plan can verify a
|
|
1287
|
+
// discriminator's changedCodeAnchor occurs verbatim in the diff without
|
|
1288
|
+
// an extra file read of its own.
|
|
1289
|
+
const DIFF_TEXT_CAP = 200_000;
|
|
1290
|
+
const diffText = diffData?.diffContent ? diffData.diffContent.slice(0, DIFF_TEXT_CAP) : undefined;
|
|
1281
1291
|
// Compute UI context from the diff's changed files using the shared
|
|
1282
1292
|
// `isFrontendFile` classifier. Persisting this in the stateFile lets
|
|
1283
1293
|
// skyramp_analyze_test_health and the recommendation prompt consume the
|
|
@@ -1326,11 +1336,54 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1326
1336
|
(unintegratedFrontendFiles.length > 0
|
|
1327
1337
|
? `\n\n${buildUnintegratedInstructions(unintegratedFrontendFiles)}`
|
|
1328
1338
|
: "");
|
|
1339
|
+
// ── SKYR-3879 Path B: budget context + pre-seeded approved plan ──
|
|
1340
|
+
// Computed BEFORE the state write (not after, alongside the
|
|
1341
|
+
// recommendation prompt) so both this pre-seed and buildRecommendationPrompt
|
|
1342
|
+
// derive their GENERATE/ADDITIONAL split from the exact same numbers —
|
|
1343
|
+
// computeScoredCandidates is the single source of truth for both.
|
|
1344
|
+
const topN = params.topN ?? MAX_RECOMMENDATIONS;
|
|
1345
|
+
const scoredResult = computeScoredCandidates(fullAnalysis, analysisScope, topN, params.maxGenerate);
|
|
1346
|
+
const externalCoverage = buildExternalCoverageSet(testLocationsByType);
|
|
1347
|
+
const planBudgetContext = {
|
|
1348
|
+
maxGenerate: scoredResult.maxGen,
|
|
1349
|
+
maxTotal: topN,
|
|
1350
|
+
isUIOnlyPR: scoredResult.isUIOnlyPR,
|
|
1351
|
+
hasFrontendChanges: scoredResult.hasFrontendChanges,
|
|
1352
|
+
externalCoverageKeys: [...externalCoverage],
|
|
1353
|
+
};
|
|
1354
|
+
let approvedPlan;
|
|
1355
|
+
if (scoredResult.scored.length > 0) {
|
|
1356
|
+
const budgetCtx = {
|
|
1357
|
+
maxGenerate: planBudgetContext.maxGenerate,
|
|
1358
|
+
maxTotal: planBudgetContext.maxTotal,
|
|
1359
|
+
isUIOnlyPR: planBudgetContext.isUIOnlyPR,
|
|
1360
|
+
hasFrontendChanges: planBudgetContext.hasFrontendChanges,
|
|
1361
|
+
externalCoverage,
|
|
1362
|
+
};
|
|
1363
|
+
const serverCandidates = scoredResult.scored.map(({ scenario, priority, novelty }) => ({
|
|
1364
|
+
scenario,
|
|
1365
|
+
priority,
|
|
1366
|
+
novelty,
|
|
1367
|
+
source: CandidateSource.SERVER,
|
|
1368
|
+
candidateId: computeCandidateId(scenario),
|
|
1369
|
+
}));
|
|
1370
|
+
const plan = selectPlan(serverCandidates, budgetCtx);
|
|
1371
|
+
approvedPlan = {
|
|
1372
|
+
planId: crypto.randomUUID(),
|
|
1373
|
+
createdAt: new Date().toISOString(),
|
|
1374
|
+
generate: plan.generate.map(buildApprovedPlanItem),
|
|
1375
|
+
additional: plan.additional.map(buildApprovedPlanItem),
|
|
1376
|
+
demotions: [],
|
|
1377
|
+
};
|
|
1378
|
+
}
|
|
1329
1379
|
const unifiedState = {
|
|
1330
1380
|
existingTests,
|
|
1331
1381
|
newEndpoints: newEndpointsForDrafting,
|
|
1332
1382
|
analysisScope,
|
|
1333
1383
|
...(uiContext ? { uiContext } : {}),
|
|
1384
|
+
...(diffText ? { diffText } : {}),
|
|
1385
|
+
planBudgetContext,
|
|
1386
|
+
...(approvedPlan ? { approvedPlan } : {}),
|
|
1334
1387
|
repositoryAnalysis: {
|
|
1335
1388
|
skeletonEndpoints,
|
|
1336
1389
|
projectMeta,
|
|
@@ -1419,7 +1472,7 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1419
1472
|
}
|
|
1420
1473
|
await sendProgress(90, 100, "Generating ranked recommendations...");
|
|
1421
1474
|
// ── Step 12: Generate ranked recommendations inline ──
|
|
1422
|
-
|
|
1475
|
+
// topN computed earlier (SKYR-3879 Path B pre-seed, before the state write).
|
|
1423
1476
|
// ── Step 13: Fetch PR comment history for deduplication ──
|
|
1424
1477
|
let prContext;
|
|
1425
1478
|
if (params.prNumber) {
|
|
@@ -9,6 +9,18 @@ jest.unstable_mockModule("../../services/AnalyticsService.js", () => ({
|
|
|
9
9
|
}));
|
|
10
10
|
jest.unstable_mockModule("../../prompts/test-recommendation/test-recommendation-prompt.js", () => ({
|
|
11
11
|
buildRecommendationPrompt: jest.fn(),
|
|
12
|
+
// SKYR-3879 Path B pre-seed reads this before the state write — default to
|
|
13
|
+
// "nothing pre-ranked" so existing tests (which don't exercise the plan
|
|
14
|
+
// pre-seed) see byte-identical unifiedState/writeRepoData behavior.
|
|
15
|
+
computeScoredCandidates: jest.fn(() => ({
|
|
16
|
+
scored: [],
|
|
17
|
+
filteredChangedFiles: [],
|
|
18
|
+
isUIOnlyPR: false,
|
|
19
|
+
hasFrontendChanges: false,
|
|
20
|
+
hasApiChanges: false,
|
|
21
|
+
maxGen: 0,
|
|
22
|
+
seed: "",
|
|
23
|
+
})),
|
|
12
24
|
}));
|
|
13
25
|
jest.unstable_mockModule("../../prompts/test-recommendation/recommendationSections.js", () => ({
|
|
14
26
|
MAX_RECOMMENDATIONS: 10,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export { registerAnalyzeChangesTool } from "./analyzeChangesTool.js";
|
|
2
2
|
export { registerAnalyzeTestHealthTool } from "./analyzeTestHealthTool.js";
|
|
3
3
|
export { registerActionsTool } from "./actionsTool.js";
|
|
4
|
+
export { registerRegisterTestPlanTool } from "./registerTestPlanTool.js";
|
|
@@ -1,3 +1,4 @@
|
|
|
1
1
|
export { registerAnalyzeChangesTool } from "./analyzeChangesTool.js";
|
|
2
2
|
export { registerAnalyzeTestHealthTool } from "./analyzeTestHealthTool.js";
|
|
3
3
|
export { registerActionsTool } from "./actionsTool.js";
|
|
4
|
+
export { registerRegisterTestPlanTool } from "./registerTestPlanTool.js";
|