@skyramp/mcp 0.2.1-rc.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +10 -0
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +98 -87
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +92 -60
- package/build/prompts/test-maintenance/driftAnalysisSections.js +139 -197
- package/build/prompts/test-recommendation/scopeAssessment.js +106 -5
- package/build/prompts/test-recommendation/scopeAssessment.test.js +128 -1
- package/build/prompts/testbot/testbot-prompts.js +6 -9
- package/build/prompts/testbot/testbot-prompts.test.js +38 -22
- package/build/services/TestDiscoveryService.js +39 -9
- package/build/tools/test-management/actionsTool.js +166 -148
- package/build/tools/test-management/analyzeChangesTool.js +10 -12
- package/build/tools/test-management/analyzeTestHealthTool.js +10 -22
- package/build/tools/test-management/uiAnalyzeChangesTool.js +8 -2
- package/build/tools/test-management/uiAnalyzeChangesTool.test.js +47 -0
- package/build/utils/dartRouteExtractor.js +319 -0
- package/build/utils/dartRouteExtractor.test.js +307 -0
- package/build/utils/docker.test.js +1 -1
- package/build/utils/uiPageEnumerator.js +67 -0
- package/build/utils/uiPageEnumerator.test.js +222 -0
- package/build/utils/versions.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/assertApiRequestTool.js +46 -0
- package/node_modules/playwright/lib/mcp/skyramp/index.js +10 -0
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +313 -0
- package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +146 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +519 -52
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +32 -14
- package/package.json +2 -2
- package/node_modules/playwright/lib/mcp/browser/tools/domAnalyzer.js +0 -261
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.3.tgz +0 -0
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.4.tgz +0 -0
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.5.tgz +0 -0
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.6.tgz +0 -0
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
jest.mock("@skyramp/skyramp", () => ({
|
|
2
2
|
WorkspaceConfigManager: { create: jest.fn() },
|
|
3
3
|
}));
|
|
4
|
-
import
|
|
4
|
+
import * as fs from "fs";
|
|
5
|
+
import * as path from "path";
|
|
6
|
+
import * as os from "os";
|
|
7
|
+
import { hasFlutterSdkDep, isFrontendFile, isTestFile, buildScopeAssessmentSection } from "./scopeAssessment.js";
|
|
5
8
|
// ---------------------------------------------------------------------------
|
|
6
9
|
// isFrontendFile
|
|
7
10
|
// ---------------------------------------------------------------------------
|
|
@@ -56,6 +59,130 @@ describe("isFrontendFile", () => {
|
|
|
56
59
|
it("returns false for a plain TS API client in utils/ (no frontend dir)", () => {
|
|
57
60
|
expect(isFrontendFile("utils/apiClient.ts")).toBe(false);
|
|
58
61
|
});
|
|
62
|
+
// Flutter / Dart support — gated on hasFlutterSdkDep.
|
|
63
|
+
// Without the flag, .dart is unrecognised (legacy behaviour). With it,
|
|
64
|
+
// .dart is always-frontend (tier 2, before API_DIR_PATTERN).
|
|
65
|
+
describe(".dart files (Flutter)", () => {
|
|
66
|
+
it("returns false for .dart without hasFlutterSdkDep (default)", () => {
|
|
67
|
+
expect(isFrontendFile("lib/main.dart")).toBe(false);
|
|
68
|
+
});
|
|
69
|
+
it("returns false for .dart with hasFlutterSdkDep: false", () => {
|
|
70
|
+
expect(isFrontendFile("lib/main.dart", { hasFlutterSdkDep: false })).toBe(false);
|
|
71
|
+
});
|
|
72
|
+
it("returns true for .dart with hasFlutterSdkDep: true", () => {
|
|
73
|
+
expect(isFrontendFile("lib/main.dart", { hasFlutterSdkDep: true })).toBe(true);
|
|
74
|
+
});
|
|
75
|
+
it("returns true for nested .dart files in a Flutter project", () => {
|
|
76
|
+
expect(isFrontendFile("lib/widgets/game_board.dart", { hasFlutterSdkDep: true })).toBe(true);
|
|
77
|
+
});
|
|
78
|
+
it("returns true for .dart even in a directory that would otherwise look API-like (Flutter has no server-side Dart in scope)", () => {
|
|
79
|
+
// .dart sits in tier 2 (above API_DIR_PATTERN) so it's frontend even
|
|
80
|
+
// under api/ — there's no server-side Dart web framework that would
|
|
81
|
+
// make this ambiguous in the Flutter case.
|
|
82
|
+
expect(isFrontendFile("lib/api/client.dart", { hasFlutterSdkDep: true })).toBe(true);
|
|
83
|
+
});
|
|
84
|
+
it("returns true for game.dart at repo root in a Flutter project", () => {
|
|
85
|
+
expect(isFrontendFile("game.dart", { hasFlutterSdkDep: true })).toBe(true);
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
// ---------------------------------------------------------------------------
|
|
90
|
+
// hasFlutterSdkDep
|
|
91
|
+
// ---------------------------------------------------------------------------
|
|
92
|
+
describe("hasFlutterSdkDep", () => {
|
|
93
|
+
let tmpDir;
|
|
94
|
+
beforeEach(() => {
|
|
95
|
+
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "flutter-pubspec-"));
|
|
96
|
+
});
|
|
97
|
+
afterEach(() => {
|
|
98
|
+
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
99
|
+
});
|
|
100
|
+
it("returns false when pubspec.yaml is absent", () => {
|
|
101
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(false);
|
|
102
|
+
});
|
|
103
|
+
it("returns true when pubspec.yaml has the canonical Flutter SDK dep", () => {
|
|
104
|
+
fs.writeFileSync(path.join(tmpDir, "pubspec.yaml"), `name: birdle
|
|
105
|
+
description: A Flutter app.
|
|
106
|
+
dependencies:
|
|
107
|
+
flutter:
|
|
108
|
+
sdk: flutter
|
|
109
|
+
cupertino_icons: ^1.0.6
|
|
110
|
+
`);
|
|
111
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(true);
|
|
112
|
+
});
|
|
113
|
+
it("returns false for a pure Dart project (no flutter SDK dep)", () => {
|
|
114
|
+
// Pure Dart server (e.g. shelf, dart_frog) has pubspec.yaml but no flutter SDK
|
|
115
|
+
fs.writeFileSync(path.join(tmpDir, "pubspec.yaml"), `name: my_dart_server
|
|
116
|
+
description: A Dart server using shelf.
|
|
117
|
+
environment:
|
|
118
|
+
sdk: ">=3.0.0 <4.0.0"
|
|
119
|
+
dependencies:
|
|
120
|
+
shelf: ^1.4.0
|
|
121
|
+
shelf_router: ^1.1.4
|
|
122
|
+
`);
|
|
123
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(false);
|
|
124
|
+
});
|
|
125
|
+
it("returns false for a Dart CLI tool (no flutter SDK dep)", () => {
|
|
126
|
+
fs.writeFileSync(path.join(tmpDir, "pubspec.yaml"), `name: my_cli
|
|
127
|
+
description: A Dart CLI.
|
|
128
|
+
environment:
|
|
129
|
+
sdk: ^3.5.0
|
|
130
|
+
dependencies:
|
|
131
|
+
args: ^2.5.0
|
|
132
|
+
`);
|
|
133
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(false);
|
|
134
|
+
});
|
|
135
|
+
it("returns false on a malformed pubspec.yaml (graceful failure)", () => {
|
|
136
|
+
fs.writeFileSync(path.join(tmpDir, "pubspec.yaml"), "::: not yaml ::: \x00");
|
|
137
|
+
// Our regex-based check still works on string content; this is more of
|
|
138
|
+
// a "doesn't throw" test. Returns false because the malformed content
|
|
139
|
+
// doesn't match the sdk: flutter pattern.
|
|
140
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(false);
|
|
141
|
+
});
|
|
142
|
+
it("returns true even with extra whitespace and comments in pubspec.yaml", () => {
|
|
143
|
+
fs.writeFileSync(path.join(tmpDir, "pubspec.yaml"), `name: birdle
|
|
144
|
+
# This is the SDK dep
|
|
145
|
+
dependencies:
|
|
146
|
+
flutter:
|
|
147
|
+
sdk: flutter
|
|
148
|
+
`);
|
|
149
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(true);
|
|
150
|
+
});
|
|
151
|
+
// Subdir search — common shapes in real Flutter repos.
|
|
152
|
+
it("returns true when pubspec.yaml lives in app/", () => {
|
|
153
|
+
fs.mkdirSync(path.join(tmpDir, "app"));
|
|
154
|
+
fs.writeFileSync(path.join(tmpDir, "app", "pubspec.yaml"), "name: app\ndependencies:\n flutter:\n sdk: flutter\n");
|
|
155
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(true);
|
|
156
|
+
});
|
|
157
|
+
it("returns true when pubspec.yaml lives in mobile/", () => {
|
|
158
|
+
fs.mkdirSync(path.join(tmpDir, "mobile"));
|
|
159
|
+
fs.writeFileSync(path.join(tmpDir, "mobile", "pubspec.yaml"), "name: app\ndependencies:\n flutter:\n sdk: flutter\n");
|
|
160
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(true);
|
|
161
|
+
});
|
|
162
|
+
it("returns true for monorepo with apps/<name>/pubspec.yaml", () => {
|
|
163
|
+
fs.mkdirSync(path.join(tmpDir, "apps", "customer-app"), { recursive: true });
|
|
164
|
+
fs.writeFileSync(path.join(tmpDir, "apps", "customer-app", "pubspec.yaml"), "name: customer_app\ndependencies:\n flutter:\n sdk: flutter\n");
|
|
165
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(true);
|
|
166
|
+
});
|
|
167
|
+
it("returns true for monorepo with packages/<name>/pubspec.yaml", () => {
|
|
168
|
+
fs.mkdirSync(path.join(tmpDir, "packages", "ui"), { recursive: true });
|
|
169
|
+
fs.writeFileSync(path.join(tmpDir, "packages", "ui", "pubspec.yaml"), "name: ui\ndependencies:\n flutter:\n sdk: flutter\n");
|
|
170
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(true);
|
|
171
|
+
});
|
|
172
|
+
it("returns false when monorepo subdirs only contain pure-Dart packages", () => {
|
|
173
|
+
// Mixed monorepo: a Dart CLI in packages/cli/, no Flutter app anywhere.
|
|
174
|
+
fs.mkdirSync(path.join(tmpDir, "packages", "cli"), { recursive: true });
|
|
175
|
+
fs.writeFileSync(path.join(tmpDir, "packages", "cli", "pubspec.yaml"), "name: cli\ndependencies:\n args: ^2.5.0\n");
|
|
176
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(false);
|
|
177
|
+
});
|
|
178
|
+
it("does not walk arbitrary subdirs (e.g. node_modules-style nested deps)", () => {
|
|
179
|
+
// A pubspec deep under an unsupported subdir should NOT be picked up.
|
|
180
|
+
// Without this guard, a transitive dependency's pubspec could falsely
|
|
181
|
+
// flag a non-Flutter repo as Flutter.
|
|
182
|
+
fs.mkdirSync(path.join(tmpDir, "vendor", "nested", "lib"), { recursive: true });
|
|
183
|
+
fs.writeFileSync(path.join(tmpDir, "vendor", "nested", "lib", "pubspec.yaml"), "name: nested\ndependencies:\n flutter:\n sdk: flutter\n");
|
|
184
|
+
expect(hasFlutterSdkDep(tmpDir)).toBe(false);
|
|
185
|
+
});
|
|
59
186
|
});
|
|
60
187
|
// ---------------------------------------------------------------------------
|
|
61
188
|
// isTestFile
|
|
@@ -2,6 +2,7 @@ import { z } from "zod";
|
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
3
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
4
4
|
import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PARAM_UUID_GUIDANCE, AUTH_CONFLICT_ERROR_MSG, } from "../test-recommendation/recommendationSections.js";
|
|
5
|
+
import { buildDriftAnalysisPrompt } from "../test-maintenance/drift-analysis-prompt.js";
|
|
5
6
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
6
7
|
import { isContractConsumerModeEnabled } from "../../utils/featureFlags.js";
|
|
7
8
|
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
@@ -65,13 +66,9 @@ Use those recommendations as your baseline. Only add or remove tests that the us
|
|
|
65
66
|
**If \`skyramp_analyze_changes\` returns an error:** retry once only if the error is transient (timeout, network blip, temporary unavailability) — do NOT retry for permanent errors (invalid repository path, missing required parameter, authentication failure). If it fails again, call \`skyramp_submit_report\` with a minimal valid payload: leave all test arrays empty and add the error to \`issuesFound\`. Refer to the \`skyramp_submit_report\` schema for required fields. Do NOT attempt Task 2 without a valid stateFile.
|
|
66
67
|
**If all changed files are non-application** (CI/CD, docs, lock files, config) → skip to Task 3 (Submit Report) with empty arrays and a single \`issuesFound\` entry explaining why (same format as the zero-test path below).
|
|
67
68
|
|
|
68
|
-
3. **Maintain existing tests
|
|
69
|
+
3. **Maintain existing tests** using the rules in \`<drift_analysis_rules>\` below. For each existing test reported by \`skyramp_analyze_changes\`, score it and choose the action exactly as directed by the Action Decision Matrix in \`<drift_analysis_rules>\`. Only read test files that require action per that matrix — do NOT read files that will be IGNORED. **Do NOT read source files (routers, models, CRUD, components) — all the information you need is in the \`skyramp_analyze_changes\` output and the diff.** When reading multiple test files, **read them all in a single parallel batch** — do NOT read them one at a time. Apply actions directly. Results go in \`testMaintenance\`.
|
|
69
70
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
b. For each test scored UPDATE or REGENERATE, write \`updateInstructions\` (a concise description of what must change) **before** calling \`skyramp_actions\`. This articulation step prevents the LLM from letting file content override diff-based reasoning.
|
|
73
|
-
|
|
74
|
-
c. Call \`skyramp_actions\` with \`stateFile\` (from step 2) and your \`recommendations[]\` — one entry per test assessed, including IGNORE and VERIFY. The tool returns file content for each UPDATE/REGENERATE test — apply the edits. Results go in \`testMaintenance\`.
|
|
71
|
+
${buildDriftAnalysisPrompt({ existingTests: [], scannedEndpoints: [], repositoryPath })}
|
|
75
72
|
|
|
76
73
|
4. **Code review:** From the \`skyramp_analyze_changes\` output and the existing test files you read for maintenance, note any logic bugs. Do NOT read additional source files just for code review — use what is already available from the analysis and test file reads. Common patterns to flag:
|
|
77
74
|
- Computed fields not recalculated after mutation (e.g. \`total_amount\` unchanged after items are added/removed)
|
|
@@ -130,7 +127,7 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
130
127
|
Keep advancing until you have created exactly ${maxGenerate} new test files OR exhausted all candidates.
|
|
131
128
|
- Example: If enrichment reveals that sending \`discount_value\` without \`discount_type\` silently orphans the value (a concrete bug), complete all planned GENERATE items first, then generate this discovered scenario as an extra test and report it in \`newTestsCreated\`.
|
|
132
129
|
- Total generated: Follow the "Budget: N generate" line in the Execution Plan. Process every GENERATE-tagged item in order. Backfill from ADDITIONAL candidates (highest-ranked first) until \`newTestsCreated\` reaches ${maxGenerate} or all candidates are exhausted.
|
|
133
|
-
- **UI test priority**: If the
|
|
130
|
+
- **UI test priority**: If the PR scope assessment shows any UI/E2E budget OR \`uiContext.changedFrontendFiles\` is non-empty (the deterministic server signal — populated for all supported frontend file types including \`.tsx\`/\`.jsx\`/\`.vue\`/\`.svelte\`/\`.dart\`), you MUST attempt to generate at least one UI test. Use \`browser_navigate\` to the app's base URL — if the app responds, record a trace and generate the test.
|
|
134
131
|
**Skip only if one of these conditions is met:**
|
|
135
132
|
- **(a) App is unreachable** — \`browser_navigate\` fails or connection is refused.
|
|
136
133
|
- **(b) Unintegrated non-route component** — the changed file is a leaf component (not a framework route/entrypoint) that has no integration point in the running app. To confirm:
|
|
@@ -276,7 +273,7 @@ If a test **generation** tool call fails:
|
|
|
276
273
|
1. **Retry once** with the same parameters.
|
|
277
274
|
2. If it fails again, **skip** that candidate and move to the next ranked candidate.
|
|
278
275
|
3. If all candidates in the GENERATE set fail, fall back to generating the **simplest possible test**: a single contract test for the highest-scored endpoint (GET → 200 or POST → 201).
|
|
279
|
-
**Exception — frontend-only PRs**: If the diff modifies ONLY frontend files (\`.tsx\`, \`.jsx\`, \`.vue\`, \`.svelte\`, \`.css\`, \`.html\`) AND browser recording was not possible, do NOT generate a backend fallback contract test — it is irrelevant to the PR. Instead move ALL GENERATE candidates to \`additionalRecommendations\` and proceed to Task 3.
|
|
276
|
+
**Exception — frontend-only PRs**: If the diff modifies ONLY frontend files (\`.tsx\`, \`.jsx\`, \`.vue\`, \`.svelte\`, \`.dart\`, \`.css\`, \`.html\`) AND browser recording was not possible, do NOT generate a backend fallback contract test — it is irrelevant to the PR. Instead move ALL GENERATE candidates to \`additionalRecommendations\` and proceed to Task 3.
|
|
280
277
|
4. Log skipped candidates in \`issuesFound\` with the error message.
|
|
281
278
|
|
|
282
279
|
If a test **execution** (\`skyramp_execute_test\`) fails for a newly generated test:
|
|
@@ -334,7 +331,7 @@ Call \`skyramp_submit_report\` with \`summaryOutputFile\`: "${summaryOutputFile}
|
|
|
334
331
|
- **additionalRecommendations**: AT MOST ${maxRecommendations - maxGenerate} items.
|
|
335
332
|
- For \`testType: "contract"\` entries: **\`primaryEndpoint\` is required** (e.g. \`"GET /api/v1/users/{user_id}"\`). The tool will reject the submission without it — do not omit it or you will be forced to resubmit.
|
|
336
333
|
- For \`testType: "integration"\` or \`"e2e"\` entries: omit \`primaryEndpoint\` — use \`description\` to list the endpoints involved instead.
|
|
337
|
-
- **testMaintenance**: Use \`[]\` **only** if no existing Skyramp tests were found in the repository. If existing tests were found (any score), include one entry per test. Set \`action\` to the exact drift action
|
|
334
|
+
- **testMaintenance**: Use \`[]\` **only** if no existing Skyramp tests were found in the repository. If existing tests were found (any score), include one entry per test. Set \`action\` to the exact drift action you chose from the Action Decision Matrix (\`UPDATE\`, \`REGENERATE\`, \`DELETE\`, \`VERIFY\`, or \`IGNORE\`). For UPDATE/REGENERATE/DELETE tests that were modified and executed, populate all fields from real before/after execution results. For VERIFY/IGNORE tests (not modified), derive \`beforeStatus\` from the \`skyramp_analyze_test_health\` health score (typically \`"Pass"\` if drift score is 0 and no health issues were flagged), set \`afterStatus\` to \`"Skipped"\`, and use \`afterDetails\` to explain why (e.g. "IGNORE: drift score 0 — endpoint not modified in this PR"). Do **not** add entries for tests that were not returned by the health analysis.
|
|
338
335
|
|
|
339
336
|
---
|
|
340
337
|
|
|
@@ -202,40 +202,35 @@ describe("uiCredentials in getTestbotPrompt", () => {
|
|
|
202
202
|
.toThrow("</ui-credentials>");
|
|
203
203
|
});
|
|
204
204
|
});
|
|
205
|
-
describe("drift analysis
|
|
206
|
-
|
|
207
|
-
|
|
205
|
+
describe("drift analysis inline embedding", () => {
|
|
206
|
+
beforeAll(() => { process.env.SKYRAMP_FEATURE_TESTBOT = "1"; });
|
|
207
|
+
afterAll(() => { delete process.env.SKYRAMP_FEATURE_TESTBOT; });
|
|
208
208
|
function basePrompt() {
|
|
209
209
|
return getTestbotPrompt(baseArgs.prTitle, baseArgs.prDescription, baseArgs.summaryOutputFile, baseArgs.repositoryPath);
|
|
210
210
|
}
|
|
211
|
-
it("
|
|
211
|
+
it("wraps inline drift rules in XML tags", () => {
|
|
212
212
|
const prompt = basePrompt();
|
|
213
|
-
expect(prompt).toContain("
|
|
213
|
+
expect(prompt).toContain("<drift_analysis_rules>");
|
|
214
|
+
expect(prompt).toContain("</drift_analysis_rules>");
|
|
214
215
|
});
|
|
215
|
-
it("
|
|
216
|
+
it("does not include a persona statement inside the inline XML block", () => {
|
|
216
217
|
const prompt = basePrompt();
|
|
217
|
-
|
|
218
|
-
|
|
218
|
+
const start = prompt.indexOf("<drift_analysis_rules>");
|
|
219
|
+
const end = prompt.indexOf("</drift_analysis_rules>");
|
|
220
|
+
const block = prompt.slice(start, end);
|
|
221
|
+
expect(block).not.toContain("You are acting as a Skyramp Integration Architect");
|
|
219
222
|
});
|
|
220
|
-
it("
|
|
223
|
+
it("drift_analysis_rules block appears inside Task 1, before Task 2", () => {
|
|
221
224
|
const prompt = basePrompt();
|
|
222
225
|
const task1Pos = prompt.indexOf("## Task 1");
|
|
223
|
-
const
|
|
226
|
+
const rulesPos = prompt.indexOf("<drift_analysis_rules>");
|
|
224
227
|
const task2Pos = prompt.indexOf("## Task 2");
|
|
225
|
-
expect(
|
|
226
|
-
expect(
|
|
228
|
+
expect(rulesPos).toBeGreaterThan(task1Pos);
|
|
229
|
+
expect(rulesPos).toBeLessThan(task2Pos);
|
|
227
230
|
});
|
|
228
|
-
it("
|
|
229
|
-
// The rules are now fetched at runtime via skyramp_analyze_test_health —
|
|
230
|
-
// the <drift_analysis_rules> tag may appear as a reference in prose,
|
|
231
|
-
// but the actual rule content (Action Decision Tree) must not be baked in.
|
|
231
|
+
it("Task 1 step 3 prose references drift_analysis_rules tag", () => {
|
|
232
232
|
const prompt = basePrompt();
|
|
233
|
-
expect(prompt).
|
|
234
|
-
expect(prompt).not.toContain("Update Execution Rules\n\nWhen applying UPDATE actions");
|
|
235
|
-
});
|
|
236
|
-
it("does not contain a persona statement (no nested identity from old embed)", () => {
|
|
237
|
-
const prompt = basePrompt();
|
|
238
|
-
expect(prompt).not.toContain("You are acting as a Skyramp Integration Architect");
|
|
233
|
+
expect(prompt).toContain("rules in `<drift_analysis_rules>`");
|
|
239
234
|
});
|
|
240
235
|
});
|
|
241
236
|
describe("UI grounding via Task 2 capture-act-capture", () => {
|
|
@@ -365,4 +360,25 @@ describe("testbot prompt blueprint-grounded recommendations (slice 4)", () => {
|
|
|
365
360
|
// Make sure we removed the old capturedBlueprints threading directive.
|
|
366
361
|
expect(prompt).not.toMatch(/capturedBlueprints/);
|
|
367
362
|
});
|
|
363
|
+
// Flutter support — both the generalised UI trigger wording and the
|
|
364
|
+
// canvas/empty-ARIA issuesFound rule should appear in the prompt.
|
|
365
|
+
it("UI test priority defers to the deterministic server signal (uiContext.changedFrontendFiles), not just hard-coded extensions", () => {
|
|
366
|
+
const prompt = getTestbotPrompt(baseArgs.prTitle, baseArgs.prDescription, baseArgs.summaryOutputFile, baseArgs.repositoryPath);
|
|
367
|
+
// The trigger must reference the server signal (changedFrontendFiles)
|
|
368
|
+
// — that's what makes it framework-agnostic. Hard-coded extension lists
|
|
369
|
+
// become illustrative, not gating.
|
|
370
|
+
expect(prompt).toMatch(/uiContext.*changedFrontendFiles|changedFrontendFiles.*uiContext/);
|
|
371
|
+
// .dart should appear in the supported-types example list
|
|
372
|
+
expect(prompt).toMatch(/\.dart/);
|
|
373
|
+
});
|
|
374
|
+
it("frontend-only PR exception lists .dart alongside other frontend extensions", () => {
|
|
375
|
+
const prompt = getTestbotPrompt(baseArgs.prTitle, baseArgs.prDescription, baseArgs.summaryOutputFile, baseArgs.repositoryPath);
|
|
376
|
+
// The frontend-only-PR exception block (Task 2 backend-fallback skip)
|
|
377
|
+
// must include .dart so a Flutter-only PR doesn't emit a backend
|
|
378
|
+
// contract test as a fallback.
|
|
379
|
+
const exceptionBlock = prompt.slice(prompt.indexOf("Exception — frontend-only PRs"), prompt.indexOf("Exception — frontend-only PRs") + 800);
|
|
380
|
+
expect(exceptionBlock).toContain(".dart");
|
|
381
|
+
expect(exceptionBlock).toContain(".tsx");
|
|
382
|
+
expect(exceptionBlock).toContain(".vue");
|
|
383
|
+
});
|
|
368
384
|
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as fs from "fs";
|
|
2
2
|
import * as path from "path";
|
|
3
|
+
import { simpleGit } from "simple-git";
|
|
3
4
|
import { logger } from "../utils/logger.js";
|
|
4
5
|
import { TestSource } from "../types/TestAnalysis.js";
|
|
5
6
|
import fg from "fast-glob";
|
|
@@ -53,8 +54,11 @@ export class TestDiscoveryService {
|
|
|
53
54
|
/[\\/]__tests__[\\/]/,
|
|
54
55
|
/[\\/]spec[\\/]/,
|
|
55
56
|
];
|
|
57
|
+
// Cache git client and repo status per repository
|
|
58
|
+
gitClientCache = new Map();
|
|
59
|
+
isGitRepoCache = new Map();
|
|
56
60
|
/**
|
|
57
|
-
* Discover all tests
|
|
61
|
+
* Discover all tests in a repository — both Skyramp-generated and external (user-written).
|
|
58
62
|
* Uses fast-glob for cross-platform file scanning, then classifies discovered files
|
|
59
63
|
* as Skyramp-generated tests, external tests, or not-a-test during processing.
|
|
60
64
|
*
|
|
@@ -64,17 +68,19 @@ export class TestDiscoveryService {
|
|
|
64
68
|
* rather than flooding context with irrelevant files.
|
|
65
69
|
* - `undefined` (full-repo mode, no diff): cap at MAX_EXTERNAL_FULL_REPO.
|
|
66
70
|
*/
|
|
67
|
-
async discoverTests(
|
|
68
|
-
logger.info(`Starting test discovery in: ${
|
|
69
|
-
if (!fs.existsSync(
|
|
70
|
-
throw new Error(`
|
|
71
|
+
async discoverTests(repositoryPath, options = {}) {
|
|
72
|
+
logger.info(`Starting test discovery in: ${repositoryPath}`);
|
|
73
|
+
if (!fs.existsSync(repositoryPath)) {
|
|
74
|
+
throw new Error(`Repository path does not exist: ${repositoryPath}`);
|
|
71
75
|
}
|
|
72
|
-
const stats = fs.statSync(
|
|
76
|
+
const stats = fs.statSync(repositoryPath);
|
|
73
77
|
if (!stats.isDirectory()) {
|
|
74
|
-
throw new Error(`Path is not a directory: ${
|
|
78
|
+
throw new Error(`Path is not a directory: ${repositoryPath}`);
|
|
75
79
|
}
|
|
76
|
-
//
|
|
77
|
-
|
|
80
|
+
// Initialize git client cache for this repository
|
|
81
|
+
await this.initializeGitClient(repositoryPath);
|
|
82
|
+
// File classification: skyramp vs external vs not-a-test (carries content forward)
|
|
83
|
+
const classified = this.classifyTestFiles(repositoryPath);
|
|
78
84
|
logger.info(`Found ${classified.skyramp.length} Skyramp test files, ${classified.external.length} external test files`);
|
|
79
85
|
// Process Skyramp tests (content already cached from classification)
|
|
80
86
|
const skyrampTests = await this.processFilesInBatches(classified.skyramp, false, classified.contentCache);
|
|
@@ -133,6 +139,9 @@ export class TestDiscoveryService {
|
|
|
133
139
|
}));
|
|
134
140
|
const externalTests = [...relevantExternalTests, ...otherExternalTests];
|
|
135
141
|
logger.info(`Discovered ${skyrampTests.length} Skyramp tests, ${externalTests.length} external tests`);
|
|
142
|
+
// Clean up caches to free memory
|
|
143
|
+
this.gitClientCache.clear();
|
|
144
|
+
this.isGitRepoCache.clear();
|
|
136
145
|
return {
|
|
137
146
|
tests: [...skyrampTests, ...externalTests],
|
|
138
147
|
// Expose the relevant file paths so callers can build read instructions for the LLM.
|
|
@@ -177,6 +186,27 @@ export class TestDiscoveryService {
|
|
|
177
186
|
}
|
|
178
187
|
return { relevant, other };
|
|
179
188
|
}
|
|
189
|
+
/**
|
|
190
|
+
* Initialize git client and check if repository is a git repo
|
|
191
|
+
*/
|
|
192
|
+
async initializeGitClient(repositoryPath) {
|
|
193
|
+
try {
|
|
194
|
+
const git = simpleGit(repositoryPath);
|
|
195
|
+
this.gitClientCache.set(repositoryPath, git);
|
|
196
|
+
const isRepo = await git.checkIsRepo();
|
|
197
|
+
this.isGitRepoCache.set(repositoryPath, isRepo);
|
|
198
|
+
if (isRepo) {
|
|
199
|
+
logger.debug(`Git repository detected at: ${repositoryPath}`);
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
logger.debug(`Not a git repository: ${repositoryPath}`);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
logger.debug(`Could not initialize git client: ${error.message}`);
|
|
207
|
+
this.isGitRepoCache.set(repositoryPath, false);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
180
210
|
/**
|
|
181
211
|
* Process test files in parallel batches with concurrency control
|
|
182
212
|
* @param isExternal When true, uses external test metadata extraction
|