@skyramp/mcp 0.3.5 → 0.3.6-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +92 -30
- package/build/playwright/traceRecordingPrompt.d.ts +6 -0
- package/build/playwright/traceRecordingPrompt.js +6 -2
- package/build/prompts/code-reuse.d.ts +1 -2
- package/build/prompts/code-reuse.js +182 -77
- package/build/prompts/modularization/integration-test-modularization.d.ts +2 -0
- package/build/prompts/modularization/integration-test-modularization.js +83 -41
- package/build/prompts/modularization/render.d.ts +18 -0
- package/build/prompts/modularization/render.js +12 -0
- package/build/prompts/modularization/ui-test-modularization.d.ts +3 -1
- package/build/prompts/modularization/ui-test-modularization.js +89 -47
- package/build/prompts/pom-aware-code-reuse.js +3 -1
- package/build/prompts/shared-helper-policy.d.ts +57 -0
- package/build/prompts/shared-helper-policy.js +135 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.js +62 -56
- package/build/prompts/test-recommendation/fullRepoCatalog.js +19 -8
- package/build/prompts/test-recommendation/recommendationShared.d.ts +28 -6
- package/build/prompts/test-recommendation/recommendationShared.js +90 -16
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +22 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.d.ts +2 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +3 -3
- package/build/prompts/testbot/testbot-prompts.js +80 -34
- package/build/recommendation/budgeters/shared.js +105 -27
- package/build/recommendation/discriminators.js +13 -2
- package/build/recommendation/planRanker.d.ts +6 -6
- package/build/recommendation/planRanker.js +6 -61
- package/build/services/AnalyticsService.d.ts +7 -0
- package/build/services/AnalyticsService.js +7 -1
- package/build/services/ModularizationService.js +1 -3
- package/build/services/TestDiscoveryService.d.ts +0 -2
- package/build/services/TestDiscoveryService.js +2 -37
- package/build/services/TestGenerationService.d.ts +16 -0
- package/build/services/TestGenerationService.js +86 -10
- package/build/services/containerEnv.js +13 -12
- package/build/tools/code-refactor/codeReuseTool.js +279 -93
- package/build/tools/code-refactor/enhance-state.d.ts +49 -0
- package/build/tools/code-refactor/enhance-state.js +109 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.js +34 -1
- package/build/tools/code-refactor/modularizationTool.js +9 -2
- package/build/tools/code-refactor/reuse-outcome.d.ts +23 -1
- package/build/tools/code-refactor/reuse-outcome.js +14 -4
- package/build/tools/code-refactor/reuse-state.d.ts +127 -5
- package/build/tools/code-refactor/reuse-state.js +628 -16
- package/build/tools/code-refactor/utils-verify-gates.d.ts +26 -0
- package/build/tools/code-refactor/utils-verify-gates.js +100 -0
- package/build/tools/code-refactor/verify-gates.d.ts +2 -1
- package/build/tools/code-refactor/verify-gates.js +90 -25
- package/build/tools/executeSkyrampTestTool.d.ts +19 -0
- package/build/tools/executeSkyrampTestTool.js +158 -8
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +2 -2
- package/build/tools/generate-tests/generateE2ERestTool.js +16 -0
- package/build/tools/generate-tests/generateUIRestTool.d.ts +1 -0
- package/build/tools/generate-tests/generateUIRestTool.js +22 -0
- package/build/tools/generate-tests/scenarioLint.d.ts +2 -0
- package/build/tools/generate-tests/scenarioLint.js +127 -19
- package/build/tools/generate-tests/trace-reuse-guard.d.ts +20 -0
- package/build/tools/generate-tests/trace-reuse-guard.js +93 -0
- package/build/tools/submitReportTool.d.ts +38 -38
- package/build/tools/submitReportTool.js +411 -114
- package/build/tools/test-management/analyzeChangesTool.d.ts +24 -1
- package/build/tools/test-management/analyzeChangesTool.js +71 -10
- package/build/tools/test-management/analyzeTestHealthTool.js +7 -7
- package/build/tools/test-management/registerTestPlanTool.d.ts +203 -0
- package/build/tools/test-management/registerTestPlanTool.js +70 -12
- package/build/types/Recommendation.d.ts +34 -5
- package/build/types/RepositoryAnalysis.d.ts +133 -114
- package/build/types/RepositoryAnalysis.js +1 -1
- package/build/types/ReuseOutcome.d.ts +102 -6
- package/build/types/ReuseOutcome.js +16 -2
- package/build/types/TestRecommendation.js +21 -3
- package/build/types/TestTypes.js +14 -8
- package/build/types/TestbotReport.d.ts +10 -1
- package/build/types/index.d.ts +2 -2
- package/build/types/index.js +1 -1
- package/build/utils/AnalysisStateManager.d.ts +57 -1
- package/build/utils/AnalysisStateManager.js +54 -5
- package/build/utils/branchDiff.d.ts +10 -0
- package/build/utils/branchDiff.js +28 -0
- package/build/utils/changedRoutes.d.ts +29 -0
- package/build/utils/changedRoutes.js +87 -0
- package/build/utils/featureFlags.d.ts +21 -0
- package/build/utils/featureFlags.js +23 -0
- package/build/utils/frontendIntegration.js +34 -4
- package/build/utils/importerHop.d.ts +2 -8
- package/build/utils/importerHop.js +15 -53
- package/build/utils/pathMatching.d.ts +38 -0
- package/build/utils/pathMatching.js +71 -0
- package/build/utils/pathSignatures.d.ts +22 -0
- package/build/utils/pathSignatures.js +57 -0
- package/build/utils/planMatchKeys.d.ts +16 -3
- package/build/utils/planMatchKeys.js +26 -10
- package/build/utils/pluralization.d.ts +10 -0
- package/build/utils/pluralization.js +18 -0
- package/build/utils/pom-catalog-parse.d.ts +52 -0
- package/build/utils/pom-catalog-parse.js +141 -0
- package/build/utils/pom-scope/selector-extractor.d.ts +12 -0
- package/build/utils/pom-scope/selector-extractor.js +34 -8
- package/build/utils/pom-verify/verify.d.ts +6 -5
- package/build/utils/pom-verify/verify.js +8 -6
- package/build/utils/reportVerification.d.ts +64 -4
- package/build/utils/reportVerification.js +228 -3
- package/build/utils/reuseRouting.d.ts +3 -0
- package/build/utils/reuseRouting.js +50 -0
- package/build/utils/routeParsers.d.ts +2 -0
- package/build/utils/routeParsers.js +65 -8
- package/build/utils/scenarioDrafting.d.ts +1 -1
- package/build/utils/scenarioDrafting.js +57 -45
- package/build/utils/subjectEndpoints.d.ts +19 -0
- package/build/utils/subjectEndpoints.js +98 -0
- package/build/utils/testFileClassification.d.ts +11 -0
- package/build/utils/testFileClassification.js +47 -0
- package/build/utils/uiPageEnumerator.d.ts +45 -19
- package/build/utils/uiPageEnumerator.js +95 -51
- package/build/utils/utils-verify/allow.d.ts +16 -0
- package/build/utils/utils-verify/allow.js +68 -0
- package/build/utils/utils-verify/call-sites.d.ts +34 -0
- package/build/utils/utils-verify/call-sites.js +154 -0
- package/build/utils/utils-verify/index.d.ts +7 -0
- package/build/utils/utils-verify/index.js +7 -0
- package/build/utils/utils-verify/language-spec.d.ts +91 -0
- package/build/utils/utils-verify/language-spec.js +210 -0
- package/build/utils/utils-verify/locate.d.ts +39 -0
- package/build/utils/utils-verify/locate.js +199 -0
- package/build/utils/utils-verify/parse.d.ts +34 -0
- package/build/utils/utils-verify/parse.js +177 -0
- package/build/utils/utils-verify/stage.d.ts +24 -0
- package/build/utils/utils-verify/stage.js +107 -0
- package/build/utils/utils-verify/verify.d.ts +63 -0
- package/build/utils/utils-verify/verify.js +168 -0
- package/build/utils/utils.d.ts +3 -1
- package/build/utils/utils.js +3 -1
- package/build/workspace/workspace.d.ts +32 -32
- package/node_modules/playwright/lib/mcp/skyramp/assertTool.js +9 -5
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +16 -0
- package/node_modules/playwright/lib/mcp/skyramp/skyRampImport.js +2 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +115 -14
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +13 -1
- package/node_modules/playwright/node_modules/playwright-core/.DS_Store +0 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/server/codegen/skyramp/jsonlReader.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/htmlReport/index.html +27 -253
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/{codeMirrorModule-DtudTj_v.js → codeMirrorModule-DJMC4zNo.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BW82eAUI.js +196 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-FNMuBzX1.js → codeMirrorModule-CZfp96qZ.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-gpLo02E0.js +809 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.Bq1r1URj.js +2 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.VEfqi1qN.js +5 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/server/codegen/skyramp/jsonlReader.ts +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/recorder/assets/index-BpDwp16L.js +0 -422
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/defaultSettingsView-Co9upU5h.js +0 -1035
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.DXNIQ_dx.js +0 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.CIKB3XSv.js +0 -5
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import * as fs from "fs";
|
|
2
|
+
import * as path from "path";
|
|
3
|
+
import { SKYRAMP_UTILS_HEADER } from "../utils.js";
|
|
4
|
+
import { parseUtilsFile } from "./parse.js";
|
|
5
|
+
import { normalizePath } from "./language-spec.js";
|
|
6
|
+
import { realpath } from "./locate.js";
|
|
7
|
+
/** The codegen marker every Skyramp-GENERATED TEST carries on line 1 (`Generated by
|
|
8
|
+
* Skyramp v<version>`), as opposed to the utils header (`Generated by Skyramp on …`).
|
|
9
|
+
* The same discriminator STEP 4 of the reuse prompt hands the agent. */
|
|
10
|
+
export const CODEGEN_MARKER_RE = new RegExp(`${SKYRAMP_UTILS_HEADER.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\s+v\\d`);
|
|
11
|
+
const HEADER_SCAN_LINES = 5;
|
|
12
|
+
const REQUEST_CALL_RE = /\b(?:send_request|sendRequest)\s*\(/g;
|
|
13
|
+
/**
|
|
14
|
+
* Inline request calls in OTHER Skyramp-generated tests beside `testFile` that a
|
|
15
|
+
* utils helper already wraps — the reuse the prompt asks for and the agent does not
|
|
16
|
+
* perform (measured 0/2 seeded eval runs with the rule as its own mandatory step).
|
|
17
|
+
*
|
|
18
|
+
* A call is INLINE when it sits in a test function, not in a module-level helper of
|
|
19
|
+
* the sibling's own: a sibling that defines `create_order` locally is STEP 4b's
|
|
20
|
+
* helper-vs-helper case, not a call site. Equivalence is `c3b9157b`'s: same method
|
|
21
|
+
* and same normalised path, nothing else — literals are what the call's arguments
|
|
22
|
+
* are for.
|
|
23
|
+
*
|
|
24
|
+
* Advisory by design: it names a change to a PRE-EXISTING test, which the maintenance
|
|
25
|
+
* flow owns, so it informs rather than gates. Never throws.
|
|
26
|
+
*/
|
|
27
|
+
export async function findSiblingInlineCallSites(testFile, utilsFiles, _helpers, spec) {
|
|
28
|
+
// route → helper name + the keyword set of the helper's own request call, so a
|
|
29
|
+
// sibling call is reported only when its SHAPE matches too (same keywords once the
|
|
30
|
+
// liftable ones are set aside) — criterion 2 of the API near-duplicate definition.
|
|
31
|
+
const byRoute = new Map();
|
|
32
|
+
for (const utilsFile of utilsFiles) {
|
|
33
|
+
let content;
|
|
34
|
+
try {
|
|
35
|
+
content = await fs.promises.readFile(utilsFile, "utf8");
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
// Owners come from THIS file's own parse: `helpers` spans every utils file, and
|
|
41
|
+
// a line range alone would attribute file B's request to file A's helper.
|
|
42
|
+
const fileHelpers = parseUtilsFile(content, spec);
|
|
43
|
+
for (const m of content.matchAll(REQUEST_CALL_RE)) {
|
|
44
|
+
const start = m.index ?? 0;
|
|
45
|
+
const line = content.slice(0, start).split("\n").length;
|
|
46
|
+
const owner = fileHelpers.find((h) => h.line <= line && line <= h.endLine);
|
|
47
|
+
if (!owner?.method || !owner.normalizedPath)
|
|
48
|
+
continue;
|
|
49
|
+
const call = callText(content, start + m[0].length - 1);
|
|
50
|
+
byRoute.set(`${owner.method} ${owner.normalizedPath}`, {
|
|
51
|
+
helper: owner.name,
|
|
52
|
+
keys: shapeKeys(call),
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (byRoute.size === 0)
|
|
57
|
+
return [];
|
|
58
|
+
// Canonical, like the utils paths: otherwise a tool-supplied /tmp path and a
|
|
59
|
+
// realpath'd /private/tmp utils file look like different directories to the renderer.
|
|
60
|
+
const dir = path.dirname(await realpath(testFile));
|
|
61
|
+
// Canonical like `dir`, or a spec reached through a symlink is reported as its
|
|
62
|
+
// own sibling.
|
|
63
|
+
const exclude = new Set(await Promise.all([testFile, ...utilsFiles].map((f) => realpath(f))));
|
|
64
|
+
let entries;
|
|
65
|
+
try {
|
|
66
|
+
entries = await fs.promises.readdir(dir);
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return [];
|
|
70
|
+
}
|
|
71
|
+
const out = [];
|
|
72
|
+
for (const name of entries.sort()) {
|
|
73
|
+
const file = path.join(dir, name);
|
|
74
|
+
if (exclude.has(path.resolve(file)))
|
|
75
|
+
continue;
|
|
76
|
+
if (!spec.extensions.some((e) => name.toLowerCase().endsWith(e)))
|
|
77
|
+
continue;
|
|
78
|
+
let content;
|
|
79
|
+
try {
|
|
80
|
+
content = await fs.promises.readFile(file, "utf8");
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
if (!CODEGEN_MARKER_RE.test(content.split("\n").slice(0, HEADER_SCAN_LINES).join("\n")))
|
|
86
|
+
continue;
|
|
87
|
+
// Line ranges of the sibling's own non-test helpers: calls inside them are not inline.
|
|
88
|
+
const localHelpers = parseUtilsFile(content, spec).filter((h) => !/^test/i.test(h.name));
|
|
89
|
+
const inLocalHelper = (line) => localHelpers.some((h) => line >= h.line && line <= h.endLine);
|
|
90
|
+
for (const m of content.matchAll(REQUEST_CALL_RE)) {
|
|
91
|
+
const start = m.index ?? 0;
|
|
92
|
+
const line = content.slice(0, start).split("\n").length;
|
|
93
|
+
if (inLocalHelper(line))
|
|
94
|
+
continue;
|
|
95
|
+
const call = callText(content, start + m[0].length - 1);
|
|
96
|
+
const method = spec.methodRe.exec(call)?.[1]?.toUpperCase();
|
|
97
|
+
const p = spec.pathRe.exec(call)?.[1];
|
|
98
|
+
if (!method || !p)
|
|
99
|
+
continue;
|
|
100
|
+
const match = byRoute.get(`${method} ${normalizePath(p)}`);
|
|
101
|
+
if (!match)
|
|
102
|
+
continue;
|
|
103
|
+
if (!sameShape(shapeKeys(call), match.keys))
|
|
104
|
+
continue;
|
|
105
|
+
out.push({ file, line, method, path: p, helper: match.helper });
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return out;
|
|
109
|
+
}
|
|
110
|
+
/** Keywords whose presence differs legitimately between two versions of one request:
|
|
111
|
+
* the literals the merge rule lifts to parameters. Everything else is call shape. */
|
|
112
|
+
const LIFTABLE_KEYS = new Set([
|
|
113
|
+
"body",
|
|
114
|
+
"data",
|
|
115
|
+
"data_override",
|
|
116
|
+
"dataOverride",
|
|
117
|
+
"headers",
|
|
118
|
+
"url",
|
|
119
|
+
"path_params",
|
|
120
|
+
"pathParams",
|
|
121
|
+
]);
|
|
122
|
+
/** The keyword names of a request call (`path=`, `method=`, `query_params=` … / TS
|
|
123
|
+
* object keys), less the liftable ones — a cheap normalised call shape. */
|
|
124
|
+
export function shapeKeys(call) {
|
|
125
|
+
const keys = new Set();
|
|
126
|
+
for (const m of call.matchAll(/\b([A-Za-z_]\w*)\s*[=:](?![=])/g)) {
|
|
127
|
+
if (!LIFTABLE_KEYS.has(m[1]))
|
|
128
|
+
keys.add(m[1]);
|
|
129
|
+
}
|
|
130
|
+
return keys;
|
|
131
|
+
}
|
|
132
|
+
function sameShape(a, b) {
|
|
133
|
+
if (a.size !== b.size)
|
|
134
|
+
return false;
|
|
135
|
+
for (const k of a)
|
|
136
|
+
if (!b.has(k))
|
|
137
|
+
return false;
|
|
138
|
+
return true;
|
|
139
|
+
}
|
|
140
|
+
/** Text of the parenthesised argument list starting at the `(` at `open`. */
|
|
141
|
+
function callText(content, open) {
|
|
142
|
+
let depth = 0;
|
|
143
|
+
for (let i = open; i < content.length; i++) {
|
|
144
|
+
const ch = content[i];
|
|
145
|
+
if (ch === "(")
|
|
146
|
+
depth++;
|
|
147
|
+
else if (ch === ")") {
|
|
148
|
+
depth--;
|
|
149
|
+
if (depth === 0)
|
|
150
|
+
return content.slice(open, i + 1);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return content.slice(open);
|
|
154
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Language-parametric description of a shared SkyrampUtils file, and the invariants
|
|
3
|
+
* every helper in it must hold. Encoded as DATA rather than as per-language checks so
|
|
4
|
+
* that adding a language (or a helper family with a different assertion rule, e.g.
|
|
5
|
+
* browser helpers) is a table entry, not a second verifier.
|
|
6
|
+
*
|
|
7
|
+
* The invariants come from the prompts that author these files — the HELPER SHAPE
|
|
8
|
+
* block in the integration modularization prompt and the API near-duplicate rules in
|
|
9
|
+
* the reuse prompt — and exist here because prompt rules are LLM-enforced only:
|
|
10
|
+
* SKYR-4196 Part B runs 5 and 11 leaked a scenario-literal body assertion into a shared
|
|
11
|
+
* helper despite both rules being in the prompt.
|
|
12
|
+
*/
|
|
13
|
+
export type UtilsLanguage = "python" | "typescript" | "javascript";
|
|
14
|
+
export interface UtilsLanguageSpec {
|
|
15
|
+
language: UtilsLanguage;
|
|
16
|
+
/** File extensions (with dot) a utils file in this language can carry. */
|
|
17
|
+
extensions: string[];
|
|
18
|
+
commentPrefix: "#" | "//";
|
|
19
|
+
/** Matches the FIRST line of a module-level helper definition; group 1 is the name. */
|
|
20
|
+
helperStart: RegExp;
|
|
21
|
+
/** `method="POST"` / `method: "POST"` inside a helper body; group 1 is the verb. */
|
|
22
|
+
methodRe: RegExp;
|
|
23
|
+
/** `path="/api/…"` / `path: "/…"` inside a helper body; group 1 is the path. */
|
|
24
|
+
pathRe: RegExp;
|
|
25
|
+
/** A line that is an assertion at all. */
|
|
26
|
+
assertionRe: RegExp;
|
|
27
|
+
/** The one assertion shape an API helper may carry: a status-code comparison. Any
|
|
28
|
+
* assertion line not matching this is a body assertion — the scenario-specific
|
|
29
|
+
* claim that fails the next scenario calling the helper. */
|
|
30
|
+
allowedAssertion: RegExp;
|
|
31
|
+
/** The assertions a BROWSER helper may carry: structural waits that an element
|
|
32
|
+
* rendered / is interactable (`toBeVisible`, `toBeEnabled`, …). Value assertions
|
|
33
|
+
* (`toHaveText`, `toHaveValue`, `toHaveCount`, `toHaveURL`, any `expect` on a
|
|
34
|
+
* literal) are the browser analogue of a body assertion. Mirrors the
|
|
35
|
+
* BROWSER_HELPERS policy the modularization and reuse prompts state. */
|
|
36
|
+
allowedBrowserAssertion: RegExp;
|
|
37
|
+
/** Names `specContent` imports from the utils module whose basename stem is `stem`. */
|
|
38
|
+
importedNames(specContent: string, stem: string): string[];
|
|
39
|
+
}
|
|
40
|
+
/** Path parameters spelled `{id}`, `${id}`, `:id` all normalise to `{}` so two helpers
|
|
41
|
+
* for `/orders/{order_id}` and `/orders/{id}` compare equal. */
|
|
42
|
+
export declare function normalizePath(p: string): string;
|
|
43
|
+
export declare const UTILS_LANGUAGE_SPECS: Record<UtilsLanguage, UtilsLanguageSpec>;
|
|
44
|
+
/**
|
|
45
|
+
* The two kinds of shared helper, keyed by the test type's substrate — the same
|
|
46
|
+
* split the modularization and reuse prompts make (API_HELPERS / BROWSER_HELPERS):
|
|
47
|
+
* - `api` (integration, contract, load …): one SDK request; equivalence is
|
|
48
|
+
* method+path; the only assertion is the status code.
|
|
49
|
+
* - `browser` (ui, e2e): a Playwright action sequence; no route to key duplicates
|
|
50
|
+
* on, so the duplicate check is inert; the only assertions are structural waits.
|
|
51
|
+
*/
|
|
52
|
+
export type HelperFamily = "api" | "browser";
|
|
53
|
+
export interface HelperFamilySpec {
|
|
54
|
+
family: HelperFamily;
|
|
55
|
+
/** Whether two helpers wrapping the same method+path are a violation. */
|
|
56
|
+
routeDuplicates: boolean;
|
|
57
|
+
/** Picks the permitted-assertion pattern out of a language spec. */
|
|
58
|
+
allowedAssertion(spec: UtilsLanguageSpec): RegExp;
|
|
59
|
+
/** Whether one complete assertion expression is permitted for this family. */
|
|
60
|
+
isAllowed(spec: UtilsLanguageSpec, expr: string): boolean;
|
|
61
|
+
/** One clause for the failure text: what a helper of this family may assert. */
|
|
62
|
+
assertionRule: string;
|
|
63
|
+
}
|
|
64
|
+
export declare const HELPER_FAMILIES: Record<HelperFamily, HelperFamilySpec>;
|
|
65
|
+
/** The helper family a test type's shared helpers follow. Unlike `isBrowserTestType`,
|
|
66
|
+
* an ABSENT test type resolves to `api`: this runs against a utils file that already
|
|
67
|
+
* exists, and the browser rules are the more permissive of the two on duplicates, so
|
|
68
|
+
* guessing "browser" would silently switch a check off. */
|
|
69
|
+
export declare function helperFamilyFor(testType: string | undefined): HelperFamilySpec;
|
|
70
|
+
/** The spec for a language string as the tools receive it, or `undefined` for a
|
|
71
|
+
* language this verifier does not cover (Java: utils reuse is disabled for it). */
|
|
72
|
+
export declare function utilsSpecFor(language: string | undefined): UtilsLanguageSpec | undefined;
|
|
73
|
+
/** The spec whose extensions cover `file`, for callers that know a path but not a language. */
|
|
74
|
+
export declare function utilsSpecForFile(file: string): UtilsLanguageSpec | undefined;
|
|
75
|
+
/**
|
|
76
|
+
* Helper names are `<verb>_<resource>` only (`create_order`, `update_order`); scenario
|
|
77
|
+
* data belongs in arguments. A name carrying a scenario adjective or a literal is the
|
|
78
|
+
* signature of a helper that will not merge with the same step from the next test
|
|
79
|
+
* (`update_order_50_discount` beside `update_order_discount`, SKYR-4196 run 4).
|
|
80
|
+
*
|
|
81
|
+
* Tokens are matched after splitting on `_` and camelCase boundaries, so the rule
|
|
82
|
+
* applies unchanged to Python and TS/JS names.
|
|
83
|
+
*/
|
|
84
|
+
export declare const SCENARIO_NAME_RULE: {
|
|
85
|
+
forbiddenTokens: Set<string>;
|
|
86
|
+
/** A bare number as its OWN token — `update_order_50_discount` — never a digit inside
|
|
87
|
+
* an identifier (`s3`, `oauth2`, `ipv4`, `sha256`, `base64`, `iso8601`, `2fa`). */
|
|
88
|
+
digitToken: RegExp;
|
|
89
|
+
};
|
|
90
|
+
/** The reason a name is scenario-flavoured, or `undefined` when it is canonical. */
|
|
91
|
+
export declare function scenarioNameViolation(name: string): string | undefined;
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/** Path parameters spelled `{id}`, `${id}`, `:id` all normalise to `{}` so two helpers
|
|
2
|
+
* for `/orders/{order_id}` and `/orders/{id}` compare equal. */
|
|
3
|
+
export function normalizePath(p) {
|
|
4
|
+
return (p
|
|
5
|
+
// Absolute URLs and query strings compare as their path.
|
|
6
|
+
.replace(/^[a-z][a-z0-9+.-]*:\/\/[^/]+/i, "")
|
|
7
|
+
.replace(/[?#].*$/, "")
|
|
8
|
+
.replace(/\$\{[^}]*\}/g, "{}")
|
|
9
|
+
.replace(/\{[^}]*\}/g, "{}")
|
|
10
|
+
// `:id` is a parameter only as a whole segment; `/users:activate` is a
|
|
11
|
+
// custom method (Google AIP style) and a different endpoint.
|
|
12
|
+
.replace(/\/:[A-Za-z_]\w*(?=\/|$)/g, "/{}")
|
|
13
|
+
.replace(/\/+$/, "") || "/");
|
|
14
|
+
}
|
|
15
|
+
const PYTHON = {
|
|
16
|
+
language: "python",
|
|
17
|
+
extensions: [".py"],
|
|
18
|
+
commentPrefix: "#",
|
|
19
|
+
helperStart: /^(?:async\s+)?def\s+([A-Za-z_]\w*)\s*\(/,
|
|
20
|
+
methodRe: /\bmethod\s*=\s*["']([A-Za-z]+)["']/,
|
|
21
|
+
pathRe: /\bpath\s*=\s*[frbu]{0,2}["']([^"']+)["']/,
|
|
22
|
+
assertionRe: /^\s*(?:assert\b|(?:await\s+)?expect(?:\.soft)?\s*\()/,
|
|
23
|
+
// A status assertion is any assertion about the status code and nothing else — the
|
|
24
|
+
// comparison may be reversed, `in (...)`, or carry a trailing comment. Anything that
|
|
25
|
+
// reads the body (`json()`, `get_response_value`, `.text`, subscripts) is a body
|
|
26
|
+
// assertion whatever else the line says.
|
|
27
|
+
allowedAssertion: /^\s*assert\b(?=.*\bstatus_code\b)(?!.*(?:\.json\(|get_response_value|\.text\b|\.content\b|\[["']))/,
|
|
28
|
+
// Playwright for Python spells the structural matchers in snake_case.
|
|
29
|
+
allowedBrowserAssertion: /\bexpect\s*\((?:[^()]|\((?:[^()]|\([^()]*\))*\))*\)\s*\.\s*(?:not\.)?to_be_(?:visible|hidden|attached|enabled|disabled|editable|focused|in_viewport)\s*\(/,
|
|
30
|
+
importedNames(spec, stem) {
|
|
31
|
+
const names = new Set();
|
|
32
|
+
const stemRe = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
33
|
+
// `from SkyrampUtils import a, b` — or parenthesised across lines. The two
|
|
34
|
+
// forms are separate alternatives: an optional `(` with a lazy body let `$`
|
|
35
|
+
// match at the end of the `import (` line and captured nothing.
|
|
36
|
+
const fromRe = new RegExp(`^\\s*from\\s+(?:[\\w.]*\\.)?${stemRe}\\s+import\\s+(?:\\(([^)]*)\\)|([^\\n]+))`, "gm");
|
|
37
|
+
for (const m of spec.matchAll(fromRe)) {
|
|
38
|
+
for (const raw of (m[1] ?? m[2]).split(",")) {
|
|
39
|
+
const name = raw
|
|
40
|
+
.trim()
|
|
41
|
+
.split(/\s+as\s+/)[0]
|
|
42
|
+
.trim();
|
|
43
|
+
if (/^[A-Za-z_]\w*$/.test(name))
|
|
44
|
+
names.add(name);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// `import SkyrampUtils` / `import SkyrampUtils as utils` — every
|
|
48
|
+
// `<module-or-alias>.<name>` use counts.
|
|
49
|
+
const mod = new RegExp(`^\\s*import\\s+(?:[\\w.]*\\.)?${stemRe}(?:\\s+as\\s+([A-Za-z_]\\w*))?\\s*$`, "m").exec(spec);
|
|
50
|
+
if (mod) {
|
|
51
|
+
const qualifier = (mod[1] ?? stem).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
52
|
+
for (const m of spec.matchAll(new RegExp(`\\b${qualifier}\\.([A-Za-z_]\\w*)`, "g")))
|
|
53
|
+
names.add(m[1]);
|
|
54
|
+
}
|
|
55
|
+
return [...names];
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
function tsImportedNames(spec, stem) {
|
|
59
|
+
const names = new Set();
|
|
60
|
+
const stemRe = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
61
|
+
const from = `["'][^"']*\\/${stemRe}(?:\\.[cm]?[jt]s)?["']`;
|
|
62
|
+
for (const m of spec.matchAll(new RegExp(`import\\s*\\{([^}]*)\\}\\s*from\\s*${from}`, "g"))) {
|
|
63
|
+
for (const raw of m[1].split(",")) {
|
|
64
|
+
// The EXPORTED name (before `as`): the count checks what the utils file defines,
|
|
65
|
+
// so an alias would under-count. Same choice as the Python `from … import x as y`.
|
|
66
|
+
const name = raw
|
|
67
|
+
.trim()
|
|
68
|
+
.split(/\s+as\s+/)[0]
|
|
69
|
+
.trim();
|
|
70
|
+
if (/^[A-Za-z_$][\w$]*$/.test(name))
|
|
71
|
+
names.add(name);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
for (const m of spec.matchAll(new RegExp(`import\\s*\\*\\s*as\\s+([A-Za-z_$][\\w$]*)\\s*from\\s*${from}`, "g"))) {
|
|
75
|
+
for (const use of spec.matchAll(new RegExp(`\\b${m[1]}\\.([A-Za-z_$][\\w$]*)`, "g")))
|
|
76
|
+
names.add(use[1]);
|
|
77
|
+
}
|
|
78
|
+
for (const m of spec.matchAll(new RegExp(`(?:const|let|var)\\s*\\{([^}]*)\\}\\s*=\\s*require\\(\\s*${from}\\s*\\)`, "g"))) {
|
|
79
|
+
for (const raw of m[1].split(",")) {
|
|
80
|
+
// `{ a: b } = require(…)` — `a` is the exported name; `b` is the local alias.
|
|
81
|
+
const name = raw.trim().split(":")[0].trim();
|
|
82
|
+
if (/^[A-Za-z_$][\w$]*$/.test(name))
|
|
83
|
+
names.add(name);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
return [...names];
|
|
87
|
+
}
|
|
88
|
+
const TYPESCRIPT = {
|
|
89
|
+
language: "typescript",
|
|
90
|
+
extensions: [".ts", ".mts", ".cts"],
|
|
91
|
+
commentPrefix: "//",
|
|
92
|
+
// `function f(`, `const f = (…) =>`, `const f = async function (`, `const f = <T,>(…) =>`
|
|
93
|
+
// — and a signature Prettier wrapped, whose `(` still opens on the definition line.
|
|
94
|
+
helperStart: /^(?:export\s+)?(?:(?:async\s+)?function\s+([A-Za-z_$][\w$]*)\s*[<(]|(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*(?::[^=]*)?=\s*(?:async\s*)?(?:function\b|<[^>]*>\s*\(|\(|[A-Za-z_$][\w$]*\s*=>))/,
|
|
95
|
+
methodRe: /\bmethod\s*:\s*["'`]([A-Za-z]+)["'`]/,
|
|
96
|
+
pathRe: /\bpath\s*:\s*["'`]([^"'`]+)["'`]/,
|
|
97
|
+
assertionRe: /\bexpect(?:\.soft)?\s*\(/,
|
|
98
|
+
allowedAssertion:
|
|
99
|
+
// The receiver is the response itself, never a body/json/data field of it:
|
|
100
|
+
// `expect(res.body.status).toBe("paid")` is a body assertion. An optional
|
|
101
|
+
// string message argument (`expect(r.statusCode, 'status code')`) is the
|
|
102
|
+
// form codegen emits and still a status-only assertion.
|
|
103
|
+
/\bexpect(?:\.soft)?\s*\(\s*(?![\w.]*\.(?:body|json|data|payload)\b)[\w.]+\.status(?:_?[cC]ode)?(?:\(\))?\s*(?:,\s*(?:"[^"]*"|'[^']*'|`[^`]*`)\s*)?\)\s*\.\s*(?:not\.)?(?:toBe|toEqual|toStrictEqual|toBeGreaterThan|toBeLessThan|toBeOneOf)\s*\(/,
|
|
104
|
+
allowedBrowserAssertion: /\bexpect(?:\.soft)?\s*\((?:[^()]|\((?:[^()]|\([^()]*\))*\))*\)\s*\.\s*(?:not\.)?toBe(?:Visible|Hidden|Attached|Enabled|Disabled|Editable|Focused|InViewport)\s*\(/,
|
|
105
|
+
importedNames: tsImportedNames,
|
|
106
|
+
};
|
|
107
|
+
const JAVASCRIPT = {
|
|
108
|
+
...TYPESCRIPT,
|
|
109
|
+
language: "javascript",
|
|
110
|
+
extensions: [".js", ".mjs", ".cjs"],
|
|
111
|
+
};
|
|
112
|
+
export const UTILS_LANGUAGE_SPECS = {
|
|
113
|
+
python: PYTHON,
|
|
114
|
+
typescript: TYPESCRIPT,
|
|
115
|
+
javascript: JAVASCRIPT,
|
|
116
|
+
};
|
|
117
|
+
const SCENARIO_LITERAL_RE = /["'`][^"'`]*[\d$%€£][^"'`]*["'`]/;
|
|
118
|
+
/** The text inside `expect(...)`, up to the matcher. */
|
|
119
|
+
function expectArgs(expr) {
|
|
120
|
+
const open = expr.indexOf("(", expr.indexOf("expect"));
|
|
121
|
+
const close = expr.search(/\)\s*\.\s*(?:not\.)?to/);
|
|
122
|
+
return open === -1 || close === -1 ? expr : expr.slice(open + 1, close);
|
|
123
|
+
}
|
|
124
|
+
export const HELPER_FAMILIES = {
|
|
125
|
+
api: {
|
|
126
|
+
family: "api",
|
|
127
|
+
routeDuplicates: true,
|
|
128
|
+
allowedAssertion: (spec) => spec.allowedAssertion,
|
|
129
|
+
isAllowed: (spec, expr) => spec.allowedAssertion.test(expr),
|
|
130
|
+
assertionRule: "a shared API helper asserts only the status code; move this assertion to the test body, applied to the response the helper returns",
|
|
131
|
+
},
|
|
132
|
+
browser: {
|
|
133
|
+
family: "browser",
|
|
134
|
+
routeDuplicates: false,
|
|
135
|
+
allowedAssertion: (spec) => spec.allowedBrowserAssertion,
|
|
136
|
+
// A structural wait on a locator that carries a scenario LITERAL
|
|
137
|
+
// (`getByText("Total: $899.98")`) is a value assertion in disguise: digits or
|
|
138
|
+
// currency in any string inside the expect(...) arguments disqualify it.
|
|
139
|
+
isAllowed: (spec, expr) => spec.allowedBrowserAssertion.test(expr) &&
|
|
140
|
+
!SCENARIO_LITERAL_RE.test(expectArgs(expr)),
|
|
141
|
+
assertionRule: "a shared browser helper holds actions and structural waits only (toBeVisible, toBeEnabled, …); move this value assertion to the test body immediately after the helper call, selector verbatim",
|
|
142
|
+
},
|
|
143
|
+
};
|
|
144
|
+
/** The helper family a test type's shared helpers follow. Unlike `isBrowserTestType`,
|
|
145
|
+
* an ABSENT test type resolves to `api`: this runs against a utils file that already
|
|
146
|
+
* exists, and the browser rules are the more permissive of the two on duplicates, so
|
|
147
|
+
* guessing "browser" would silently switch a check off. */
|
|
148
|
+
export function helperFamilyFor(testType) {
|
|
149
|
+
const t = (testType ?? "").toLowerCase();
|
|
150
|
+
return HELPER_FAMILIES[t === "ui" || t === "e2e" ? "browser" : "api"];
|
|
151
|
+
}
|
|
152
|
+
/** The spec for a language string as the tools receive it, or `undefined` for a
|
|
153
|
+
* language this verifier does not cover (Java: utils reuse is disabled for it). */
|
|
154
|
+
export function utilsSpecFor(language) {
|
|
155
|
+
const key = (language ?? "").toLowerCase();
|
|
156
|
+
return UTILS_LANGUAGE_SPECS[key];
|
|
157
|
+
}
|
|
158
|
+
/** The spec whose extensions cover `file`, for callers that know a path but not a language. */
|
|
159
|
+
export function utilsSpecForFile(file) {
|
|
160
|
+
const lower = file.toLowerCase();
|
|
161
|
+
return Object.values(UTILS_LANGUAGE_SPECS).find((s) => s.extensions.some((e) => lower.endsWith(e)));
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Helper names are `<verb>_<resource>` only (`create_order`, `update_order`); scenario
|
|
165
|
+
* data belongs in arguments. A name carrying a scenario adjective or a literal is the
|
|
166
|
+
* signature of a helper that will not merge with the same step from the next test
|
|
167
|
+
* (`update_order_50_discount` beside `update_order_discount`, SKYR-4196 run 4).
|
|
168
|
+
*
|
|
169
|
+
* Tokens are matched after splitting on `_` and camelCase boundaries, so the rule
|
|
170
|
+
* applies unchanged to Python and TS/JS names.
|
|
171
|
+
*/
|
|
172
|
+
export const SCENARIO_NAME_RULE = {
|
|
173
|
+
// Scenario ADJECTIVES only — words that never name a resource or a REST filter.
|
|
174
|
+
// Resource nouns that describe conditions (`error`, `test`, `new`, `first`) and
|
|
175
|
+
// filter states (`expired`, `failed`, `valid`, `missing`, `positive`) are legitimate:
|
|
176
|
+
// `create_test_case`, `list_expired_subscriptions`, `get_failed_jobs`. A false hit
|
|
177
|
+
// here blocks execution, so the list stays short.
|
|
178
|
+
forbiddenTokens: new Set([
|
|
179
|
+
"invalid",
|
|
180
|
+
"wrong",
|
|
181
|
+
"bad",
|
|
182
|
+
"nonexistent",
|
|
183
|
+
"nonexisting",
|
|
184
|
+
"unauthorized",
|
|
185
|
+
"unauthenticated",
|
|
186
|
+
"forbidden",
|
|
187
|
+
"notfound",
|
|
188
|
+
"scenario",
|
|
189
|
+
"twice",
|
|
190
|
+
"again",
|
|
191
|
+
]),
|
|
192
|
+
/** A bare number as its OWN token — `update_order_50_discount` — never a digit inside
|
|
193
|
+
* an identifier (`s3`, `oauth2`, `ipv4`, `sha256`, `base64`, `iso8601`, `2fa`). */
|
|
194
|
+
digitToken: /^\d+$/,
|
|
195
|
+
};
|
|
196
|
+
/** The reason a name is scenario-flavoured, or `undefined` when it is canonical. */
|
|
197
|
+
export function scenarioNameViolation(name) {
|
|
198
|
+
const tokens = name
|
|
199
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1_$2")
|
|
200
|
+
.toLowerCase()
|
|
201
|
+
.split(/[_$]+/)
|
|
202
|
+
.filter(Boolean);
|
|
203
|
+
const hit = tokens.find((t) => SCENARIO_NAME_RULE.digitToken.test(t) ||
|
|
204
|
+
SCENARIO_NAME_RULE.forbiddenTokens.has(t));
|
|
205
|
+
if (!hit)
|
|
206
|
+
return undefined;
|
|
207
|
+
return SCENARIO_NAME_RULE.digitToken.test(hit)
|
|
208
|
+
? `contains the literal "${hit}" — scenario values belong in arguments, not the name`
|
|
209
|
+
: `contains the scenario word "${hit}" — name helpers by method + resource only`;
|
|
210
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/** The header `generateSkyrampHeader` writes into a utils file: `Generated by Skyramp on
|
|
2
|
+
* <timestamp>`. Deliberately NOT the bare `SKYRAMP_UTILS_HEADER`, which every generated
|
|
3
|
+
* TEST also carries as its codegen marker (`Generated by Skyramp v<version>`). */
|
|
4
|
+
export declare const UTILS_FILE_HEADER_RE: RegExp;
|
|
5
|
+
export interface LocateUtilsParams {
|
|
6
|
+
/** Absolute path of the delivered test file. */
|
|
7
|
+
testFile: string;
|
|
8
|
+
/** Tool-supplied language; inferred from the test file's extension when absent. */
|
|
9
|
+
language?: string;
|
|
10
|
+
/** Override for the git root (tests); detected from `testFile` otherwise. */
|
|
11
|
+
cwd?: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Every shared utils file relevant to `testFile`, by CONTENT, never by name.
|
|
15
|
+
*
|
|
16
|
+
* The reuse prompt names a conventional file (`SkyrampUtils.py`) but explicitly allows
|
|
17
|
+
* "a new file with a different name" once that one is large, so any scheme deriving the
|
|
18
|
+
* path from the test path or a fixed filename fails silently on exactly that branch. A
|
|
19
|
+
* utils file is instead recognised by the header the prompt makes the agent write.
|
|
20
|
+
*
|
|
21
|
+
* Two sources, unioned, both scoped to the spec:
|
|
22
|
+
* 1. files this run touched (untracked, modified, or staged) IN THE SPEC'S OWN
|
|
23
|
+
* DIRECTORY, in the language's extensions — the file the agent just created or
|
|
24
|
+
* extended beside the test, which is what staging needs. Not repo-wide: a monorepo
|
|
25
|
+
* has one utils file per service directory, and one service's broken helper must
|
|
26
|
+
* not fail — or stage — another service's verify (review finding on #792).
|
|
27
|
+
* 2. modules the delivered spec imports, resolved beside it and required to stay
|
|
28
|
+
* inside the repository — a pre-existing utils file the spec merely imports from
|
|
29
|
+
* (SKYR-4196 run 8: byte-identical, so absent from source 1).
|
|
30
|
+
*
|
|
31
|
+
* Never throws: outside a git repo source 1 is empty and source 2 still runs; any
|
|
32
|
+
* other failure logs and contributes nothing.
|
|
33
|
+
*/
|
|
34
|
+
export declare function locateUtilsFiles(params: LocateUtilsParams): Promise<string[]>;
|
|
35
|
+
/** Canonical path, or the input unchanged when it does not exist (yet). */
|
|
36
|
+
export declare function realpath(p: string): Promise<string>;
|
|
37
|
+
/** True when the first lines of `file` carry the utils header. Reads only the head of
|
|
38
|
+
* the file — a utils file is small, but nothing here should depend on that. */
|
|
39
|
+
export declare function hasUtilsHeader(file: string): Promise<boolean>;
|