@skyramp/mcp 0.3.6-rc.2.ac20 → 0.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/adapters/jestAdapter.js +0 -3
- package/build/adapters/mochaAdapter.js +0 -2
- package/build/adapters/playwrightAdapter.js +0 -3
- package/build/adapters/pytestAdapter.js +0 -12
- package/build/prompts/code-reuse.js +17 -2
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +1 -1
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +2 -1
- package/build/prompts/modularization/ui-test-modularization.js +9 -6
- package/build/prompts/pom-aware-code-reuse.js +1 -1
- package/build/prompts/shared-helper-policy.js +5 -5
- package/build/prompts/testbot/testbot-prompts.js +1 -1
- package/build/services/TestGenerationService.js +28 -1
- package/build/tools/code-refactor/assertion-state.d.ts +91 -0
- package/build/tools/code-refactor/assertion-state.js +375 -0
- package/build/tools/code-refactor/codeReuseTool.js +6 -4
- package/build/tools/code-refactor/enhanceAssertionsTool.js +73 -18
- package/build/tools/code-refactor/retrofit-state.d.ts +53 -0
- package/build/tools/code-refactor/retrofit-state.js +162 -0
- package/build/tools/code-refactor/reuse-outcome.d.ts +7 -0
- package/build/tools/code-refactor/reuse-state.d.ts +9 -0
- package/build/tools/code-refactor/reuse-state.js +42 -4
- package/build/tools/code-refactor/utils-verify-gates.js +69 -15
- package/build/tools/executeSkyrampTestTool.js +19 -14
- package/build/tools/generate-tests/batchMockGenerationTool.js +25 -0
- package/build/tools/generateEnrichedIntegrationTestTool.js +10 -0
- package/build/tools/runExistingTestsTool.d.ts +2 -34
- package/build/tools/runExistingTestsTool.js +4 -104
- package/build/tools/submitReportTool.js +87 -134
- package/build/tools/workspace/initializeWorkspaceTool.js +99 -27
- package/build/types/AssertionOutcome.d.ts +68 -0
- package/build/types/AssertionOutcome.js +1 -0
- package/build/types/ExternalTestExecution.d.ts +1 -67
- package/build/types/ReuseOutcome.d.ts +16 -0
- package/build/types/TestTypes.d.ts +4 -0
- package/build/types/TestTypes.js +8 -0
- package/build/types/TestbotReport.d.ts +13 -0
- package/build/types/index.d.ts +1 -1
- package/build/utils/AnalysisStateManager.d.ts +20 -7
- package/build/utils/assertion-verify/api-shared-lints.d.ts +5 -0
- package/build/utils/assertion-verify/api-shared-lints.js +315 -0
- package/build/utils/assertion-verify/contract-lints.d.ts +3 -0
- package/build/utils/assertion-verify/contract-lints.js +87 -0
- package/build/utils/assertion-verify/format.d.ts +5 -0
- package/build/utils/assertion-verify/format.js +65 -0
- package/build/utils/assertion-verify/helper-imports.d.ts +6 -0
- package/build/utils/assertion-verify/helper-imports.js +178 -0
- package/build/utils/assertion-verify/index.d.ts +3 -0
- package/build/utils/assertion-verify/index.js +7 -0
- package/build/utils/assertion-verify/integration-lints.d.ts +3 -0
- package/build/utils/assertion-verify/integration-lints.js +36 -0
- package/build/utils/assertion-verify/js-regex-blank.d.ts +1 -0
- package/build/utils/assertion-verify/js-regex-blank.js +153 -0
- package/build/utils/assertion-verify/lint-types.d.ts +33 -0
- package/build/utils/assertion-verify/lint-types.js +57 -0
- package/build/utils/assertion-verify/marker.d.ts +27 -0
- package/build/utils/assertion-verify/marker.js +61 -0
- package/build/utils/assertion-verify/metrics.d.ts +30 -0
- package/build/utils/assertion-verify/metrics.js +341 -0
- package/build/utils/assertion-verify/python-strip.d.ts +6 -0
- package/build/utils/assertion-verify/python-strip.js +75 -0
- package/build/utils/assertion-verify/strip-dispatch.d.ts +19 -0
- package/build/utils/assertion-verify/strip-dispatch.js +42 -0
- package/build/utils/assertion-verify/ui-lints.d.ts +8 -0
- package/build/utils/assertion-verify/ui-lints.js +244 -0
- package/build/utils/assertion-verify/verify.d.ts +61 -0
- package/build/utils/assertion-verify/verify.js +215 -0
- package/build/utils/executorWorkDir.d.ts +36 -0
- package/build/utils/executorWorkDir.js +77 -0
- package/build/utils/featureFlags.d.ts +12 -2
- package/build/utils/featureFlags.js +33 -3
- package/build/utils/reportVerification.d.ts +4 -0
- package/build/utils/reportVerification.js +32 -4
- package/build/utils/utils-verify/allow.d.ts +22 -4
- package/build/utils/utils-verify/allow.js +8 -2
- package/build/utils/utils-verify/call-sites.d.ts +40 -1
- package/build/utils/utils-verify/call-sites.js +196 -30
- package/build/utils/utils-verify/importers.d.ts +31 -0
- package/build/utils/utils-verify/importers.js +78 -0
- package/build/utils/utils-verify/index.d.ts +1 -0
- package/build/utils/utils-verify/index.js +1 -0
- package/build/utils/utils-verify/language-spec.d.ts +13 -2
- package/build/utils/utils-verify/language-spec.js +12 -2
- package/build/utils/utils-verify/parse.d.ts +31 -3
- package/build/utils/utils-verify/parse.js +190 -9
- package/build/utils/utils-verify/retrofit-equivalence.d.ts +43 -0
- package/build/utils/utils-verify/retrofit-equivalence.js +218 -0
- package/build/utils/utils-verify/stage.d.ts +6 -0
- package/build/utils/utils-verify/stage.js +12 -2
- package/build/utils/utils-verify/verify.d.ts +54 -4
- package/build/utils/utils-verify/verify.js +224 -12
- package/node_modules/playwright/node_modules/playwright-core/lib/generated/injectedScriptSource.js +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{codeMirrorModule-CZfp96qZ.js → codeMirrorModule-LNgEKtdV.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/assets/{defaultSettingsView-gpLo02E0.js → defaultSettingsView-Bwr1eMKC.js} +135 -135
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.Bq1r1URj.js → index.-Id052Lr.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{uiMode.VEfqi1qN.js → uiMode.BPopbasy.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/uiMode.html +2 -2
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/node_modules/playwright-core/src/generated/injectedScriptSource.ts +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +2 -2
- package/build/tools/code-refactor/enhance-state.d.ts +0 -49
- package/build/tools/code-refactor/enhance-state.js +0 -109
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import { strippedSources } from "./strip-dispatch.js";
|
|
2
|
+
import { balancedCloseIndex, identifierRe, lineOfOffset, } from "./lint-types.js";
|
|
3
|
+
import { lintApiShared } from "./api-shared-lints.js";
|
|
4
|
+
/**
|
|
5
|
+
* Provider-contract lints: the shared API checks plus the machine-checkable
|
|
6
|
+
* slice of the D5 `value_sources` dim — contract assertions must use inline
|
|
7
|
+
* request values, never data provisioned in beforeAll/beforeEach.
|
|
8
|
+
*/
|
|
9
|
+
const SETUP_BLOCK_RE = /\b(?:beforeAll|beforeEach)\s*\(/g;
|
|
10
|
+
const DECL_RE = /(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=/g;
|
|
11
|
+
/** Identifiers assigned inside beforeAll/beforeEach blocks and the block
|
|
12
|
+
* ranges themselves (JS/TS only — python contract tests use pytest fixtures,
|
|
13
|
+
* out of scope here). The ranges matter: codegen's own provisioning status
|
|
14
|
+
* checks live INSIDE the setup blocks and reference these names legitimately
|
|
15
|
+
* — only TEST-BODY assertions using setup data violate the rubric (measured:
|
|
16
|
+
* 7 of 8 pristine provider-contract goldens false-warned without this). */
|
|
17
|
+
function setupDeclarations(stripped) {
|
|
18
|
+
const names = new Set();
|
|
19
|
+
const ranges = [];
|
|
20
|
+
SETUP_BLOCK_RE.lastIndex = 0;
|
|
21
|
+
let m;
|
|
22
|
+
while ((m = SETUP_BLOCK_RE.exec(stripped)) !== null) {
|
|
23
|
+
const close = balancedCloseIndex(stripped, m.index + m[0].length - 1);
|
|
24
|
+
if (close === -1)
|
|
25
|
+
continue;
|
|
26
|
+
ranges.push([m.index, close]);
|
|
27
|
+
const block = stripped.slice(m.index, close + 1);
|
|
28
|
+
DECL_RE.lastIndex = 0;
|
|
29
|
+
let decl;
|
|
30
|
+
while ((decl = DECL_RE.exec(block)) !== null)
|
|
31
|
+
names.add(decl[1]);
|
|
32
|
+
}
|
|
33
|
+
return { names, ranges };
|
|
34
|
+
}
|
|
35
|
+
/** An assertion references a value provisioned in beforeAll/beforeEach.
|
|
36
|
+
* Warn-only: block-scoped `const` inside beforeAll is not reachable from a
|
|
37
|
+
* test body in real code, so hits are usually closure vars — a strong smell,
|
|
38
|
+
* but shared infra handles (clients, tokens) are legitimate closure vars too. */
|
|
39
|
+
function lintSetupDataInAssertions(stripped) {
|
|
40
|
+
const { names, ranges } = setupDeclarations(stripped);
|
|
41
|
+
if (names.size === 0)
|
|
42
|
+
return [];
|
|
43
|
+
// Compile once per name, not per (site × name).
|
|
44
|
+
const nameRes = new Map([...names].map((name) => [name, identifierRe(name)]));
|
|
45
|
+
const findings = [];
|
|
46
|
+
const expectRe = /\bexpect(?:\s*\.\s*soft)?\s*\(/g;
|
|
47
|
+
let m;
|
|
48
|
+
while ((m = expectRe.exec(stripped)) !== null) {
|
|
49
|
+
// Setup-internal assertions (provisioning status checks) are codegen's
|
|
50
|
+
// own sanctioned shape — only test-body references are the violation.
|
|
51
|
+
const at = m.index;
|
|
52
|
+
if (ranges.some(([start, end]) => at >= start && at <= end))
|
|
53
|
+
continue;
|
|
54
|
+
const close = balancedCloseIndex(stripped, m.index + m[0].length - 1);
|
|
55
|
+
if (close === -1)
|
|
56
|
+
continue;
|
|
57
|
+
// The whole statement line, so matcher arguments count too.
|
|
58
|
+
const lineEnd = stripped.indexOf("\n", close);
|
|
59
|
+
const stmt = stripped.slice(m.index, lineEnd === -1 ? undefined : lineEnd);
|
|
60
|
+
for (const [name, re] of nameRes) {
|
|
61
|
+
if (re.test(stmt)) {
|
|
62
|
+
findings.push({
|
|
63
|
+
rule: "setup-data-in-assertion",
|
|
64
|
+
severity: "warn",
|
|
65
|
+
line: lineOfOffset(stripped, m.index),
|
|
66
|
+
message: `Assertion references \`${name}\`, which is provisioned in beforeAll/beforeEach.`,
|
|
67
|
+
remediation: "Contract assertions must use values from the inline request body, path, query, or expected_response_body — not setup-provisioned data.",
|
|
68
|
+
});
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
return findings;
|
|
74
|
+
}
|
|
75
|
+
export function lintContractTest(raw, language, opts) {
|
|
76
|
+
// Java is count+hash gated only (a documented limit of the verifier): the
|
|
77
|
+
// lint regexes and strength weights are written for the python/ts/js SDK
|
|
78
|
+
// helper idioms, and applying them to JUnit source would produce false
|
|
79
|
+
// findings, not protection. Same guard as helper-imports.ts.
|
|
80
|
+
if (language === "java")
|
|
81
|
+
return [];
|
|
82
|
+
const findings = lintApiShared(raw, language, opts);
|
|
83
|
+
if (opts?.scopeLines === undefined && language !== "python") {
|
|
84
|
+
findings.push(...lintSetupDataInAssertions(strippedSources(raw, language).stripped));
|
|
85
|
+
}
|
|
86
|
+
return findings;
|
|
87
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { AssertionVerifyResult } from "./verify.js";
|
|
2
|
+
/** Renders the verify result as the text returned to the agent — PASSED/FAILED
|
|
3
|
+
* with one actionable remediation per violation, in the style of the reuse
|
|
4
|
+
* verify report (verify-gates.ts). */
|
|
5
|
+
export declare function formatAssertionVerifyReport(r: AssertionVerifyResult, testFile: string): string;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import * as path from "path";
|
|
2
|
+
import { detectAssertionLanguage } from "./metrics.js";
|
|
3
|
+
import { markerCommentToken } from "./marker.js";
|
|
4
|
+
/** Renders the verify result as the text returned to the agent — PASSED/FAILED
|
|
5
|
+
* with one actionable remediation per violation, in the style of the reuse
|
|
6
|
+
* verify report (verify-gates.ts). */
|
|
7
|
+
export function formatAssertionVerifyReport(r, testFile) {
|
|
8
|
+
const spec = path.basename(testFile);
|
|
9
|
+
// Paste-ready marker lines must use THIS file's comment token — a `//`
|
|
10
|
+
// marker in a .py file is a syntax error.
|
|
11
|
+
const token = markerCommentToken(detectAssertionLanguage(testFile) ?? "typescript");
|
|
12
|
+
const finding = (f) => `- ${f.line !== undefined ? `line ${f.line}: ` : ""}${f.message} Fix: ${f.remediation}`;
|
|
13
|
+
if (r.ok) {
|
|
14
|
+
// Maintenance drops the strength-family gates (value-only fixes are
|
|
15
|
+
// legitimate), so its PASS is a weaker claim — say so, rather than letting
|
|
16
|
+
// the agent and the eval judge read it as a full verification.
|
|
17
|
+
const heading = r.enhanceType === "maintenance" && r.baselinePresent
|
|
18
|
+
? r.maintenanceNoAssertionChanges
|
|
19
|
+
? `ASSERTION MAINTENANCE CHECK PASSED — ${spec}: no assertion changes since hand-out; nothing to verify.`
|
|
20
|
+
: `ASSERTION MAINTENANCE CHECK PASSED — ${spec}: ${r.currentMetrics?.count ?? "?"} assertion sites (differential strength gates are not applicable in maintenance; added/changed assertions were linted and removals guarded).`
|
|
21
|
+
: `ASSERTION VERIFICATION PASSED — ${spec}: ${r.currentMetrics?.count ?? "?"} assertion sites` +
|
|
22
|
+
(r.strengthDelta !== undefined && r.strengthDelta > 0
|
|
23
|
+
? `, strength +${r.strengthDelta} over the generated baseline.`
|
|
24
|
+
: ".");
|
|
25
|
+
const lines = [heading];
|
|
26
|
+
if (r.replacedCount !== undefined && r.replacedCount > 0) {
|
|
27
|
+
lines.push(`${r.replacedCount} generated assertion${r.replacedCount === 1 ? "" : "s"} replaced on the same subject — accepted.`);
|
|
28
|
+
}
|
|
29
|
+
if (r.movedToHelperCount !== undefined && r.movedToHelperCount > 0) {
|
|
30
|
+
lines.push(`${r.movedToHelperCount} assertion${r.movedToHelperCount === 1 ? "" : "s"} now covered by an imported helper file — accepted.`);
|
|
31
|
+
}
|
|
32
|
+
if (r.markerReason !== undefined) {
|
|
33
|
+
lines.push(`Documented decline accepted: ${r.markerReason}`);
|
|
34
|
+
}
|
|
35
|
+
if (!r.baselinePresent) {
|
|
36
|
+
lines.push("No enhancement baseline was recorded for this file, so only the stateless checks (rubric lints, markers) ran.");
|
|
37
|
+
}
|
|
38
|
+
return [...lines, ...warningsBlock(r.warnings, finding)].join("\n");
|
|
39
|
+
}
|
|
40
|
+
const lines = [`ASSERTION VERIFICATION FAILED — ${spec}:`];
|
|
41
|
+
if (r.hashUnchanged) {
|
|
42
|
+
lines.push("- The file is byte-identical to when the enhancement instructions were handed out — nothing was applied. Read the file, apply the instructions, and re-verify.");
|
|
43
|
+
}
|
|
44
|
+
if (r.removedFingerprints !== undefined) {
|
|
45
|
+
lines.push(`- Assertions were REMOVED — their subjects are asserted neither in this file nor in any helper file it imports. Replacing an assertion with a stronger one on the same subject is fine — losing coverage is not, and a documented decline does NOT clear this. Restore the assertions in THIS spec file${r.baselineFilePath !== undefined ? ` (the generated baseline is preserved at ${r.baselineFilePath})` : ""}. Do NOT edit shared helper/utils files — they are imported by other tests.`, r.removedCount !== undefined &&
|
|
46
|
+
r.removedCount > r.removedFingerprints.length
|
|
47
|
+
? ` Missing (showing ${r.removedFingerprints.length} of ${r.removedCount} — restore ALL of them in this pass):`
|
|
48
|
+
: " Missing:", ...r.removedFingerprints.map((fp) => ` ${fp}`));
|
|
49
|
+
}
|
|
50
|
+
lines.push(...r.hardFindings.map(finding));
|
|
51
|
+
if (r.strengthGateFailed) {
|
|
52
|
+
lines.push("- Assertion strength did not increase over the generated baseline: no assertion was added and no weak matcher was strengthened (e.g. `toBeVisible` → `toHaveText`). Apply the enhancement rules. A documented decline (`assertions complete` comment) is accepted only AFTER at least one assertion was added or replaced — a marker-only edit does not pass this gate.");
|
|
53
|
+
}
|
|
54
|
+
if (r.weakAdditionsOnly) {
|
|
55
|
+
lines.push("- Every assertion added since the baseline is existence/visibility-tier (`toBeVisible`, `not.toBeNull`, `is not None`, ranges). When the exact text/value/count is knowable from the trace, source, or request, assert it exactly (`toHaveText`/`toHaveCount`/`toBe`/`==`). If only existence is genuinely knowable, document the decline:", ` ${token} assertions complete: ${spec} — <reason exact values are not knowable>`);
|
|
56
|
+
}
|
|
57
|
+
lines.push(...warningsBlock(r.warnings, finding));
|
|
58
|
+
lines.push("Re-run this verification (verify: true) after every fix pass — skyramp_execute_test runs the same checks automatically and will not execute this test until they pass.");
|
|
59
|
+
return lines.join("\n");
|
|
60
|
+
}
|
|
61
|
+
function warningsBlock(warnings, finding) {
|
|
62
|
+
if (warnings.length === 0)
|
|
63
|
+
return [];
|
|
64
|
+
return ["WARNINGS (informational — do not block):", ...warnings.map(finding)];
|
|
65
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assertion SUBJECTS found in the spec's locally imported, actually-called
|
|
3
|
+
* helper files. Strict-by-default failure direction: an unreadable/oversized/
|
|
4
|
+
* uncalled helper is skipped, so its subjects don't excuse anything.
|
|
5
|
+
*/
|
|
6
|
+
export declare function importedHelperSubjects(testFile: string, content: string): Promise<Set<string>>;
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import * as path from "path";
|
|
2
|
+
import { readFile, stat } from "fs/promises";
|
|
3
|
+
import { logger } from "../logger.js";
|
|
4
|
+
import { strippedSources } from "./strip-dispatch.js";
|
|
5
|
+
import { identifierRe } from "./lint-types.js";
|
|
6
|
+
import { computeAssertionMetrics, detectAssertionLanguage, subjectOf, } from "./metrics.js";
|
|
7
|
+
/**
|
|
8
|
+
* Read-only scan of the spec's locally imported AND CALLED modules (the
|
|
9
|
+
* modularized skyramp-utils/helper files), used by the removal gate: a
|
|
10
|
+
* baseline subject missing from the spec is not "removed" if an imported
|
|
11
|
+
* helper still asserts it.
|
|
12
|
+
*
|
|
13
|
+
* Two deliberate restrictions:
|
|
14
|
+
* - One-directional — helper assertions can only EXCUSE removals, never add
|
|
15
|
+
* to the spec's count or strength score: a shared utils file is imported by
|
|
16
|
+
* many specs, and crediting its assertions to each one would inflate every
|
|
17
|
+
* baseline comparison.
|
|
18
|
+
* - CALLED, not merely imported — an import statement plus a dead helper file
|
|
19
|
+
* would otherwise turn a correctly-blocked gutting into a PASS (same
|
|
20
|
+
* call-position requirement as the reuse gate's call-site matching). Only
|
|
21
|
+
* helpers whose imported bindings appear at a call position in the spec
|
|
22
|
+
* contribute subjects.
|
|
23
|
+
*
|
|
24
|
+
* Helpers are never modified.
|
|
25
|
+
*/
|
|
26
|
+
const MAX_HELPER_FILES = 5;
|
|
27
|
+
const MAX_HELPER_BYTES = 256 * 1024;
|
|
28
|
+
// Import forms that bind local names to a RELATIVE module. Dynamic import()
|
|
29
|
+
// is deliberately absent: it binds no statically-trackable name, and dropping
|
|
30
|
+
// it errs strict (the removal stays a removal).
|
|
31
|
+
const JS_IMPORT_RES = [
|
|
32
|
+
// import defaultName from './x' | import * as ns from './x' |
|
|
33
|
+
// import { a, b as c } from './x' | import d, { a } from './x'
|
|
34
|
+
/import\s+([^'"()]+?)\s*from\s*['"](\.{1,2}\/[^'"]+)['"]/g,
|
|
35
|
+
// const d = require('./x') | const { a, b: c } = require('./x')
|
|
36
|
+
/(?:const|let|var)\s+([^=]+?)\s*=\s*require\s*\(\s*['"](\.{1,2}\/[^'"]+)['"]\s*\)/g,
|
|
37
|
+
];
|
|
38
|
+
const PY_IMPORT_RES = [
|
|
39
|
+
// from .x import a, b as c | from x import a
|
|
40
|
+
/^[ \t]*from\s+(\.*[\w.]+)\s+import\s+([^\n#]+)/gm,
|
|
41
|
+
// import x (module itself becomes the callable namespace)
|
|
42
|
+
/^[ \t]*import\s+([\w.]+)(?:\s+as\s+([\w]+))?/gm,
|
|
43
|
+
];
|
|
44
|
+
const JS_RESOLVE_SUFFIXES = [
|
|
45
|
+
"",
|
|
46
|
+
".ts",
|
|
47
|
+
".tsx",
|
|
48
|
+
".js",
|
|
49
|
+
".jsx",
|
|
50
|
+
".mjs",
|
|
51
|
+
".cjs",
|
|
52
|
+
".mts",
|
|
53
|
+
".cts",
|
|
54
|
+
"/index.ts",
|
|
55
|
+
"/index.js",
|
|
56
|
+
];
|
|
57
|
+
function pyCandidates(specDir, specifier) {
|
|
58
|
+
const moduleName = specifier.replace(/^\.+/, "").split(".")[0];
|
|
59
|
+
return moduleName ? [path.resolve(specDir, `${moduleName}.py`)] : [];
|
|
60
|
+
}
|
|
61
|
+
/** Local names bound by a JS import/require clause ("d, { a, b as c }"). */
|
|
62
|
+
function jsBoundNames(clause) {
|
|
63
|
+
const names = [];
|
|
64
|
+
const star = /\*\s*as\s+([\w$]+)/.exec(clause);
|
|
65
|
+
if (star)
|
|
66
|
+
names.push(star[1]);
|
|
67
|
+
const braces = /\{([^}]*)\}/.exec(clause);
|
|
68
|
+
if (braces) {
|
|
69
|
+
for (const part of braces[1].split(",")) {
|
|
70
|
+
const name = (part.split(/\s+as\s+|\s*:\s*/).pop() ?? "").trim();
|
|
71
|
+
if (/^[\w$]+$/.test(name))
|
|
72
|
+
names.push(name);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
const bare = clause
|
|
76
|
+
.replace(/\{[^}]*\}/, "")
|
|
77
|
+
.replace(/\*\s*as\s+[\w$]+/, "")
|
|
78
|
+
.replace(/,/g, " ")
|
|
79
|
+
.trim();
|
|
80
|
+
if (/^[\w$]+$/.test(bare))
|
|
81
|
+
names.push(bare);
|
|
82
|
+
return names;
|
|
83
|
+
}
|
|
84
|
+
/** Relative/local imports with the names they bind. */
|
|
85
|
+
function localImports(content, language) {
|
|
86
|
+
const commentless = strippedSources(content, language).commentless;
|
|
87
|
+
const imports = [];
|
|
88
|
+
if (language === "python") {
|
|
89
|
+
const [fromRe, importRe] = PY_IMPORT_RES;
|
|
90
|
+
fromRe.lastIndex = 0;
|
|
91
|
+
let m;
|
|
92
|
+
while ((m = fromRe.exec(commentless)) !== null) {
|
|
93
|
+
const names = m[2]
|
|
94
|
+
.split(",")
|
|
95
|
+
.map((part) => (part.split(/\s+as\s+/).pop() ?? "").trim())
|
|
96
|
+
.filter((n) => /^\w+$/.test(n));
|
|
97
|
+
imports.push({ specifier: m[1], localNames: names });
|
|
98
|
+
}
|
|
99
|
+
importRe.lastIndex = 0;
|
|
100
|
+
while ((m = importRe.exec(commentless)) !== null) {
|
|
101
|
+
imports.push({ specifier: m[1], localNames: [m[2] ?? m[1]] });
|
|
102
|
+
}
|
|
103
|
+
return imports;
|
|
104
|
+
}
|
|
105
|
+
for (const re of JS_IMPORT_RES) {
|
|
106
|
+
re.lastIndex = 0;
|
|
107
|
+
let m;
|
|
108
|
+
while ((m = re.exec(commentless)) !== null) {
|
|
109
|
+
imports.push({ specifier: m[2], localNames: jsBoundNames(m[1]) });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return imports;
|
|
113
|
+
}
|
|
114
|
+
/** True when any of the bound names appears at a CALL position in the spec —
|
|
115
|
+
* `name(...)` or `name.member(...)` — on comment/string-stripped source. */
|
|
116
|
+
function anyNameCalled(strippedSpec, names) {
|
|
117
|
+
// identifierRe gives the identifier boundary; the extra `(?<!\.)` keeps a
|
|
118
|
+
// same-named METHOD on another object (`other.name(...)`) from counting;
|
|
119
|
+
// the suffix requires a CALL position — `name(...)` or `name.member(...)`.
|
|
120
|
+
return names.some((name) => new RegExp(`(?<!\\.)${identifierRe(name).source}\\s*(?:\\(|\\.\\s*[\\w$]+\\s*\\()`).test(strippedSpec));
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Assertion SUBJECTS found in the spec's locally imported, actually-called
|
|
124
|
+
* helper files. Strict-by-default failure direction: an unreadable/oversized/
|
|
125
|
+
* uncalled helper is skipped, so its subjects don't excuse anything.
|
|
126
|
+
*/
|
|
127
|
+
export async function importedHelperSubjects(testFile, content) {
|
|
128
|
+
const subjects = new Set();
|
|
129
|
+
const language = detectAssertionLanguage(testFile);
|
|
130
|
+
// Not a no-op: an empty subject set means NO removal gets excused by a
|
|
131
|
+
// helper — the strict direction. Unknown languages have no import scanner;
|
|
132
|
+
// java is count+hash gated only (no fingerprint subjects exist to excuse —
|
|
133
|
+
// same documented limit as contract-lints.ts).
|
|
134
|
+
if (language === undefined || language === "java")
|
|
135
|
+
return subjects;
|
|
136
|
+
const strippedSpec = strippedSources(content, language).stripped;
|
|
137
|
+
const specDir = path.dirname(testFile);
|
|
138
|
+
let scanned = 0;
|
|
139
|
+
for (const { specifier, localNames } of localImports(content, language)) {
|
|
140
|
+
if (scanned >= MAX_HELPER_FILES) {
|
|
141
|
+
logger.warning("Assertion verification scanned the maximum number of imported helper files — remaining imports ignored", { testFile, max: MAX_HELPER_FILES });
|
|
142
|
+
break;
|
|
143
|
+
}
|
|
144
|
+
// An import whose bindings are never invoked cannot vouch for coverage —
|
|
145
|
+
// a dead helper file must not excuse a gutted spec.
|
|
146
|
+
if (localNames.length === 0 || !anyNameCalled(strippedSpec, localNames)) {
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const candidates = language === "python"
|
|
150
|
+
? pyCandidates(specDir, specifier)
|
|
151
|
+
: JS_RESOLVE_SUFFIXES.map((s) => path.resolve(specDir, specifier) + s);
|
|
152
|
+
for (const candidate of candidates) {
|
|
153
|
+
try {
|
|
154
|
+
const info = await stat(candidate);
|
|
155
|
+
// A directory hit (extensionless specifier) must not shadow the real
|
|
156
|
+
// module file — keep trying the remaining suffix candidates.
|
|
157
|
+
if (!info.isFile() || info.size > MAX_HELPER_BYTES)
|
|
158
|
+
continue;
|
|
159
|
+
const helperLanguage = detectAssertionLanguage(candidate);
|
|
160
|
+
if (helperLanguage === undefined)
|
|
161
|
+
continue;
|
|
162
|
+
const helperContent = await readFile(candidate, "utf8");
|
|
163
|
+
for (const fp of computeAssertionMetrics(helperContent, helperLanguage)
|
|
164
|
+
.fingerprints) {
|
|
165
|
+
const subject = subjectOf(fp);
|
|
166
|
+
if (subject !== undefined)
|
|
167
|
+
subjects.add(subject);
|
|
168
|
+
}
|
|
169
|
+
scanned++;
|
|
170
|
+
break;
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
// Candidate doesn't exist / unreadable — try the next suffix.
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return subjects;
|
|
178
|
+
}
|
|
@@ -0,0 +1,3 @@
|
|
|
1
|
+
export { detectAssertionLanguage, computeAssertionMetrics } from "./metrics.js";
|
|
2
|
+
export { verifyAssertionEnhancement, sha256Of, type AssertionBaseline, type AssertionEnhanceType, type AssertionVerifyResult, } from "./verify.js";
|
|
3
|
+
export { formatAssertionVerifyReport } from "./format.js";
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// Barrel: only what crosses the package boundary (assertion-state.ts and
|
|
2
|
+
// enhanceAssertionsTool.ts). Internal helpers and the per-test-type lint
|
|
3
|
+
// modules are imported directly by their sibling files and by tests — listing
|
|
4
|
+
// them here made every internal name look like public API.
|
|
5
|
+
export { detectAssertionLanguage, computeAssertionMetrics } from "./metrics.js";
|
|
6
|
+
export { verifyAssertionEnhancement, sha256Of, } from "./verify.js";
|
|
7
|
+
export { formatAssertionVerifyReport } from "./format.js";
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { strippedSources } from "./strip-dispatch.js";
|
|
2
|
+
import { lineOfOffset, } from "./lint-types.js";
|
|
3
|
+
import { lintApiShared } from "./api-shared-lints.js";
|
|
4
|
+
/**
|
|
5
|
+
* Integration-test lints: the shared API checks plus the machine-checkable
|
|
6
|
+
* slice of the D5 `chaining` dim. Whether chained values are the RIGHT ones is
|
|
7
|
+
* judge territory; a literal UUID embedded in a request is not — an ID that
|
|
8
|
+
* exists before the run started cannot have come from a prior response.
|
|
9
|
+
*/
|
|
10
|
+
const UUID_IN_REQUEST_RE = /(?:sendRequest|send_request)\s*\([^)]*['"`][^'"`]*\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b/gi;
|
|
11
|
+
/** A hardcoded UUID inside a request call — chained IDs should come from a
|
|
12
|
+
* prior step's response. Warn-only: fixtures may legitimately pin seeded IDs. */
|
|
13
|
+
function lintHardcodedRequestIds(commentless) {
|
|
14
|
+
const findings = [];
|
|
15
|
+
UUID_IN_REQUEST_RE.lastIndex = 0;
|
|
16
|
+
let m;
|
|
17
|
+
while ((m = UUID_IN_REQUEST_RE.exec(commentless)) !== null) {
|
|
18
|
+
findings.push({
|
|
19
|
+
rule: "hardcoded-request-id",
|
|
20
|
+
severity: "warn",
|
|
21
|
+
line: lineOfOffset(commentless, m.index),
|
|
22
|
+
message: "A request embeds a hardcoded UUID.",
|
|
23
|
+
remediation: "Extract the ID from the prior step's response (SDK helper) instead of hardcoding it, unless it is intentionally seeded fixture data.",
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
return findings;
|
|
27
|
+
}
|
|
28
|
+
export function lintIntegrationTest(raw, language, opts) {
|
|
29
|
+
if (language === "java")
|
|
30
|
+
return [];
|
|
31
|
+
const findings = lintApiShared(raw, language, opts);
|
|
32
|
+
if (opts?.scopeLines === undefined) {
|
|
33
|
+
findings.push(...lintHardcodedRequestIds(strippedSources(raw, language).commentless));
|
|
34
|
+
}
|
|
35
|
+
return findings;
|
|
36
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function blankJsRegexLiterals(src: string): string;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Offset-preserving blanking of JS/TS regex literals. `pom-scope/strip.ts`
|
|
3
|
+
* knows nothing about regex literals, so an apostrophe inside one (e.g.
|
|
4
|
+
* `getByText(/Don't have an account/)`) opens a phantom string span that
|
|
5
|
+
* swallows real code through the next quote — corrupting metrics and lints on
|
|
6
|
+
* perfectly valid specs. Blanking regex interiors first removes the hazard.
|
|
7
|
+
*
|
|
8
|
+
* Heuristic regex-start detection (`/` is a regex only where an expression may
|
|
9
|
+
* begin): the previous non-space character is an operator/opener/keyword end,
|
|
10
|
+
* not an identifier/literal end. Division therefore stays untouched. Only used
|
|
11
|
+
* as a pre-pass for scanners that never need regex CONTENT — lints that match
|
|
12
|
+
* regex text (permissive-status) run on comment-stripped source instead.
|
|
13
|
+
*/
|
|
14
|
+
const REGEX_MAY_FOLLOW = new Set([
|
|
15
|
+
"(",
|
|
16
|
+
",",
|
|
17
|
+
"=",
|
|
18
|
+
":",
|
|
19
|
+
"[",
|
|
20
|
+
"!",
|
|
21
|
+
"&",
|
|
22
|
+
"|",
|
|
23
|
+
"?",
|
|
24
|
+
"{",
|
|
25
|
+
";",
|
|
26
|
+
"+",
|
|
27
|
+
"-",
|
|
28
|
+
"*",
|
|
29
|
+
"%",
|
|
30
|
+
"<",
|
|
31
|
+
">",
|
|
32
|
+
"~",
|
|
33
|
+
"^",
|
|
34
|
+
]);
|
|
35
|
+
// Whole-token match only: an identifier merely ENDING in a keyword (`min`,
|
|
36
|
+
// `margin`, `todo`, `login`) is an ordinary value and a `/` after it is
|
|
37
|
+
// division — suffix-matching here silently blanked real code.
|
|
38
|
+
const KEYWORDS_BEFORE_REGEX = new Set([
|
|
39
|
+
"return",
|
|
40
|
+
"typeof",
|
|
41
|
+
"case",
|
|
42
|
+
"in",
|
|
43
|
+
"of",
|
|
44
|
+
"instanceof",
|
|
45
|
+
"new",
|
|
46
|
+
"do",
|
|
47
|
+
"else",
|
|
48
|
+
"void",
|
|
49
|
+
"delete",
|
|
50
|
+
"throw",
|
|
51
|
+
"await",
|
|
52
|
+
"yield",
|
|
53
|
+
]);
|
|
54
|
+
export function blankJsRegexLiterals(src) {
|
|
55
|
+
const out = src.split("");
|
|
56
|
+
let i = 0;
|
|
57
|
+
const n = src.length;
|
|
58
|
+
let inLineComment = false;
|
|
59
|
+
let inBlockComment = false;
|
|
60
|
+
let stringQuote;
|
|
61
|
+
while (i < n) {
|
|
62
|
+
const c = src[i];
|
|
63
|
+
if (inLineComment) {
|
|
64
|
+
if (c === "\n")
|
|
65
|
+
inLineComment = false;
|
|
66
|
+
i++;
|
|
67
|
+
continue;
|
|
68
|
+
}
|
|
69
|
+
if (inBlockComment) {
|
|
70
|
+
if (c === "*" && src[i + 1] === "/") {
|
|
71
|
+
inBlockComment = false;
|
|
72
|
+
i += 2;
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
i++;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
if (stringQuote !== undefined) {
|
|
79
|
+
if (c === "\\") {
|
|
80
|
+
i += 2;
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (c === stringQuote || (c === "\n" && stringQuote !== "`")) {
|
|
84
|
+
stringQuote = undefined;
|
|
85
|
+
}
|
|
86
|
+
i++;
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (c === "/" && src[i + 1] === "/") {
|
|
90
|
+
inLineComment = true;
|
|
91
|
+
i += 2;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
if (c === "/" && src[i + 1] === "*") {
|
|
95
|
+
inBlockComment = true;
|
|
96
|
+
i += 2;
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if (c === "'" || c === '"' || c === "`") {
|
|
100
|
+
stringQuote = c;
|
|
101
|
+
i++;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
if (c === "/") {
|
|
105
|
+
// Decide regex vs division from the previous non-whitespace token.
|
|
106
|
+
let j = i - 1;
|
|
107
|
+
while (j >= 0 && /\s/.test(src[j]))
|
|
108
|
+
j--;
|
|
109
|
+
const prev = j >= 0 ? src[j] : "";
|
|
110
|
+
// The COMPLETE identifier before the slash (scan across [\w$]) — only an
|
|
111
|
+
// exact keyword counts; after any other identifier, `/` is division.
|
|
112
|
+
let tokenStart = j;
|
|
113
|
+
while (tokenStart >= 0 && /[\w$]/.test(src[tokenStart]))
|
|
114
|
+
tokenStart--;
|
|
115
|
+
const prevToken = /[\w$]/.test(prev)
|
|
116
|
+
? src.slice(tokenStart + 1, j + 1)
|
|
117
|
+
: "";
|
|
118
|
+
const regexStart = j < 0 ||
|
|
119
|
+
REGEX_MAY_FOLLOW.has(prev) ||
|
|
120
|
+
KEYWORDS_BEFORE_REGEX.has(prevToken);
|
|
121
|
+
if (regexStart) {
|
|
122
|
+
// Scan to the closing unescaped `/` (a `/` inside [...] doesn't close).
|
|
123
|
+
let k = i + 1;
|
|
124
|
+
let inClass = false;
|
|
125
|
+
let closed = false;
|
|
126
|
+
while (k < n && src[k] !== "\n") {
|
|
127
|
+
const ck = src[k];
|
|
128
|
+
if (ck === "\\") {
|
|
129
|
+
k += 2;
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
if (ck === "[")
|
|
133
|
+
inClass = true;
|
|
134
|
+
else if (ck === "]")
|
|
135
|
+
inClass = false;
|
|
136
|
+
else if (ck === "/" && !inClass) {
|
|
137
|
+
closed = true;
|
|
138
|
+
break;
|
|
139
|
+
}
|
|
140
|
+
k++;
|
|
141
|
+
}
|
|
142
|
+
if (closed) {
|
|
143
|
+
for (let b = i; b <= k; b++)
|
|
144
|
+
out[b] = " ";
|
|
145
|
+
i = k + 1;
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
i++;
|
|
151
|
+
}
|
|
152
|
+
return out.join("");
|
|
153
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared types and text helpers for the per-test-type deterministic lints
|
|
3
|
+
* (ui-lints.ts, integration-lints.ts, contract-lints.ts). Each lint mirrors a
|
|
4
|
+
* dim of the eval D5 assertion-depth rubric; only near-zero-false-positive
|
|
5
|
+
* rules are `hard` (they block the verify pass), everything a legitimate spec
|
|
6
|
+
* could trip ships as `warn` until run evidence justifies promotion.
|
|
7
|
+
*/
|
|
8
|
+
export interface LintFinding {
|
|
9
|
+
rule: string;
|
|
10
|
+
severity: "hard" | "warn";
|
|
11
|
+
/** 1-indexed line, when the finding anchors to one. */
|
|
12
|
+
line?: number;
|
|
13
|
+
message: string;
|
|
14
|
+
remediation: string;
|
|
15
|
+
}
|
|
16
|
+
export interface LintOptions {
|
|
17
|
+
/** Maintenance scope: line numbers of assertion sites ADDED since the baseline.
|
|
18
|
+
* When set, site-scoped lints only report findings on these lines and
|
|
19
|
+
* structural (whole-file) lints — including the import lint — are skipped:
|
|
20
|
+
* they would flag pre-existing customer code the agent was told not to touch. */
|
|
21
|
+
scopeLines?: Set<number>;
|
|
22
|
+
}
|
|
23
|
+
export declare function lineOfOffset(src: string, offset: number): number;
|
|
24
|
+
/** `\b` breaks on `$`-names, so identifier boundaries use lookarounds. Single
|
|
25
|
+
* home for the pattern — it is subtle enough that hand-inlined copies drifted
|
|
26
|
+
* (found by the /simplify pass: six sites, one missing the f-string branch). */
|
|
27
|
+
export declare function identifierRe(name: string, flags?: string): RegExp;
|
|
28
|
+
/** A use of `name` inside a string interpolation — js/ts `${...}` or python
|
|
29
|
+
* f-string `{...}`. Interpolations are blanked wholesale by the string
|
|
30
|
+
* strippers, so callers test this against COMMENT-stripped source. */
|
|
31
|
+
export declare function interpolationRe(name: string, language: string): RegExp;
|
|
32
|
+
export declare function balancedCloseIndex(stripped: string, openIdx: number): number;
|
|
33
|
+
export declare function inScope(line: number, opts?: LintOptions): boolean;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { escapeRegExp } from "../regex.js";
|
|
2
|
+
// One-entry memo of a source's newline offsets: every scanner asks for line
|
|
3
|
+
// numbers of many offsets in the SAME string (assertion sites, lint findings),
|
|
4
|
+
// so counting newlines from index 0 per call was O(n × sites). The memo is
|
|
5
|
+
// per-string; a new source replaces it.
|
|
6
|
+
let newlineIndexSrc;
|
|
7
|
+
let newlineIndex = [];
|
|
8
|
+
export function lineOfOffset(src, offset) {
|
|
9
|
+
if (src !== newlineIndexSrc) {
|
|
10
|
+
newlineIndexSrc = src;
|
|
11
|
+
newlineIndex = [];
|
|
12
|
+
for (let i = 0; i < src.length; i++) {
|
|
13
|
+
if (src[i] === "\n")
|
|
14
|
+
newlineIndex.push(i);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
// Binary search: number of newlines strictly before `offset`, plus 1.
|
|
18
|
+
let lo = 0;
|
|
19
|
+
let hi = newlineIndex.length;
|
|
20
|
+
while (lo < hi) {
|
|
21
|
+
const mid = (lo + hi) >> 1;
|
|
22
|
+
if (newlineIndex[mid] < offset)
|
|
23
|
+
lo = mid + 1;
|
|
24
|
+
else
|
|
25
|
+
hi = mid;
|
|
26
|
+
}
|
|
27
|
+
return lo + 1;
|
|
28
|
+
}
|
|
29
|
+
/** `\b` breaks on `$`-names, so identifier boundaries use lookarounds. Single
|
|
30
|
+
* home for the pattern — it is subtle enough that hand-inlined copies drifted
|
|
31
|
+
* (found by the /simplify pass: six sites, one missing the f-string branch). */
|
|
32
|
+
export function identifierRe(name, flags = "") {
|
|
33
|
+
return new RegExp(`(?<![\\w$])${escapeRegExp(name)}(?![\\w$])`, flags);
|
|
34
|
+
}
|
|
35
|
+
/** A use of `name` inside a string interpolation — js/ts `${...}` or python
|
|
36
|
+
* f-string `{...}`. Interpolations are blanked wholesale by the string
|
|
37
|
+
* strippers, so callers test this against COMMENT-stripped source. */
|
|
38
|
+
export function interpolationRe(name, language) {
|
|
39
|
+
const open = language === "python" ? "\\{" : "\\$\\{";
|
|
40
|
+
return new RegExp(`${open}[^}]*(?<![\\w$])${escapeRegExp(name)}(?![\\w$])`);
|
|
41
|
+
}
|
|
42
|
+
export function balancedCloseIndex(stripped, openIdx) {
|
|
43
|
+
let depth = 0;
|
|
44
|
+
for (let i = openIdx; i < stripped.length; i++) {
|
|
45
|
+
if (stripped[i] === "(")
|
|
46
|
+
depth++;
|
|
47
|
+
else if (stripped[i] === ")") {
|
|
48
|
+
depth--;
|
|
49
|
+
if (depth === 0)
|
|
50
|
+
return i;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return -1;
|
|
54
|
+
}
|
|
55
|
+
export function inScope(line, opts) {
|
|
56
|
+
return opts?.scopeLines === undefined || opts.scopeLines.has(line);
|
|
57
|
+
}
|