@skyramp/mcp 0.3.6-rc.2.ac20 → 0.3.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/adapters/jestAdapter.js +0 -3
- package/build/adapters/mochaAdapter.js +0 -2
- package/build/adapters/playwrightAdapter.js +0 -3
- package/build/adapters/pytestAdapter.js +0 -12
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +2 -1
- package/build/prompts/testbot/testbot-prompts.js +1 -1
- package/build/services/TestGenerationService.js +13 -0
- package/build/tools/generate-tests/batchMockGenerationTool.js +25 -0
- package/build/tools/generateEnrichedIntegrationTestTool.js +10 -0
- package/build/tools/runExistingTestsTool.d.ts +2 -34
- package/build/tools/runExistingTestsTool.js +4 -104
- package/build/tools/submitReportTool.js +6 -126
- package/build/tools/workspace/initializeWorkspaceTool.js +99 -27
- package/build/types/ExternalTestExecution.d.ts +1 -67
- package/build/utils/executorWorkDir.d.ts +36 -0
- package/build/utils/executorWorkDir.js +77 -0
- package/package.json +1 -1
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import * as path from "path";
|
|
2
1
|
/**
|
|
3
2
|
* Shared jest + vitest adapter for skyramp_run_existing_tests. vitest's json
|
|
4
3
|
* reporter is jest-compatible, so both use ONE parser; only the reporter flag
|
|
@@ -41,7 +40,6 @@ export function parseJestJson(report, opts) {
|
|
|
41
40
|
results.push({
|
|
42
41
|
testId: `${name} › (file failed to run)`,
|
|
43
42
|
file: name,
|
|
44
|
-
...(path.isAbsolute(name) ? { absoluteFile: name } : {}),
|
|
45
43
|
status: "error",
|
|
46
44
|
message: stripVTControlCharacters(raw).trim() || undefined,
|
|
47
45
|
durationMs: 0,
|
|
@@ -57,7 +55,6 @@ export function parseJestJson(report, opts) {
|
|
|
57
55
|
results.push({
|
|
58
56
|
testId: `${name} › ${titlePath}`,
|
|
59
57
|
file: name,
|
|
60
|
-
...(path.isAbsolute(name) ? { absoluteFile: name } : {}),
|
|
61
58
|
status,
|
|
62
59
|
message,
|
|
63
60
|
durationMs: Math.round(a.duration ?? 0),
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import * as path from "path";
|
|
2
1
|
/**
|
|
3
2
|
* Mocha adapter for skyramp_run_existing_tests. Normalizes `mocha --reporter
|
|
4
3
|
* json` (emitted to stdout) into the neutral shape. Mocha does not tag
|
|
@@ -30,7 +29,6 @@ function toResult(t, status) {
|
|
|
30
29
|
return {
|
|
31
30
|
testId: `${file} › ${full}`,
|
|
32
31
|
file,
|
|
33
|
-
...(path.isAbsolute(file) ? { absoluteFile: file } : {}),
|
|
34
32
|
status,
|
|
35
33
|
message: status === "fail" || status === "error" ? messageOf(t) : undefined,
|
|
36
34
|
durationMs: Math.round(t.duration ?? 0),
|
|
@@ -12,7 +12,6 @@
|
|
|
12
12
|
* builds the flags appended to the suite's `testRunCommand`. The IO shell that
|
|
13
13
|
* actually spawns the run lives in `runExistingTestsTool`.
|
|
14
14
|
*/
|
|
15
|
-
import * as path from "path";
|
|
16
15
|
import { stripVTControlCharacters } from "util";
|
|
17
16
|
/**
|
|
18
17
|
* A Playwright project (or spec file) is "infra" when it exists to bring up /
|
|
@@ -68,7 +67,6 @@ function messageOf(test) {
|
|
|
68
67
|
}
|
|
69
68
|
export function parsePlaywrightJson(report, opts) {
|
|
70
69
|
const rep = (report ?? {});
|
|
71
|
-
const rootDir = rep.config?.rootDir;
|
|
72
70
|
const collected = [];
|
|
73
71
|
for (const fileSuite of rep.suites ?? []) {
|
|
74
72
|
collectSpecs(fileSuite, fileSuite.file ?? "", [], collected);
|
|
@@ -120,7 +118,6 @@ export function parsePlaywrightJson(report, opts) {
|
|
|
120
118
|
results.push({
|
|
121
119
|
testId,
|
|
122
120
|
file,
|
|
123
|
-
...(rootDir ? { absoluteFile: path.resolve(rootDir, file) } : {}),
|
|
124
121
|
status,
|
|
125
122
|
message: status === "fail" || status === "error" ? messageOf(test) : undefined,
|
|
126
123
|
durationMs,
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import * as path from "path";
|
|
2
1
|
/**
|
|
3
2
|
* pytest adapter for skyramp_run_existing_tests. Normalizes the
|
|
4
3
|
* `pytest-json-report` (`--json-report`) output into the neutral result shape.
|
|
@@ -40,8 +39,6 @@ function durationMsOf(test) {
|
|
|
40
39
|
export function parsePytestJson(report, opts) {
|
|
41
40
|
const rep = (report ?? {});
|
|
42
41
|
const tests = rep.tests ?? [];
|
|
43
|
-
const root = rep.root;
|
|
44
|
-
const abs = (f) => (root ? { absoluteFile: path.resolve(root, f) } : {});
|
|
45
42
|
// Real test outcomes.
|
|
46
43
|
const testResults = tests.map((t) => {
|
|
47
44
|
const status = mapOutcome(t.outcome);
|
|
@@ -49,7 +46,6 @@ export function parsePytestJson(report, opts) {
|
|
|
49
46
|
return {
|
|
50
47
|
testId: nodeid,
|
|
51
48
|
file: fileOf(nodeid),
|
|
52
|
-
...abs(fileOf(nodeid)),
|
|
53
49
|
status,
|
|
54
50
|
message: status === "fail" || status === "error" ? messageOf(t) : undefined,
|
|
55
51
|
durationMs: durationMsOf(t),
|
|
@@ -65,7 +61,6 @@ export function parsePytestJson(report, opts) {
|
|
|
65
61
|
return {
|
|
66
62
|
testId: `${nodeid} › (collection error)`,
|
|
67
63
|
file: fileOf(nodeid),
|
|
68
|
-
...abs(fileOf(nodeid)),
|
|
69
64
|
status: "error",
|
|
70
65
|
message: lastLine(c.longrepr),
|
|
71
66
|
durationMs: 0,
|
|
@@ -77,13 +72,6 @@ export function parsePytestJson(report, opts) {
|
|
|
77
72
|
// When collection failed AND no real test ran, the suite as a whole could not be
|
|
78
73
|
// collected — environmental (a wall of red), not PR signal. When some real tests
|
|
79
74
|
// DID run, the collection errors above stand as ordinary PR-signal error results.
|
|
80
|
-
// NOTE: a collection error names the file that failed to import, but says nothing
|
|
81
|
-
// about WHY. A missing settings module, a database that is down or any shared
|
|
82
|
-
// dependency failure surfaces per-module, landing on exactly the files the run
|
|
83
|
-
// selected — indistinguishable here from a file the PR itself broke. So a run that
|
|
84
|
-
// collected nothing stays unhealthy even when scoped, and the file keeps its
|
|
85
|
-
// Unknown baseline. Treating it as PR signal would let an environment outage be
|
|
86
|
-
// reported as a test the change broke, and then repaired.
|
|
87
75
|
if (collectionErrors.length > 0 && testResults.length === 0) {
|
|
88
76
|
environmentHealthy = false;
|
|
89
77
|
const first = collectionErrors[0];
|
|
@@ -91,6 +91,7 @@ Create one service entry per deployable unit. You MUST include every backend/API
|
|
|
91
91
|
- Single service: set testDirectory to tests/skyramp.
|
|
92
92
|
- Multiple services or monorepos: set testDirectory to tests/skyramp/<serviceDirName>, where <serviceDirName> is the service directory name with path separators and whitespace replaced by hyphens.
|
|
93
93
|
Framework config takes precedence. Use the Skyramp deterministic fallback only when no framework-configured test directory is available.
|
|
94
|
+
NEVER set testDirectory to \`.skyramp\`, to any path inside it, or to \`.\` or the repo root, whatever the framework config or existing test files say. \`.skyramp\` is the executor's own working area: it holds the run videos and executor artefacts. skyramp_init_workspace rejects all of these, and the generation tools refuse to write there.
|
|
94
95
|
</basic_fields>
|
|
95
96
|
|
|
96
97
|
### API fields
|
|
@@ -160,7 +161,7 @@ Before calling skyramp_init_workspace, confirm all of the following:
|
|
|
160
161
|
4. Every service has api.baseUrl set to a valid, discoverable URL. Use localhost for local services or the actual deployment URL for cloud or external services. Never fabricate a URL.
|
|
161
162
|
5. Every service with authType apiKey has authHeader explicitly set to the actual custom header name (such as "X-API-Key" or "X-Admin-Key"). If you cannot find the header name in the source code, env vars, or README, do NOT use authType apiKey. Use authType none instead and add a YAML comment explaining auth is unresolved.
|
|
162
163
|
6. framework matches language (python uses pytest or robot, typescript or javascript uses playwright, java uses junit).
|
|
163
|
-
7. testDirectory follows the stable resolution rules above: framework config file when present (Playwright testDir in playwright.config.ts, pytest testpaths in pytest.ini or pyproject.toml, JUnit test source dir in pom.xml or build.gradle); otherwise the deterministic default (tests/skyramp for a single service, tests/skyramp/<serviceDirName> for multiple services).
|
|
164
|
+
7. testDirectory follows the stable resolution rules above: framework config file when present (Playwright testDir in playwright.config.ts, pytest testpaths in pytest.ini or pyproject.toml, JUnit test source dir in pom.xml or build.gradle); otherwise the deterministic default (tests/skyramp for a single service, tests/skyramp/<serviceDirName> for multiple services). No service uses \`.skyramp\`, \`.\` or the repo root.
|
|
164
165
|
8. If serverStartCommand is provided, it matches the runtime. If serverStopCommand is provided, runtime is "docker" and the command is a Docker command.
|
|
165
166
|
9. For services in docker-compose.yml: runtime MUST be "docker" and the command MUST be a docker command such as "docker compose up -d --build <service-name>" when the service has a build context. Always include it since it is derivable from the service name.
|
|
166
167
|
10. NEVER use application-level commands (uvicorn, npm, node, python, java, etc.) with runtime "docker".
|
|
@@ -212,7 +212,7 @@ ${maintenanceBeforeExecStep}
|
|
|
212
212
|
|
|
213
213
|
e. Call \`skyramp_actions\` with \`stateFile\` (from \`skyramp_analyze_changes\` output) and apply the edits it returns.
|
|
214
214
|
|
|
215
|
-
f. Verify external-test fixes. **This step is not optional and it is the easiest one to forget — you have just edited files in step 2(e), so come back here before you move on to anything else.** It applies whenever step 2(a) reported a real pass/fail result for a file you then edited. It does NOT apply when step 2(a) returned \`skipped: true\` or \`ran: 0\` for every suite — there is no baseline to compare against, so say so in your report instead of re-running. When it applies: re-run those \`[external]\` files with \`skyramp_run_existing_tests\` (\`mode: "verify"\`, \`stateFile\`)
|
|
215
|
+
f. Verify external-test fixes. **This step is not optional and it is the easiest one to forget — you have just edited files in step 2(e), so come back here before you move on to anything else.** It applies whenever step 2(a) reported a real pass/fail result for a file you then edited. It does NOT apply when step 2(a) returned \`skipped: true\` or \`ran: 0\` for every suite — there is no baseline to compare against, so say so in your report instead of re-running. When it applies: re-run those \`[external]\` files with \`skyramp_run_existing_tests\` (\`mode: "verify"\`, \`stateFile\`) and record each file's result as its \`afterStatus\`. Editing an \`[external]\` file that step 2(a) confirmed failing and NOT re-running it leaves your own fix unverified — you would be reporting a repair you never saw work. A still-failing verify is surfaced in the report — do not loop.
|
|
216
216
|
|
|
217
217
|
3. **Code review:** Find the logic bugs in the code that this change touches. Read the implementation of each changed endpoint: the route handler, and the functions that it calls to read or write data. For a changed screen, read the component and the functions that it calls. Read these files even when the diff does not contain them — a defect often sits in the code that the change depends on. Report each finding in \`issuesFound\` with a severity, and say which file and line holds it. Common patterns to flag:
|
|
218
218
|
- Computed fields not recalculated after mutation (e.g. \`total_amount\` unchanged after items are added/removed)
|
|
@@ -11,6 +11,7 @@ import { TestType } from "../types/TestTypes.js";
|
|
|
11
11
|
import { logger } from "../utils/logger.js";
|
|
12
12
|
import { normalizeLanguageParams } from "../utils/normalizeParams.js";
|
|
13
13
|
import { stageGeneratedPaths, resolveOutputDir } from "../utils/gitStaging.js";
|
|
14
|
+
import { isInsideExecutorWorkDir, executorWorkDirRefusal, generationTargets, } from "../utils/executorWorkDir.js";
|
|
14
15
|
import { getTestsRepoDir } from "../utils/AnalysisStateManager.js";
|
|
15
16
|
import { recordReuseHandOff } from "../tools/code-refactor/reuse-state.js";
|
|
16
17
|
import { isModularizeFirstTarget } from "../utils/reuseRouting.js";
|
|
@@ -73,6 +74,18 @@ export class TestGenerationService {
|
|
|
73
74
|
});
|
|
74
75
|
params.outputDir = resolved;
|
|
75
76
|
}
|
|
77
|
+
// Every generation tool reaches the filesystem through this method, so one
|
|
78
|
+
// check here covers them all. Nothing written under `.skyramp` is delivered
|
|
79
|
+
// as a test — an observed run put 7 specs there because the only absolute
|
|
80
|
+
// path the prompt gave it was the trace-zip directory. The error names the
|
|
81
|
+
// field and the replacement so the agent can correct itself in one round trip.
|
|
82
|
+
const badTarget = generationTargets(params.outputDir, params.output).find(isInsideExecutorWorkDir);
|
|
83
|
+
if (badTarget) {
|
|
84
|
+
return {
|
|
85
|
+
content: [{ type: "text", text: executorWorkDirRefusal(badTarget) }],
|
|
86
|
+
isError: true,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
76
89
|
// Log prompt parameter using reusable utility
|
|
77
90
|
logger.info("Generating test", {
|
|
78
91
|
prompt: params.prompt,
|
|
@@ -10,6 +10,7 @@ import { getRestMockMethodValidationError, normalizeRestMockMethod, } from "../.
|
|
|
10
10
|
import { getGrpcMockResponseValidationError } from "../../utils/grpcMockValidation.js";
|
|
11
11
|
import { validateMockCompatibility } from "../../utils/mockCompatibility.js";
|
|
12
12
|
import { buildTrafficConfigOptions, validateTrafficConfig, } from "./generateMockRestTool.js";
|
|
13
|
+
import { isInsideExecutorWorkDir, executorWorkDirRefusal, generationTargets, } from "../../utils/executorWorkDir.js";
|
|
13
14
|
export const BATCH_MOCK_PROTOCOL_EXAMPLES = `
|
|
14
15
|
<example protocol="rest">
|
|
15
16
|
Batch REST item. Required fields: protocol="rest", endpointURL, method unless apiSchema is provided.
|
|
@@ -361,6 +362,15 @@ function buildGenerateOptions(spec, shared) {
|
|
|
361
362
|
}
|
|
362
363
|
export async function executeBatchMockGeneration(params) {
|
|
363
364
|
const { mocks, outputDir } = params;
|
|
365
|
+
// This tool calls the client directly rather than through
|
|
366
|
+
// TestGenerationService, so it carries its own working-area check. mkdirSync
|
|
367
|
+
// below would otherwise create the directory before anything is validated.
|
|
368
|
+
if (isInsideExecutorWorkDir(outputDir)) {
|
|
369
|
+
return {
|
|
370
|
+
content: [{ type: "text", text: executorWorkDirRefusal(outputDir) }],
|
|
371
|
+
isError: true,
|
|
372
|
+
};
|
|
373
|
+
}
|
|
364
374
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
365
375
|
const client = new SkyrampClient();
|
|
366
376
|
const results = [];
|
|
@@ -402,6 +412,21 @@ export async function executeBatchMockGeneration(params) {
|
|
|
402
412
|
continue;
|
|
403
413
|
}
|
|
404
414
|
const resolvedOutputDir = spec.outputDir ?? outputDir;
|
|
415
|
+
// A per-mock outputDir or output name can point back into the working area
|
|
416
|
+
// even when the batch-level one does not.
|
|
417
|
+
const badTarget = generationTargets(resolvedOutputDir, spec.output).find(isInsideExecutorWorkDir);
|
|
418
|
+
if (badTarget) {
|
|
419
|
+
results.push({
|
|
420
|
+
index: i,
|
|
421
|
+
protocol,
|
|
422
|
+
endpointURL: spec.endpointURL,
|
|
423
|
+
method,
|
|
424
|
+
topic: spec.kafkaTopic,
|
|
425
|
+
success: false,
|
|
426
|
+
error: executorWorkDirRefusal(badTarget),
|
|
427
|
+
});
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
405
430
|
fs.mkdirSync(resolvedOutputDir, { recursive: true });
|
|
406
431
|
// Build options and generate
|
|
407
432
|
const options = buildGenerateOptions(spec, {
|
|
@@ -7,6 +7,7 @@ import { isAuthorizationHeaderName } from "../utils/workspaceAuth.js";
|
|
|
7
7
|
import { WORKER_CONTROL_PORT } from "../utils/versions.js";
|
|
8
8
|
import * as fs from "fs";
|
|
9
9
|
import * as path from "path";
|
|
10
|
+
import { isInsideExecutorWorkDir, executorWorkDirRefusal, generationTargets, } from "../utils/executorWorkDir.js";
|
|
10
11
|
const TOOL_NAME = "skyramp_generate_enriched_integration_test";
|
|
11
12
|
const enrichedIntegrationTestSchema = {
|
|
12
13
|
scenarioFile: z
|
|
@@ -132,6 +133,15 @@ export async function executeGenerateEnrichedIntegrationTest(params) {
|
|
|
132
133
|
if (resolvedAuthType !== undefined && isAuthorizationHeaderName(resolvedAuthHeader || "")) {
|
|
133
134
|
resolvedAuthHeader = undefined;
|
|
134
135
|
}
|
|
136
|
+
// This tool calls the client directly instead of going through
|
|
137
|
+
// TestGenerationService, so it needs its own copy of the working-area check.
|
|
138
|
+
const badTarget = generationTargets(outputDir, output).find(isInsideExecutorWorkDir);
|
|
139
|
+
if (badTarget) {
|
|
140
|
+
return {
|
|
141
|
+
content: [{ type: "text", text: executorWorkDirRefusal(badTarget) }],
|
|
142
|
+
isError: true,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
135
145
|
const client = new SkyrampClient();
|
|
136
146
|
const result = await client.generateRestTest({
|
|
137
147
|
traceFilePath: scenarioFile,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
2
|
import { WorkspaceConfig } from "../workspace/workspace.js";
|
|
3
|
-
import {
|
|
3
|
+
import { ParsedExternalRun } from "../types/ExternalTestExecution.js";
|
|
4
4
|
import { SUPPORTED_FRAMEWORKS } from "../workspace/frameworks.js";
|
|
5
5
|
/** Test-run lifecycle resolved from a service's workspace.yml runtimeDetails. */
|
|
6
6
|
export interface ResolvedRunConfig {
|
|
@@ -42,9 +42,7 @@ export interface RunExistingTestsParams {
|
|
|
42
42
|
workspacePath: string;
|
|
43
43
|
service?: string;
|
|
44
44
|
testSelectors?: string[];
|
|
45
|
-
|
|
46
|
-
* post-edit re-run. A wrong label is read as fact by the report. */
|
|
47
|
-
mode: "confirm" | "verify";
|
|
45
|
+
mode?: "confirm" | "verify";
|
|
48
46
|
project?: string;
|
|
49
47
|
stateFile?: string;
|
|
50
48
|
repository?: string;
|
|
@@ -130,36 +128,6 @@ export declare function aggregateRuns(parsed: ParsedExternalRun[]): ParsedExtern
|
|
|
130
128
|
*/
|
|
131
129
|
export declare function runExternalSuite(spec: ReaderSpec, cfg: ResolvedRunConfig, opts: PlaywrightRunOpts): Promise<ParsedExternalRun>;
|
|
132
130
|
export declare function runPlaywright(cfg: ResolvedRunConfig, opts: PlaywrightRunOpts): Promise<ParsedExternalRun>;
|
|
133
|
-
/**
|
|
134
|
-
* Reduce a run's flat result list to one outcome per spec file.
|
|
135
|
-
*
|
|
136
|
-
* This is the only place that knows BOTH what the framework reported and what the
|
|
137
|
-
* run asked it for, which is what `complete` needs. Deriving it later from the
|
|
138
|
-
* results alone is not possible: a file re-run for one test looks identical to a
|
|
139
|
-
* file whose other tests silently vanished.
|
|
140
|
-
*
|
|
141
|
-
* `partialFiles` are the files a `file::test title` selector narrowed to a subset of
|
|
142
|
-
* their tests — the one narrowing this tool applies itself, and so the only one it
|
|
143
|
-
* can state with certainty.
|
|
144
|
-
*/
|
|
145
|
-
export declare function summarizeFilesInRun(results: ExternalTestResult[], opts: {
|
|
146
|
-
partialFiles: Set<string>;
|
|
147
|
-
}): ExternalFileOutcome[];
|
|
148
|
-
/** Placeholder identity for a record that names no suite because none resolved.
|
|
149
|
-
* Deliberately not a valid suiteIdOf output, so it can never collide with one. */
|
|
150
|
-
export declare const NO_RUNNABLE_SUITE = "(no runnable suite)";
|
|
151
|
-
/** Stable identity for ONE configured suite. A service can declare several
|
|
152
|
-
* (workspace.yml testSuites), so its name alone collides — two suites under one
|
|
153
|
-
* service would share a slot and the later would mask the other instead of both
|
|
154
|
-
* counting. */
|
|
155
|
-
export declare function suiteIdOf(cfg: {
|
|
156
|
-
serviceName: string;
|
|
157
|
-
framework: string;
|
|
158
|
-
testRunCommand: string;
|
|
159
|
-
}): string;
|
|
160
|
-
/** The files a selector narrowed to individual tests. A bare `file` selector runs
|
|
161
|
-
* the whole file; `file::test title` does not. */
|
|
162
|
-
export declare function partialFilesFrom(selectors: string[]): Set<string>;
|
|
163
131
|
/**
|
|
164
132
|
* Orchestrates an external test run: resolve config → apply maxTests cap → run →
|
|
165
133
|
* persist a record to the state file. Pure of MCP/registration concerns and
|
|
@@ -440,95 +440,6 @@ const defaultDeps = {
|
|
|
440
440
|
},
|
|
441
441
|
now: () => new Date().toISOString(),
|
|
442
442
|
};
|
|
443
|
-
/** Worst outcome wins for a file. `error` (never reached its assertions) outranks
|
|
444
|
-
* `fail` so a collection failure is never reported as an assertion failure, and
|
|
445
|
-
* `skipped` sits below everything: it only wins when nothing else ran. */
|
|
446
|
-
const STATUS_RANK = {
|
|
447
|
-
skipped: 0,
|
|
448
|
-
pass: 1,
|
|
449
|
-
fail: 2,
|
|
450
|
-
error: 3,
|
|
451
|
-
};
|
|
452
|
-
/**
|
|
453
|
-
* Reduce a run's flat result list to one outcome per spec file.
|
|
454
|
-
*
|
|
455
|
-
* This is the only place that knows BOTH what the framework reported and what the
|
|
456
|
-
* run asked it for, which is what `complete` needs. Deriving it later from the
|
|
457
|
-
* results alone is not possible: a file re-run for one test looks identical to a
|
|
458
|
-
* file whose other tests silently vanished.
|
|
459
|
-
*
|
|
460
|
-
* `partialFiles` are the files a `file::test title` selector narrowed to a subset of
|
|
461
|
-
* their tests — the one narrowing this tool applies itself, and so the only one it
|
|
462
|
-
* can state with certainty.
|
|
463
|
-
*/
|
|
464
|
-
export function summarizeFilesInRun(results, opts) {
|
|
465
|
-
const byFile = new Map();
|
|
466
|
-
for (const r of results) {
|
|
467
|
-
if (typeof r.file !== "string" || r.file === "")
|
|
468
|
-
continue;
|
|
469
|
-
const key = r.absoluteFile ?? r.file;
|
|
470
|
-
byFile.set(key, [...(byFile.get(key) ?? []), r]);
|
|
471
|
-
}
|
|
472
|
-
return [...byFile.values()].map((group) => {
|
|
473
|
-
const counts = { pass: 0, fail: 0, error: 0, skipped: 0 };
|
|
474
|
-
for (const r of group)
|
|
475
|
-
counts[r.status] = (counts[r.status] ?? 0) + 1;
|
|
476
|
-
const status = group.reduce((a, b) => STATUS_RANK[b.status] > STATUS_RANK[a.status] ? b : a).status;
|
|
477
|
-
const first = group[0];
|
|
478
|
-
return {
|
|
479
|
-
file: first.file,
|
|
480
|
-
...(first.absoluteFile ? { absoluteFile: first.absoluteFile } : {}),
|
|
481
|
-
status,
|
|
482
|
-
counts,
|
|
483
|
-
complete: !isPartial(first, opts.partialFiles),
|
|
484
|
-
errors: group
|
|
485
|
-
.filter((r) => r.status === "fail" || r.status === "error")
|
|
486
|
-
.map((r) => `${r.testId}: ${r.message ?? r.status}`),
|
|
487
|
-
durationMs: group.reduce((sum, r) => sum + (r.durationMs ?? 0), 0),
|
|
488
|
-
};
|
|
489
|
-
});
|
|
490
|
-
}
|
|
491
|
-
/** A selector spelling and a result spelling need not agree — the selector is
|
|
492
|
-
* repo-relative, the result may be absolute or framework-root-relative — so match
|
|
493
|
-
* on a path-segment suffix in either direction. */
|
|
494
|
-
function isPartial(r, partialFiles) {
|
|
495
|
-
if (partialFiles.size === 0)
|
|
496
|
-
return false;
|
|
497
|
-
const candidates = [r.file, r.absoluteFile].filter((c) => typeof c === "string" && c !== "");
|
|
498
|
-
for (const sel of partialFiles) {
|
|
499
|
-
for (const c of candidates) {
|
|
500
|
-
if (c === sel || c.endsWith("/" + sel) || sel.endsWith("/" + c))
|
|
501
|
-
return true;
|
|
502
|
-
}
|
|
503
|
-
}
|
|
504
|
-
return false;
|
|
505
|
-
}
|
|
506
|
-
/** Placeholder identity for a record that names no suite because none resolved.
|
|
507
|
-
* Deliberately not a valid suiteIdOf output, so it can never collide with one. */
|
|
508
|
-
export const NO_RUNNABLE_SUITE = "(no runnable suite)";
|
|
509
|
-
/** Stable identity for ONE configured suite. A service can declare several
|
|
510
|
-
* (workspace.yml testSuites), so its name alone collides — two suites under one
|
|
511
|
-
* service would share a slot and the later would mask the other instead of both
|
|
512
|
-
* counting. */
|
|
513
|
-
export function suiteIdOf(cfg) {
|
|
514
|
-
return `${cfg.serviceName}::${cfg.framework}::${cfg.testRunCommand}`;
|
|
515
|
-
}
|
|
516
|
-
/** The files a selector narrowed to individual tests. A bare `file` selector runs
|
|
517
|
-
* the whole file; `file::test title` does not. */
|
|
518
|
-
export function partialFilesFrom(selectors) {
|
|
519
|
-
const out = new Set();
|
|
520
|
-
for (const sel of selectors) {
|
|
521
|
-
const [file, ...rest] = sel.split("::");
|
|
522
|
-
// A trailing `::` names no title, so every builder drops it and the whole file
|
|
523
|
-
// runs. Treating it as partial would throw away a real status.
|
|
524
|
-
if (rest.join("::").trim() === "")
|
|
525
|
-
continue;
|
|
526
|
-
const trimmed = file.trim();
|
|
527
|
-
if (trimmed)
|
|
528
|
-
out.add(trimmed);
|
|
529
|
-
}
|
|
530
|
-
return out;
|
|
531
|
-
}
|
|
532
443
|
/** Appends a run record into UnifiedAnalysisState.externalTestResults for the
|
|
533
444
|
* given repo section. Best-effort: a state failure never fails the run. */
|
|
534
445
|
async function persistExternalRun(stateFile, repository, record) {
|
|
@@ -552,7 +463,7 @@ async function persistExternalRun(stateFile, repository, record) {
|
|
|
552
463
|
* fully injectable for tests.
|
|
553
464
|
*/
|
|
554
465
|
export async function runExternalTests(params, deps = defaultDeps) {
|
|
555
|
-
const mode = params.mode;
|
|
466
|
+
const mode = params.mode ?? "confirm";
|
|
556
467
|
const selectors = params.testSelectors ?? [];
|
|
557
468
|
let effective = selectors;
|
|
558
469
|
let truncated = false;
|
|
@@ -575,11 +486,6 @@ export async function runExternalTests(params, deps = defaultDeps) {
|
|
|
575
486
|
if (params.stateFile) {
|
|
576
487
|
await persistExternalRun(params.stateFile, params.repository, {
|
|
577
488
|
...skippedRun,
|
|
578
|
-
files: [],
|
|
579
|
-
// No run config resolved, so there is no suite to identify — and a service
|
|
580
|
-
// name here would read as one. The reader never keys on it (the record is
|
|
581
|
-
// skipped, and skipped records are dropped first), so name it for what it is.
|
|
582
|
-
suite: NO_RUNNABLE_SUITE,
|
|
583
489
|
mode,
|
|
584
490
|
ranAt: deps.now(),
|
|
585
491
|
repository: params.repository,
|
|
@@ -612,8 +518,6 @@ export async function runExternalTests(params, deps = defaultDeps) {
|
|
|
612
518
|
if (params.stateFile) {
|
|
613
519
|
await persistExternalRun(params.stateFile, params.repository, {
|
|
614
520
|
...routedOut,
|
|
615
|
-
files: [],
|
|
616
|
-
suite: suiteIdOf(cfg),
|
|
617
521
|
mode,
|
|
618
522
|
ranAt: deps.now(),
|
|
619
523
|
repository: params.repository,
|
|
@@ -638,13 +542,8 @@ export async function runExternalTests(params, deps = defaultDeps) {
|
|
|
638
542
|
};
|
|
639
543
|
perSuite.push(suiteResult);
|
|
640
544
|
if (params.stateFile) {
|
|
641
|
-
const files = summarizeFilesInRun(suiteResult.results, {
|
|
642
|
-
partialFiles: partialFilesFrom(routing.kept),
|
|
643
|
-
});
|
|
644
545
|
await persistExternalRun(params.stateFile, params.repository, {
|
|
645
546
|
...suiteResult,
|
|
646
|
-
files,
|
|
647
|
-
suite: suiteIdOf(cfg),
|
|
648
547
|
mode,
|
|
649
548
|
ranAt: deps.now(),
|
|
650
549
|
repository: params.repository,
|
|
@@ -676,7 +575,8 @@ export function registerRunExistingTestsTool(server) {
|
|
|
676
575
|
.describe("Spec files to run (paths relative to the run command's cwd), optionally `file::test title` to also filter by name. Omit to run the whole configured suite."),
|
|
677
576
|
mode: z
|
|
678
577
|
.enum(["confirm", "verify"])
|
|
679
|
-
.
|
|
578
|
+
.optional()
|
|
579
|
+
.describe("confirm (default): scoped run to establish which tests a change breaks; disables the suite's fail-fast cap (e.g. Playwright's maxFailures) so ALL breakages are counted. verify: re-run after an edit."),
|
|
680
580
|
project: z
|
|
681
581
|
.string()
|
|
682
582
|
.optional()
|
|
@@ -735,7 +635,7 @@ export function registerRunExistingTestsTool(server) {
|
|
|
735
635
|
AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, {
|
|
736
636
|
workspacePath: params.workspacePath,
|
|
737
637
|
service: params.service ?? "",
|
|
738
|
-
mode: params.mode,
|
|
638
|
+
mode: params.mode ?? "confirm",
|
|
739
639
|
}).catch((err) => {
|
|
740
640
|
logger.warning("Analytics event failed", { error: String(err) });
|
|
741
641
|
});
|
|
@@ -84,96 +84,6 @@ const MAINTENANCE_CHANGE_ACTIONS = new Set([
|
|
|
84
84
|
DriftAction.Regenerate,
|
|
85
85
|
DriftAction.Delete,
|
|
86
86
|
]);
|
|
87
|
-
const EXTERNAL_TO_EXECUTION_STATUS = {
|
|
88
|
-
pass: TestExecutionStatus.Pass,
|
|
89
|
-
fail: TestExecutionStatus.Fail,
|
|
90
|
-
error: TestExecutionStatus.Error,
|
|
91
|
-
};
|
|
92
|
-
/** Does this run's file outcome describe `knownPath`?
|
|
93
|
-
*
|
|
94
|
-
* `absoluteFile` is the framework's own path resolved against the root it reported,
|
|
95
|
-
* so it names one file and is compared exactly. An outcome without one is refused:
|
|
96
|
-
* a relative spelling is a suffix, and a suffix cannot separate
|
|
97
|
-
* `packages/a/tests/x.spec.ts` from `packages/b/tests/x.spec.ts`. Guessing there and
|
|
98
|
-
* guarding the guess is what this deliberately does not do — a missing status is a
|
|
99
|
-
* row that reads Unknown, which is what it reads today; a wrong one is a false claim
|
|
100
|
-
* about someone's tests.
|
|
101
|
-
*
|
|
102
|
-
* The cost is real and accepted: a suite running inside a container reports the
|
|
103
|
-
* container's root, which cannot equal the host path discovery recorded, so those
|
|
104
|
-
* repos get no status until the run records a root the report can line up. */
|
|
105
|
-
function outcomeMatches(outcome, knownPath) {
|
|
106
|
-
return (typeof outcome.absoluteFile === "string" &&
|
|
107
|
-
path.resolve(outcome.absoluteFile) === path.resolve(knownPath));
|
|
108
|
-
}
|
|
109
|
-
/** One line describing what the suite saw, for a row whose status came from the run
|
|
110
|
-
* rather than from an execution the agent drafted a summary for. Without it the
|
|
111
|
-
* rendered cell is a bare `Error` or `Pass` with nothing behind it. */
|
|
112
|
-
function describeOutcome(o) {
|
|
113
|
-
// Defended like the record reads around it: a malformed outcome must not throw
|
|
114
|
-
// out of the handler and take the whole report submission with it.
|
|
115
|
-
const c = o.counts ?? { pass: 0, fail: 0, error: 0, skipped: 0 };
|
|
116
|
-
const parts = [
|
|
117
|
-
c.pass ? `${c.pass} passed` : "",
|
|
118
|
-
c.fail ? `${c.fail} failed` : "",
|
|
119
|
-
c.error ? `${c.error} errored` : "",
|
|
120
|
-
c.skipped ? `${c.skipped} skipped` : "",
|
|
121
|
-
].filter(Boolean);
|
|
122
|
-
const head = `${parts.join(", ") || "no tests reported"} in the repo's own suite`;
|
|
123
|
-
// Adapter messages carry stack traces, so collapse whitespace before truncating:
|
|
124
|
-
// an embedded newline would break the maintenance row this lands in.
|
|
125
|
-
const first = (o.errors ?? [])[0]?.replace(/\s+/g, " ").trim();
|
|
126
|
-
return first ? `${head} — ${first.slice(0, 200)}` : head;
|
|
127
|
-
}
|
|
128
|
-
function externalStatusesFor(records, knownPath) {
|
|
129
|
-
// LATEST per suite, then WORST across suites. Both halves matter: a suite re-run
|
|
130
|
-
// after another edit supersedes its own earlier verdict, so an obsolete failure
|
|
131
|
-
// must not outlive the fix that repaired it; but two different suites can each
|
|
132
|
-
// own the same file, and a pass in one must not mask a failure in the other.
|
|
133
|
-
const RANK = { pass: 0, fail: 1, error: 2 };
|
|
134
|
-
const statusIn = (mode) => {
|
|
135
|
-
const latestPerSuite = new Map();
|
|
136
|
-
for (const record of records ?? []) {
|
|
137
|
-
if (record.mode !== mode || record.skipped || !record.environmentHealthy)
|
|
138
|
-
continue;
|
|
139
|
-
// A suite that stopped on --max-failures left tests unrun and does not say
|
|
140
|
-
// which, so no file it touched can stand as covered. The adapter reports this;
|
|
141
|
-
// it is not a guess. (The tool forces --max-failures=0 for confirm, so this is
|
|
142
|
-
// reachable on a verify re-run under the repo's own cap.)
|
|
143
|
-
if (record.diagnostics?.maxFailuresBail)
|
|
144
|
-
continue;
|
|
145
|
-
const hit = (record.files ?? []).find((o) => outcomeMatches(o, knownPath));
|
|
146
|
-
if (!hit)
|
|
147
|
-
continue;
|
|
148
|
-
const key = record.suite ?? "";
|
|
149
|
-
const prior = latestPerSuite.get(key);
|
|
150
|
-
// `ranAt` is an ISO timestamp, so a lexical compare is chronological. The
|
|
151
|
-
// newest run wins BEFORE it is judged: a later partial re-run supersedes an
|
|
152
|
-
// earlier complete one, and must leave the row unknown rather than let the
|
|
153
|
-
// stale complete outcome stand in for it.
|
|
154
|
-
if (!prior || record.ranAt >= prior.at)
|
|
155
|
-
latestPerSuite.set(key, { at: record.ranAt, outcome: hit });
|
|
156
|
-
}
|
|
157
|
-
let worst;
|
|
158
|
-
for (const { outcome } of latestPerSuite.values()) {
|
|
159
|
-
// `skipped` means every test in the file was skipped — it was not exercised,
|
|
160
|
-
// and TestExecutionStatus.Skipped means a deliberate decision not to run it.
|
|
161
|
-
if (!outcome.complete || outcome.status === "skipped")
|
|
162
|
-
continue;
|
|
163
|
-
if (!worst || RANK[outcome.status] > RANK[worst.status])
|
|
164
|
-
worst = outcome;
|
|
165
|
-
}
|
|
166
|
-
return worst;
|
|
167
|
-
};
|
|
168
|
-
const before = statusIn("confirm");
|
|
169
|
-
const after = statusIn("verify");
|
|
170
|
-
return {
|
|
171
|
-
before: before && EXTERNAL_TO_EXECUTION_STATUS[before.status],
|
|
172
|
-
after: after && EXTERNAL_TO_EXECUTION_STATUS[after.status],
|
|
173
|
-
beforeDetail: before && describeOutcome(before),
|
|
174
|
-
afterDetail: after && describeOutcome(after),
|
|
175
|
-
};
|
|
176
|
-
}
|
|
177
87
|
const TOOL_NAME = "skyramp_submit_report";
|
|
178
88
|
const DEFAULT_COMMIT_MESSAGE = "Added recommendations by Skyramp Testbot.";
|
|
179
89
|
// Per-repo attribution. In a multi-repo run, every report item carries the
|
|
@@ -643,14 +553,14 @@ const testMaintenanceSchema = z.object({
|
|
|
643
553
|
"For passing runs: count and timing, e.g. '4 passed in 15.09s'. " +
|
|
644
554
|
"For failing runs: failure name and one-line root cause, e.g. " +
|
|
645
555
|
"'FAILED test_foo — assert 403 got 200, auth middleware not enforced'. " +
|
|
646
|
-
"
|
|
556
|
+
"Empty string for VERIFY/IGNORE entries where no before-execution was run."),
|
|
647
557
|
afterDetails: z
|
|
648
558
|
.string()
|
|
649
559
|
.describe("One line only — no embedded newlines, no raw HTTP headers or JSON blobs. " +
|
|
650
560
|
"For passing runs: count and timing, e.g. '5 passed in 10.96s'. " +
|
|
651
561
|
"For failing runs: failure name and one-line root cause, e.g. " +
|
|
652
562
|
"'FAILED test_foo — check_schema fails, order_id=1 has discount from prior PATCH test'. " +
|
|
653
|
-
"
|
|
563
|
+
"Empty string for VERIFY/IGNORE/DELETE entries where no after-execution was run."),
|
|
654
564
|
// Server-populated from stateFile execution records — never supplied by the LLM.
|
|
655
565
|
beforeStatus: z.nativeEnum(TestExecutionStatus),
|
|
656
566
|
afterStatus: z.nativeEnum(TestExecutionStatus),
|
|
@@ -1101,42 +1011,12 @@ export function registerSubmitReportTool(server) {
|
|
|
1101
1011
|
: MAINTENANCE_CHANGE_ACTIONS.has(m.action)
|
|
1102
1012
|
? TestExecutionStatus.Unknown
|
|
1103
1013
|
: TestExecutionStatus.Skipped;
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
// drops both from the rendered table.
|
|
1107
|
-
const external = externalStatusesFor(stateData.externalTestResults, m.testFilePath);
|
|
1108
|
-
const beforeStatus = recorded?.executionBefore?.status ??
|
|
1109
|
-
(defaultBeforeStatus === TestExecutionStatus.Unknown
|
|
1110
|
-
? external.before
|
|
1111
|
-
: undefined) ??
|
|
1112
|
-
defaultBeforeStatus;
|
|
1113
|
-
const afterStatus = recorded?.executionAfter?.status ??
|
|
1114
|
-
(defaultAfterStatus === TestExecutionStatus.Unknown
|
|
1115
|
-
? external.after
|
|
1116
|
-
: undefined) ??
|
|
1117
|
-
defaultAfterStatus;
|
|
1014
|
+
const beforeStatus = recorded?.executionBefore?.status ?? defaultBeforeStatus;
|
|
1015
|
+
const afterStatus = recorded?.executionAfter?.status ?? defaultAfterStatus;
|
|
1118
1016
|
// Trim before checking — a whitespace-only string is semantically blank and
|
|
1119
1017
|
// must not satisfy the "drafted a summary" requirement.
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
// never a bare verdict with nothing behind it.
|
|
1123
|
-
//
|
|
1124
|
-
// Only when there is NO recorded execution. A file the agent ran itself
|
|
1125
|
-
// owes a drafted summary, and filling it here would satisfy the check
|
|
1126
|
-
// below on the agent's behalf whenever the two statuses happened to
|
|
1127
|
-
// agree — silently dropping the requirement.
|
|
1128
|
-
//
|
|
1129
|
-
// And only where the STATUS came from the suite too. A VERIFY or DELETE
|
|
1130
|
-
// row keeps its Skipped default, so a line saying "1 passed in the repo's
|
|
1131
|
-
// own suite" beside it contradicts the very cell it annotates.
|
|
1132
|
-
const beforeFromSuite = defaultBeforeStatus === TestExecutionStatus.Unknown &&
|
|
1133
|
-
!recorded?.executionBefore;
|
|
1134
|
-
const afterFromSuite = defaultAfterStatus === TestExecutionStatus.Unknown &&
|
|
1135
|
-
!recorded?.executionAfter;
|
|
1136
|
-
const beforeDetails = detail?.beforeDetails?.trim() ||
|
|
1137
|
-
(beforeFromSuite ? (external.beforeDetail ?? "") : "");
|
|
1138
|
-
const afterDetails = detail?.afterDetails?.trim() ||
|
|
1139
|
-
(afterFromSuite ? (external.afterDetail ?? "") : "");
|
|
1018
|
+
const beforeDetails = detail?.beforeDetails?.trim() ?? "";
|
|
1019
|
+
const afterDetails = detail?.afterDetails?.trim() ?? "";
|
|
1140
1020
|
if (recorded?.executionBefore && !beforeDetails)
|
|
1141
1021
|
missingDetails.push(`${displayName} (beforeDetails)`);
|
|
1142
1022
|
if (recorded?.executionAfter && !afterDetails)
|
|
@@ -1,12 +1,68 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { WorkspaceConfigManager, serviceSchema, } from "../../workspace/workspace.js";
|
|
3
3
|
import fs from "fs/promises";
|
|
4
|
+
import fsSync from "fs";
|
|
5
|
+
import path from "path";
|
|
4
6
|
import yaml from "js-yaml";
|
|
5
7
|
import { logger } from "../../utils/logger.js";
|
|
6
8
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
9
|
import { SKYRAMP_IMAGE_VERSION } from "../../utils/versions.js";
|
|
8
10
|
import { validateAndConsumeScanToken } from "./initScanWorkspaceTool.js";
|
|
9
11
|
import { upsertServicesByRepo } from "./serviceUpsert.js";
|
|
12
|
+
import { isInsideDir } from "../../utils/gitStaging.js";
|
|
13
|
+
import { EXECUTOR_VIDEOS_DIR, EXECUTOR_WORK_DIR, isInsideExecutorWorkDir, } from "../../utils/executorWorkDir.js";
|
|
14
|
+
// Test delivery stages each service's whole testDirectory with
|
|
15
|
+
// `git add -- <testDirectory>`, so a testDirectory that encloses the executor's
|
|
16
|
+
// videos commits every .webm. This tool runs before any test is generated, so it
|
|
17
|
+
// is the only point where the choice can still be redirected.
|
|
18
|
+
/**
|
|
19
|
+
* First service whose `testDirectory` collides with the executor's working area,
|
|
20
|
+
* or undefined when none does. Two ways to collide, and both have to be rejected
|
|
21
|
+
* here: the directory sits inside `.skyramp` (which the generation tools refuse
|
|
22
|
+
* to write into, so accepting it would deadlock the agent between the two
|
|
23
|
+
* checks), or it encloses `.skyramp/videos` (`.`, "" or the repo root), which
|
|
24
|
+
* makes delivery commit every video. Relative paths resolve against
|
|
25
|
+
* `workspacePath`; an absolute one is matched on its own segments, so another
|
|
26
|
+
* repo's `.skyramp` is caught too.
|
|
27
|
+
*
|
|
28
|
+
* A directory that is itself a git repository root is rejected for the same
|
|
29
|
+
* reason, whichever repo it belongs to. The enclosure test above is anchored on
|
|
30
|
+
* the PRIMARY workspace's videos path, so a multi-repo entry naming another
|
|
31
|
+
* repo's root — `/work/repo-b` while workspacePath is `/work/repo-a` — slips
|
|
32
|
+
* past it: repo A's videos are not inside repo B. The executor writes
|
|
33
|
+
* `.skyramp/videos` at a repo root, so being one is the collision.
|
|
34
|
+
*/
|
|
35
|
+
function badTestDirectoryResult(bad) {
|
|
36
|
+
return {
|
|
37
|
+
content: [
|
|
38
|
+
{
|
|
39
|
+
type: "text",
|
|
40
|
+
text: `Service "${bad.serviceName}" sets testDirectory to "${bad.testDirectory}", which collides with ${EXECUTOR_WORK_DIR}, the executor's working area: a directory inside it is never delivered, and one that encloses ${EXECUTOR_VIDEOS_DIR}/ makes delivery commit every execution video. Use a dedicated test path — the service's framework-configured test directory, or tests/skyramp. If this service came from the existing ${EXECUTOR_WORK_DIR}/workspace.yml, re-run with force: true and the full corrected services array.`,
|
|
41
|
+
},
|
|
42
|
+
],
|
|
43
|
+
isError: true,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
/** Whether `dir` is the top of a git checkout. `.git` is a directory in a normal
|
|
47
|
+
* clone and a file in a worktree, so existence is the test. */
|
|
48
|
+
function isGitRepositoryRoot(dir) {
|
|
49
|
+
return fsSync.existsSync(path.join(dir, ".git"));
|
|
50
|
+
}
|
|
51
|
+
function findBadTestDirectory(workspacePath, services) {
|
|
52
|
+
const workspaceVideos = path.resolve(workspacePath, EXECUTOR_VIDEOS_DIR);
|
|
53
|
+
for (const svc of services) {
|
|
54
|
+
const dir = svc.testDirectory;
|
|
55
|
+
if (dir === undefined)
|
|
56
|
+
continue;
|
|
57
|
+
const resolved = path.resolve(workspacePath, dir);
|
|
58
|
+
if (isInsideExecutorWorkDir(resolved) ||
|
|
59
|
+
isInsideDir(workspaceVideos, resolved) ||
|
|
60
|
+
isGitRepositoryRoot(resolved)) {
|
|
61
|
+
return { serviceName: svc.serviceName, testDirectory: dir };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
10
66
|
function getExecutorVersion() {
|
|
11
67
|
return SKYRAMP_IMAGE_VERSION;
|
|
12
68
|
}
|
|
@@ -92,6 +148,40 @@ export function registerInitializeWorkspaceTool(server) {
|
|
|
92
148
|
isError: false,
|
|
93
149
|
};
|
|
94
150
|
}
|
|
151
|
+
if (!params.services || params.services.length === 0) {
|
|
152
|
+
return {
|
|
153
|
+
content: [
|
|
154
|
+
{
|
|
155
|
+
type: "text",
|
|
156
|
+
text: "No services provided. Follow the scanning instructions from skyramp_init_scan to discover services, then call this tool again.",
|
|
157
|
+
},
|
|
158
|
+
],
|
|
159
|
+
isError: true,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
// Both checks above and below are input-only, so they run before
|
|
163
|
+
// validateAndConsumeScanToken: that
|
|
164
|
+
// token is single-use, and a rejection after it would make the corrected
|
|
165
|
+
// retry this error asks for fail with "Invalid or expired scanToken". It
|
|
166
|
+
// is also before initialize()/updateMetadata(), which both write
|
|
167
|
+
// workspace.yml (workspace.ts:431, :460) — a later rejection would leave
|
|
168
|
+
// an initialized file behind and the retry would then short-circuit as
|
|
169
|
+
// "already initialized".
|
|
170
|
+
const badIncoming = findBadTestDirectory(workspacePath, params.services);
|
|
171
|
+
if (badIncoming)
|
|
172
|
+
return badTestDirectoryResult(badIncoming);
|
|
173
|
+
// Merge mode writes the MERGED set, so a bad testDirectory already in the
|
|
174
|
+
// file would be re-persisted. Checked on the merge, not on the existing
|
|
175
|
+
// services alone, so replacing a bad entry with a corrected one of the
|
|
176
|
+
// same identity is still the way out. Reading the file is side-effect
|
|
177
|
+
// free, so this rejection belongs up here with the other two rather than
|
|
178
|
+
// after the token is spent.
|
|
179
|
+
const mergeInto = params.merge && alreadyExists ? await manager.read() : undefined;
|
|
180
|
+
if (mergeInto) {
|
|
181
|
+
const badMerged = findBadTestDirectory(workspacePath, upsertServicesByRepo([...(mergeInto.services ?? [])], params.services));
|
|
182
|
+
if (badMerged)
|
|
183
|
+
return badTestDirectoryResult(badMerged);
|
|
184
|
+
}
|
|
95
185
|
// scanToken is required for fresh init; for edits (force:true on an
|
|
96
186
|
// existing workspace.yml) the caller supplies the full services array
|
|
97
187
|
// directly and we skip the token check.
|
|
@@ -120,33 +210,15 @@ export function registerInitializeWorkspaceTool(server) {
|
|
|
120
210
|
};
|
|
121
211
|
}
|
|
122
212
|
}
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
// Initialize (fresh) or read (merge into existing). Merge mode preserves
|
|
135
|
-
// the existing primary repo identity + previously-registered services;
|
|
136
|
-
// initialize() would reset both, dropping other repos' services.
|
|
137
|
-
let config;
|
|
138
|
-
if (params.merge && alreadyExists) {
|
|
139
|
-
config = await manager.read();
|
|
140
|
-
config = await manager.updateMetadata({
|
|
141
|
-
executorVersion: getExecutorVersion(),
|
|
142
|
-
});
|
|
143
|
-
}
|
|
144
|
-
else {
|
|
145
|
-
config = await manager.initialize();
|
|
146
|
-
config = await manager.updateMetadata({
|
|
147
|
-
executorVersion: getExecutorVersion(),
|
|
148
|
-
});
|
|
149
|
-
}
|
|
213
|
+
// Merge mode skips initialize(): it preserves the existing primary repo
|
|
214
|
+
// identity + previously-registered services, both of which initialize()
|
|
215
|
+
// would reset, dropping other repos' services. The file itself was
|
|
216
|
+
// already read above, for the merged-set check.
|
|
217
|
+
if (!mergeInto)
|
|
218
|
+
await manager.initialize();
|
|
219
|
+
const config = await manager.updateMetadata({
|
|
220
|
+
executorVersion: getExecutorVersion(),
|
|
221
|
+
});
|
|
150
222
|
// Batch all service upserts: update in memory, write once (direct YAML
|
|
151
223
|
// write avoids N+1 reads and a mid-loop validation error). Upsert is keyed
|
|
152
224
|
// on the composite (repo, serviceName) — see upsertServicesByRepo — so a
|
|
@@ -11,15 +11,8 @@ export type ExternalTestStatus = "pass" | "fail" | "error" | "skipped";
|
|
|
11
11
|
export interface ExternalTestResult {
|
|
12
12
|
/** Human-readable identity, e.g. `<file> › <describe…> › <title>`. */
|
|
13
13
|
testId: string;
|
|
14
|
-
/** Spec file the test lives in
|
|
15
|
-
* for jest and mocha, relative to the framework's root for Playwright and pytest.
|
|
16
|
-
* No framework guarantees which, so never assume: use `absoluteFile` when present. */
|
|
14
|
+
/** Spec file the test lives in. */
|
|
17
15
|
file: string;
|
|
18
|
-
/** `file` resolved against the root the framework reported, when it reported one.
|
|
19
|
-
* A relative spelling alone cannot identify a file: `tests/Button.test.tsx` fits
|
|
20
|
-
* every package that has one. Consumers matching a result to a known path must
|
|
21
|
-
* prefer this and compare exactly. */
|
|
22
|
-
absoluteFile?: string;
|
|
23
16
|
status: ExternalTestStatus;
|
|
24
17
|
/** Failure/error text (ANSI-stripped). Present only for fail/error. */
|
|
25
18
|
message?: string;
|
|
@@ -62,69 +55,10 @@ export interface ParsedExternalRun {
|
|
|
62
55
|
/** Why the run was skipped (present only when `skipped`). */
|
|
63
56
|
skipReason?: string;
|
|
64
57
|
}
|
|
65
|
-
/**
|
|
66
|
-
* What one run established about ONE spec file. Computed by the run itself, which
|
|
67
|
-
* knows what it asked the framework for; the report then reads this instead of
|
|
68
|
-
* re-deriving coverage from the flat result list, where the facts below are no
|
|
69
|
-
* longer recoverable.
|
|
70
|
-
*/
|
|
71
|
-
export interface ExternalFileOutcome {
|
|
72
|
-
/** The framework's own spelling, kept for diagnostics. */
|
|
73
|
-
file: string;
|
|
74
|
-
/** `file` resolved against the root the framework reported, when it reported one.
|
|
75
|
-
* Present here exactly when it was present on the results. */
|
|
76
|
-
absoluteFile?: string;
|
|
77
|
-
/** Worst outcome across the file's tests: error > fail > pass. `skipped` means
|
|
78
|
-
* every test in it was skipped, so the file was not exercised at all. */
|
|
79
|
-
status: ExternalTestStatus;
|
|
80
|
-
/** How many tests produced each outcome. Rendered into the maintenance row's
|
|
81
|
-
* before/after detail when the agent drafted none. */
|
|
82
|
-
counts: {
|
|
83
|
-
pass: number;
|
|
84
|
-
fail: number;
|
|
85
|
-
error: number;
|
|
86
|
-
skipped: number;
|
|
87
|
-
};
|
|
88
|
-
/**
|
|
89
|
-
* Did this run exercise every TEST in the file? False when a `file::test title`
|
|
90
|
-
* selector narrowed it — the one narrowing this tool applies per file, and so the
|
|
91
|
-
* only one it can state with certainty. A partial run says nothing about the tests
|
|
92
|
-
* it did not reach, so it cannot stand as the file's status.
|
|
93
|
-
*
|
|
94
|
-
* Two narrowings are deliberately NOT reflected here, and the reader compensates
|
|
95
|
-
* for one of them:
|
|
96
|
-
* - A `file::title` selector also becomes a run-WIDE --grep / -t on every
|
|
97
|
-
* framework except pytest, so a co-selected file ran only its matching tests
|
|
98
|
-
* while this says complete. Known and accepted.
|
|
99
|
-
* - A suite that bailed on --max-failures abandoned tests it cannot name. That one
|
|
100
|
-
* IS reported, in `diagnostics.maxFailuresBail`, and the report skips such a
|
|
101
|
-
* record outright rather than reading any file from it.
|
|
102
|
-
*
|
|
103
|
-
* A Playwright `project` scope does NOT make a file incomplete: every test in it
|
|
104
|
-
* still ran, in one project rather than all of them. That is a different axis, and
|
|
105
|
-
* discarding the status over it throws away the run's whole verdict — measured on
|
|
106
|
-
* eval run 33280302496, where it turned a real Error -> Pass into Unknown.
|
|
107
|
-
*
|
|
108
|
-
* Known gap: a repo whose runner bails on its own (a jest or mocha `bail` in
|
|
109
|
-
* config rather than on the command line) is not detectable from the report,
|
|
110
|
-
* and only Playwright reports a --max-failures bail back to us.
|
|
111
|
-
*/
|
|
112
|
-
complete: boolean;
|
|
113
|
-
/** Failure/error text for the tests that did not pass. */
|
|
114
|
-
errors: string[];
|
|
115
|
-
durationMs: number;
|
|
116
|
-
}
|
|
117
58
|
/** One persisted run of the external suite, appended to
|
|
118
59
|
* UnifiedAnalysisState.externalTestResults so drift analysis / the report can
|
|
119
60
|
* fold in CONFIRMED failures instead of guesses. */
|
|
120
61
|
export interface ExternalTestRunRecord extends ParsedExternalRun {
|
|
121
|
-
/** Per-file outcome, one entry per spec file this run produced results for.
|
|
122
|
-
* Empty for a skipped run. */
|
|
123
|
-
files: ExternalFileOutcome[];
|
|
124
|
-
/** Which configured suite produced this record. A repo can declare several, and
|
|
125
|
-
* without this a re-run of ONE suite cannot be told from a second suite's first
|
|
126
|
-
* run — so a superseded failure would outlive the fix that repaired it. */
|
|
127
|
-
suite: string;
|
|
128
62
|
mode: "confirm" | "verify";
|
|
129
63
|
/** ISO timestamp when the run completed. */
|
|
130
64
|
ranAt: string;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The executor's own working area inside a repo — run videos, trace zips and
|
|
3
|
+
* other per-run artefacts. Delivered test files do not belong here: the run
|
|
4
|
+
* cleans and rewrites it, and `git add -- <testDirectory>` on a directory that
|
|
5
|
+
* encloses it commits every .webm.
|
|
6
|
+
*/
|
|
7
|
+
export declare const EXECUTOR_WORK_DIR = ".skyramp";
|
|
8
|
+
export declare const EXECUTOR_VIDEOS_DIR = ".skyramp/videos";
|
|
9
|
+
/**
|
|
10
|
+
* Whether `p` is the executor's working area or sits inside it. Matches on whole
|
|
11
|
+
* path segments, so a sibling such as `.skyramp-tests` is not a hit, and on the
|
|
12
|
+
* canonical path, so a symlink pointing into `.skyramp` cannot spell its way past
|
|
13
|
+
* the check.
|
|
14
|
+
*/
|
|
15
|
+
export declare function isInsideExecutorWorkDir(p: string): boolean;
|
|
16
|
+
/**
|
|
17
|
+
* Refusal text for a caller that asked for output inside the working area. Names
|
|
18
|
+
* the field and the replacement so the agent can correct itself without help.
|
|
19
|
+
*/
|
|
20
|
+
export declare function executorWorkDirRefusal(destination: string): string;
|
|
21
|
+
/**
|
|
22
|
+
* Every path a generation call could write to, given an outputDir and an optional
|
|
23
|
+
* caller-supplied file name. `output` can carry `../` segments, so outputDir alone
|
|
24
|
+
* is not enough to judge.
|
|
25
|
+
*
|
|
26
|
+
* Two compositions, because the writers disagree on an ABSOLUTE `output`:
|
|
27
|
+
* `generateEnrichedIntegrationTestTool` joins it (`path.join(outputDir, output)`,
|
|
28
|
+
* which keeps outputDir as a prefix), while `path.resolve` discards outputDir
|
|
29
|
+
* entirely. A guard cannot pick one and be right for both, so it judges both and
|
|
30
|
+
* refuses if either lands in the working area.
|
|
31
|
+
*
|
|
32
|
+
* outputDir is always judged on its own as well. An `output` of `../tests/x.py`
|
|
33
|
+
* resolves clear of the working area while outputDir stays `.skyramp`, and
|
|
34
|
+
* outputDir is what codegen receives as the directory to work in.
|
|
35
|
+
*/
|
|
36
|
+
export declare function generationTargets(outputDir: string, output?: string): string[];
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
/**
|
|
4
|
+
* The executor's own working area inside a repo — run videos, trace zips and
|
|
5
|
+
* other per-run artefacts. Delivered test files do not belong here: the run
|
|
6
|
+
* cleans and rewrites it, and `git add -- <testDirectory>` on a directory that
|
|
7
|
+
* encloses it commits every .webm.
|
|
8
|
+
*/
|
|
9
|
+
export const EXECUTOR_WORK_DIR = ".skyramp";
|
|
10
|
+
export const EXECUTOR_VIDEOS_DIR = `${EXECUTOR_WORK_DIR}/videos`;
|
|
11
|
+
/**
|
|
12
|
+
* Canonical form of `p`, where `p` may not exist yet: the deepest ancestor that
|
|
13
|
+
* DOES exist is resolved through symlinks and the missing remainder is appended
|
|
14
|
+
* back.
|
|
15
|
+
*
|
|
16
|
+
* `utils-verify/locate.ts` (realpath) and `code-refactor/reuse-state.ts` (canon)
|
|
17
|
+
* both fall back to the lexical path when the target is missing. A generation
|
|
18
|
+
* outputDir usually IS missing, and the lexical path is what a symlinked parent
|
|
19
|
+
* hides behind — so neither is usable here.
|
|
20
|
+
*/
|
|
21
|
+
function realpathAllowingMissing(p) {
|
|
22
|
+
let current = path.resolve(p);
|
|
23
|
+
const missing = [];
|
|
24
|
+
for (;;) {
|
|
25
|
+
try {
|
|
26
|
+
return path.join(fs.realpathSync(current), ...[...missing].reverse());
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
const parent = path.dirname(current);
|
|
30
|
+
if (parent === current)
|
|
31
|
+
return path.resolve(p);
|
|
32
|
+
missing.push(path.basename(current));
|
|
33
|
+
current = parent;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Whether `p` is the executor's working area or sits inside it. Matches on whole
|
|
39
|
+
* path segments, so a sibling such as `.skyramp-tests` is not a hit, and on the
|
|
40
|
+
* canonical path, so a symlink pointing into `.skyramp` cannot spell its way past
|
|
41
|
+
* the check.
|
|
42
|
+
*/
|
|
43
|
+
export function isInsideExecutorWorkDir(p) {
|
|
44
|
+
return realpathAllowingMissing(p).split(path.sep).includes(EXECUTOR_WORK_DIR);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Refusal text for a caller that asked for output inside the working area. Names
|
|
48
|
+
* the field and the replacement so the agent can correct itself without help.
|
|
49
|
+
*/
|
|
50
|
+
export function executorWorkDirRefusal(destination) {
|
|
51
|
+
return `Refusing to write into "${destination}": it is inside ${EXECUTOR_WORK_DIR}, the Skyramp executor's working area (run videos and trace zips). Files written there are not delivered as tests. Set outputDir — and any output file name — so the result lands in the testDirectory of the service under test, read from ${EXECUTOR_WORK_DIR}/workspace.yml, or tests/skyramp when that service declares none. If that testDirectory is itself inside ${EXECUTOR_WORK_DIR}, it is wrong: re-run skyramp_init_workspace with a corrected one.`;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Every path a generation call could write to, given an outputDir and an optional
|
|
55
|
+
* caller-supplied file name. `output` can carry `../` segments, so outputDir alone
|
|
56
|
+
* is not enough to judge.
|
|
57
|
+
*
|
|
58
|
+
* Two compositions, because the writers disagree on an ABSOLUTE `output`:
|
|
59
|
+
* `generateEnrichedIntegrationTestTool` joins it (`path.join(outputDir, output)`,
|
|
60
|
+
* which keeps outputDir as a prefix), while `path.resolve` discards outputDir
|
|
61
|
+
* entirely. A guard cannot pick one and be right for both, so it judges both and
|
|
62
|
+
* refuses if either lands in the working area.
|
|
63
|
+
*
|
|
64
|
+
* outputDir is always judged on its own as well. An `output` of `../tests/x.py`
|
|
65
|
+
* resolves clear of the working area while outputDir stays `.skyramp`, and
|
|
66
|
+
* outputDir is what codegen receives as the directory to work in.
|
|
67
|
+
*/
|
|
68
|
+
export function generationTargets(outputDir, output) {
|
|
69
|
+
const dir = path.resolve(outputDir);
|
|
70
|
+
if (!output)
|
|
71
|
+
return [dir];
|
|
72
|
+
return [
|
|
73
|
+
dir,
|
|
74
|
+
path.resolve(outputDir, output),
|
|
75
|
+
path.resolve(path.join(outputDir, output)),
|
|
76
|
+
];
|
|
77
|
+
}
|