@skyramp/mcp 0.3.3 → 0.3.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +42 -1
- package/build/prompts/enhance-assertions/sharedAssertionRules.js +19 -0
- package/build/prompts/test-maintenance/actionsInstructions.js +2 -2
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -4
- package/build/prompts/test-recommendation/recommendationSections.d.ts +1 -1
- package/build/prompts/test-recommendation/recommendationSections.js +5 -5
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +13 -7
- package/build/prompts/testbot/testbot-prompts.js +16 -14
- package/build/recommendation/discriminators.d.ts +7 -1
- package/build/recommendation/discriminators.js +16 -3
- package/build/resources/testbotResource.js +0 -1
- package/build/services/ScenarioGenerationService.js +5 -2
- package/build/services/TestExecutionService.js +25 -1
- package/build/services/TestGenerationService.js +24 -9
- package/build/services/containerEnv.d.ts +12 -1
- package/build/services/containerEnv.js +94 -1
- package/build/tools/executeSkyrampTestTool.d.ts +9 -0
- package/build/tools/executeSkyrampTestTool.js +20 -6
- package/build/tools/execution-video-state.d.ts +21 -0
- package/build/tools/execution-video-state.js +51 -0
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +31 -11
- package/build/tools/generate-tests/planGuard.d.ts +5 -5
- package/build/tools/generate-tests/planGuard.js +5 -17
- package/build/tools/submitReportTool.d.ts +83 -10
- package/build/tools/submitReportTool.js +257 -30
- package/build/tools/test-management/actionsTool.js +40 -39
- package/build/tools/test-management/analyzeChangesTool.d.ts +11 -0
- package/build/tools/test-management/analyzeChangesTool.js +41 -35
- package/build/tools/test-management/analyzeTestHealthTool.js +3 -3
- package/build/tools/test-management/registerTestPlanTool.js +191 -41
- package/build/types/TestExecution.d.ts +14 -0
- package/build/types/TestTypes.js +3 -2
- package/build/types/TestbotPromptOptions.d.ts +0 -1
- package/build/types/TestbotReport.d.ts +24 -1
- package/build/types/TestbotReport.js +10 -1
- package/build/types/index.d.ts +1 -0
- package/build/types/index.js +1 -0
- package/build/utils/AnalysisStateManager.d.ts +48 -2
- package/build/utils/AnalysisStateManager.js +49 -13
- package/build/utils/reportLanguage.d.ts +43 -0
- package/build/utils/reportLanguage.js +125 -0
- package/build/utils/reportVerification.d.ts +10 -0
- package/build/utils/reportVerification.js +31 -0
- package/build/utils/scenarioDrafting.js +7 -1
- package/build/utils/skyrampMdContent.d.ts +1 -1
- package/build/utils/skyrampMdContent.js +1 -1
- package/build/utils/urlPath.d.ts +37 -0
- package/build/utils/urlPath.js +55 -0
- package/build/utils/utils.d.ts +45 -0
- package/build/utils/utils.js +50 -0
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/utils/workspaceAuth.d.ts +15 -15
- package/build/utils/workspaceAuth.js +32 -17
- package/build/workspace/queryParamResolution.d.ts +93 -0
- package/build/workspace/queryParamResolution.js +201 -0
- package/build/workspace/workspace.d.ts +104 -0
- package/build/workspace/workspace.js +24 -0
- package/package.json +3 -2
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { logger } from "../utils/logger.js";
|
|
1
2
|
/**
|
|
2
3
|
* Rewrite localhost / 127.0.0.1 URLs to host.docker.internal so the Docker
|
|
3
4
|
* executor container can reach services running on the host machine.
|
|
@@ -9,10 +10,67 @@
|
|
|
9
10
|
export function rewriteLocalhostForDocker(url) {
|
|
10
11
|
return url.replace(/^(https?:\/\/)(localhost|127\.0\.0\.1)([:/?#]|$)/, (_, scheme, _host, rest) => `${scheme}host.docker.internal${rest}`);
|
|
11
12
|
}
|
|
13
|
+
/** Names a workspace declaration may never forward. Matching is
|
|
14
|
+
* case-insensitive: `process.env` is case-insensitive on Windows, so a
|
|
15
|
+
* lower-case declaration there returns the real value and would otherwise
|
|
16
|
+
* pass the checks below.
|
|
17
|
+
*
|
|
18
|
+
* Two distinct concerns share one list. The first is CI identity: testbot puts
|
|
19
|
+
* the workflow's GitHub token in the MCP process env so the agent's `gh` calls
|
|
20
|
+
* authenticate, and a declaration naming it would push it into the container
|
|
21
|
+
* and from there into a query string — where the SUT's own access log records
|
|
22
|
+
* it. The second is internal signals the executor sets for itself, which a
|
|
23
|
+
* declaration must not be able to spoof or override.
|
|
24
|
+
*
|
|
25
|
+
* Deliberately NOT a blanket `*_TOKEN`: a token for the customer's own SUT is
|
|
26
|
+
* exactly the kind of value an RBAC query param carries, and refusing it would
|
|
27
|
+
* break the feature this list exists to protect. */
|
|
28
|
+
// Names the executor sets for itself. API_KEY carries the Skyramp LLM key and
|
|
29
|
+
// is forwarded unconditionally by the built-in block below, so a declaration
|
|
30
|
+
// adds nothing. UI Smart LLM features keep working either way.
|
|
31
|
+
const PASSTHROUGH_RESERVED_EXACT = new Set([
|
|
32
|
+
"API_KEY",
|
|
33
|
+
"PYTEST_ADDOPTS",
|
|
34
|
+
"PLAYWRIGHT_SAVE_STORAGE_PATH",
|
|
35
|
+
]);
|
|
36
|
+
const PASSTHROUGH_RESERVED_PREFIXES = ["SKYRAMP_"];
|
|
37
|
+
// CI identity and secret-shaped names. Testbot puts the workflow GitHub token
|
|
38
|
+
// in the MCP process env, and a declaration would send it to the container and
|
|
39
|
+
// into a query string, where the SUT access log records it.
|
|
40
|
+
const PASSTHROUGH_CREDENTIAL_EXACT = new Set(["GITHUB_TOKEN", "GH_TOKEN"]);
|
|
41
|
+
const PASSTHROUGH_CREDENTIAL_PREFIXES = ["ACTIONS_", "AWS_"];
|
|
42
|
+
const PASSTHROUGH_CREDENTIAL_SUBSTRINGS = ["_SECRET", "_PRIVATE_KEY"];
|
|
43
|
+
/** Why a name is refused, or null when it is allowed. The two reasons need
|
|
44
|
+
* different messages: a reserved name is already provided by the executor, so
|
|
45
|
+
* the declaration is only redundant, while a credential name is refused. */
|
|
46
|
+
export function passthroughDenyReason(name) {
|
|
47
|
+
const upper = name.toUpperCase();
|
|
48
|
+
if (PASSTHROUGH_CREDENTIAL_EXACT.has(upper) ||
|
|
49
|
+
PASSTHROUGH_CREDENTIAL_PREFIXES.some((p) => upper.startsWith(p)) ||
|
|
50
|
+
PASSTHROUGH_CREDENTIAL_SUBSTRINGS.some((s) => upper.includes(s))) {
|
|
51
|
+
return "credential";
|
|
52
|
+
}
|
|
53
|
+
if (PASSTHROUGH_RESERVED_EXACT.has(upper) ||
|
|
54
|
+
PASSTHROUGH_RESERVED_PREFIXES.some((p) => upper.startsWith(p))) {
|
|
55
|
+
return "reserved";
|
|
56
|
+
}
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
/** True when a name is refused regardless of who declared it. */
|
|
60
|
+
export function isDeniedPassthroughName(name) {
|
|
61
|
+
return passthroughDenyReason(name) !== null;
|
|
62
|
+
}
|
|
63
|
+
/** A name is interpolated directly into a Docker `Env` entry, which is a plain
|
|
64
|
+
* `NAME=value` string. A name holding `=`, whitespace or a newline produces a
|
|
65
|
+
* malformed entry and can inject a second variable. workspace.yml supplies
|
|
66
|
+
* these names, so `env.FOO=BAR` would yield the name `FOO=BAR`. */
|
|
67
|
+
export function isValidEnvVarName(name) {
|
|
68
|
+
return /^[A-Za-z_][A-Za-z0-9_]*$/.test(name);
|
|
69
|
+
}
|
|
12
70
|
/**
|
|
13
71
|
* Build the environment variable array for the Docker executor container.
|
|
14
72
|
*/
|
|
15
|
-
export function buildContainerEnv(options, saveStoragePath, hostEnv = process.env) {
|
|
73
|
+
export function buildContainerEnv(options, saveStoragePath, hostEnv = process.env, passthroughNames = []) {
|
|
16
74
|
const env = [
|
|
17
75
|
// Omit entirely when empty so os.getenv() returns None in the test —
|
|
18
76
|
// unauthenticated endpoints won't send an empty auth header (E7).
|
|
@@ -68,6 +126,41 @@ export function buildContainerEnv(options, saveStoragePath, hostEnv = process.en
|
|
|
68
126
|
env.push(`${key}=${val}`);
|
|
69
127
|
}
|
|
70
128
|
}
|
|
129
|
+
// Environment variables the workspace's query params declare via `env.VAR`
|
|
130
|
+
// (SKYR-4127). These references resolve where the test RUNS — inside this
|
|
131
|
+
// container — so a name that does not reach it resolves to nothing, and an
|
|
132
|
+
// unresolved query param is dropped from the request silently, yielding a
|
|
133
|
+
// passing test that exercised the wrong identity.
|
|
134
|
+
//
|
|
135
|
+
// The list comes from what workspace.yml declared, never from enumerating the
|
|
136
|
+
// ambient environment: nothing is forwarded that the workspace did not name.
|
|
137
|
+
// Forwarded on presence, not truthiness, for the same reason as the
|
|
138
|
+
// credentials above — an empty value is a legitimate declaration.
|
|
139
|
+
const seen = new Set();
|
|
140
|
+
for (const name of passthroughNames) {
|
|
141
|
+
if (seen.has(name))
|
|
142
|
+
continue;
|
|
143
|
+
seen.add(name);
|
|
144
|
+
if (!isValidEnvVarName(name)) {
|
|
145
|
+
logger.warning(`Ignoring ${JSON.stringify(name)} declared in workspace.yml: it is not a valid environment variable name.`);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const denyReason = passthroughDenyReason(name);
|
|
149
|
+
if (denyReason === "reserved") {
|
|
150
|
+
logger.warning(`Ignoring the workspace.yml declaration of ${name}: the executor sets this variable itself. ` +
|
|
151
|
+
`Its value still reaches the test.`);
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
if (denyReason === "credential") {
|
|
155
|
+
logger.warning(`Refusing to forward ${name} to the test executor: the name is credential-shaped. ` +
|
|
156
|
+
`Declare a variable that carries only the value the request needs.`);
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
const val = hostEnv[name];
|
|
160
|
+
if (val !== undefined) {
|
|
161
|
+
env.push(`${name}=${val}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
71
164
|
if (hostEnv.SKYRAMP_DEBUG) {
|
|
72
165
|
env.push(`SKYRAMP_DEBUG=${hostEnv.SKYRAMP_DEBUG}`);
|
|
73
166
|
}
|
|
@@ -10,4 +10,13 @@ export type ContractExecutionMode = (typeof CONTRACT_EXECUTION_MODES)[number];
|
|
|
10
10
|
*/
|
|
11
11
|
export declare function resolveEffectiveToken(unauthenticated?: boolean, paramToken?: string, envToken?: string): string;
|
|
12
12
|
export declare function shouldInjectSkyrampBaseUrl(testType: TestType, contractMode?: ContractExecutionMode): boolean;
|
|
13
|
+
/**
|
|
14
|
+
* Append the recorded video path to execution output.
|
|
15
|
+
*
|
|
16
|
+
* Applied to the failure return as well as the success one (SKYR-4156): a video is
|
|
17
|
+
* most useful for a test that just failed, and surfacing it only on success meant a
|
|
18
|
+
* failing UI test reported no recording at all unless Playwright happened to print
|
|
19
|
+
* its own attachment line.
|
|
20
|
+
*/
|
|
21
|
+
export declare function withVideoInfo(output: string, videoPath?: string): string;
|
|
13
22
|
export declare function registerExecuteSkyrampTestTool(server: McpServer): void;
|
|
@@ -11,6 +11,7 @@ import { StateManager, getTestsRepoDir } from "../utils/AnalysisStateManager.js"
|
|
|
11
11
|
import { TestSource } from "../types/TestAnalysis.js";
|
|
12
12
|
import { logger } from "../utils/logger.js";
|
|
13
13
|
import { toolError } from "../utils/utils.js";
|
|
14
|
+
import { recordExecutionVideo } from "./execution-video-state.js";
|
|
14
15
|
const TOOL_NAME = "skyramp_execute_test";
|
|
15
16
|
export const CONTRACT_EXECUTION_MODES = ["provider", "consumer"];
|
|
16
17
|
/**
|
|
@@ -27,6 +28,17 @@ export function resolveEffectiveToken(unauthenticated, paramToken, envToken) {
|
|
|
27
28
|
export function shouldInjectSkyrampBaseUrl(testType, contractMode) {
|
|
28
29
|
return testType !== TestType.CONTRACT || contractMode !== "consumer";
|
|
29
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Append the recorded video path to execution output.
|
|
33
|
+
*
|
|
34
|
+
* Applied to the failure return as well as the success one (SKYR-4156): a video is
|
|
35
|
+
* most useful for a test that just failed, and surfacing it only on success meant a
|
|
36
|
+
* failing UI test reported no recording at all unless Playwright happened to print
|
|
37
|
+
* its own attachment line.
|
|
38
|
+
*/
|
|
39
|
+
export function withVideoInfo(output, videoPath) {
|
|
40
|
+
return videoPath ? `${output}\n\nVideo recording: ${videoPath}` : output;
|
|
41
|
+
}
|
|
30
42
|
export function registerExecuteSkyrampTestTool(server) {
|
|
31
43
|
server.registerTool(TOOL_NAME, {
|
|
32
44
|
description: `Execute a Skyramp-generated test in isolated containerized environments for reliable, deterministic testing. Call this once a test file exists on disk (from a skyramp_*_test_generation tool). First-time execution may take longer while Docker images download — this is expected, not a failure.`,
|
|
@@ -224,23 +236,25 @@ export function registerExecuteSkyrampTestTool(server) {
|
|
|
224
236
|
// Don't fail the tool call if stateFile update fails
|
|
225
237
|
}
|
|
226
238
|
}
|
|
239
|
+
// Record the recording for the report before returning, so it happens on the
|
|
240
|
+
// failure path too (SKYR-4156). skyramp_submit_report reads these records to
|
|
241
|
+
// populate testResults[].videoPath — testbot uploads only the video
|
|
242
|
+
// directories the report references, so an unrecorded video is never seen.
|
|
243
|
+
await recordExecutionVideo(result, params.stateFile);
|
|
227
244
|
// Progress is already reported by TestExecutionService
|
|
228
245
|
// Only report final status if not already at 100%
|
|
229
246
|
if (result.status !== TestExecutionStatus.Pass) {
|
|
230
|
-
errorResult = toolError(`Test execution failed: ${stripVTControlCharacters(result.output || "")}
|
|
247
|
+
errorResult = toolError(withVideoInfo(`Test execution failed: ${stripVTControlCharacters(result.output || "")}
|
|
231
248
|
|
|
232
|
-
If this failed with a 401 status code, ask the user for the authentication token (or where to read it from) and retry — leave the generated test file unchanged
|
|
249
|
+
If this failed with a 401 status code, ask the user for the authentication token (or where to read it from) and retry — leave the generated test file unchanged.`, result.videoPath));
|
|
233
250
|
return errorResult;
|
|
234
251
|
}
|
|
235
252
|
// Success - progress already reported by TestExecutionService
|
|
236
|
-
const videoInfo = result.videoPath
|
|
237
|
-
? `\n\nVideo recording: ${result.videoPath}`
|
|
238
|
-
: "";
|
|
239
253
|
return {
|
|
240
254
|
content: [
|
|
241
255
|
{
|
|
242
256
|
type: "text",
|
|
243
|
-
text: `Test execution result: ${stripVTControlCharacters(result.output || "")}
|
|
257
|
+
text: withVideoInfo(`Test execution result: ${stripVTControlCharacters(result.output || "")}`, result.videoPath),
|
|
244
258
|
},
|
|
245
259
|
],
|
|
246
260
|
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { TestExecutionResult } from "../types/TestExecution.js";
|
|
2
|
+
/**
|
|
3
|
+
* Persistence for the server-derived video recording of a browser test execution,
|
|
4
|
+
* keyed by test-file basename (what `newTestsCreated[].fileName` carries, so
|
|
5
|
+
* `skyramp_submit_report` can match without path normalization).
|
|
6
|
+
*
|
|
7
|
+
* Why this exists (SKYR-4156): testbot uploads only the `.skyramp/videos`
|
|
8
|
+
* directories the submitted report actually references, so a recording nobody
|
|
9
|
+
* references is a recording nobody ever sees. The path is known only inside
|
|
10
|
+
* `skyramp_execute_test`, and asking the agent to carry it across tool calls left
|
|
11
|
+
* the signal prose-mediated — it decayed to zero referenced videos, and every
|
|
12
|
+
* recording of the run was discarded. Recording it here makes the reference
|
|
13
|
+
* server-derived, the same line already drawn for POM reuse and maintenance
|
|
14
|
+
* before/after status.
|
|
15
|
+
*
|
|
16
|
+
* Every write is best-effort. Video observability must never fail an execution
|
|
17
|
+
* that otherwise succeeded, so a missing/unreadable/unwritable state file degrades
|
|
18
|
+
* to "no video link in the report" — the same outcome as running outside a Testbot
|
|
19
|
+
* run, where there is no run state to record into.
|
|
20
|
+
*/
|
|
21
|
+
export declare function recordExecutionVideo(result: TestExecutionResult, explicitStateFile?: string): Promise<void>;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import * as path from "path";
|
|
2
|
+
import { logger } from "../utils/logger.js";
|
|
3
|
+
import { resolveRunStatePath, StateManager, } from "../utils/AnalysisStateManager.js";
|
|
4
|
+
/**
|
|
5
|
+
* Persistence for the server-derived video recording of a browser test execution,
|
|
6
|
+
* keyed by test-file basename (what `newTestsCreated[].fileName` carries, so
|
|
7
|
+
* `skyramp_submit_report` can match without path normalization).
|
|
8
|
+
*
|
|
9
|
+
* Why this exists (SKYR-4156): testbot uploads only the `.skyramp/videos`
|
|
10
|
+
* directories the submitted report actually references, so a recording nobody
|
|
11
|
+
* references is a recording nobody ever sees. The path is known only inside
|
|
12
|
+
* `skyramp_execute_test`, and asking the agent to carry it across tool calls left
|
|
13
|
+
* the signal prose-mediated — it decayed to zero referenced videos, and every
|
|
14
|
+
* recording of the run was discarded. Recording it here makes the reference
|
|
15
|
+
* server-derived, the same line already drawn for POM reuse and maintenance
|
|
16
|
+
* before/after status.
|
|
17
|
+
*
|
|
18
|
+
* Every write is best-effort. Video observability must never fail an execution
|
|
19
|
+
* that otherwise succeeded, so a missing/unreadable/unwritable state file degrades
|
|
20
|
+
* to "no video link in the report" — the same outcome as running outside a Testbot
|
|
21
|
+
* run, where there is no run state to record into.
|
|
22
|
+
*/
|
|
23
|
+
export async function recordExecutionVideo(result, explicitStateFile) {
|
|
24
|
+
if (!result.videoPath)
|
|
25
|
+
return;
|
|
26
|
+
const stateFile = resolveRunStatePath(explicitStateFile);
|
|
27
|
+
if (!stateFile)
|
|
28
|
+
return;
|
|
29
|
+
try {
|
|
30
|
+
const manager = StateManager.fromStatePath(stateFile);
|
|
31
|
+
const data = await manager.readData();
|
|
32
|
+
if (!data)
|
|
33
|
+
return;
|
|
34
|
+
// Last write wins: a retried execution leaves a video directory per attempt,
|
|
35
|
+
// and the report must reference the attempt whose status it reports.
|
|
36
|
+
await manager.writeData({
|
|
37
|
+
...data,
|
|
38
|
+
executionVideos: {
|
|
39
|
+
...data.executionVideos,
|
|
40
|
+
[path.basename(result.testFile)]: {
|
|
41
|
+
videoPath: result.videoPath,
|
|
42
|
+
status: result.status,
|
|
43
|
+
executedAt: result.executedAt,
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
catch (err) {
|
|
49
|
+
logger.warning("Could not record the test video — the report will omit its video link", { testFile: result.testFile, error: String(err) });
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -5,13 +5,16 @@ import { ScenarioGenerationService } from "../../services/ScenarioGenerationServ
|
|
|
5
5
|
import fs from "fs";
|
|
6
6
|
import { baseSchema, AUTH_PLACEHOLDER_TOKEN, HttpMethod } from "../../types/TestTypes.js";
|
|
7
7
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
8
|
-
import { resolveAuthFromWorkspace,
|
|
8
|
+
import { resolveAuthFromWorkspace, getWorkspaceScopedQueryParams, mergeQueryParamsJson } from "../../utils/workspaceAuth.js";
|
|
9
|
+
import { resolveQueryParamsForPath, ensureLeadingSlash, contributesNothing } from "../../workspace/queryParamResolution.js";
|
|
10
|
+
import { deriveBasePath } from "../../utils/urlPath.js";
|
|
9
11
|
import yaml from "js-yaml";
|
|
10
12
|
import { logger } from "../../utils/logger.js";
|
|
11
13
|
import { guardScenarioApproved } from "./planGuard.js";
|
|
12
14
|
import { lintScenarioChaining } from "./scenarioLint.js";
|
|
13
15
|
import { scenarioFileNameFor } from "./scenarioFileIdentity.js";
|
|
14
16
|
import { toolError } from "../../utils/utils.js";
|
|
17
|
+
import { isRecord } from "ts-is-record";
|
|
15
18
|
function isJsonValue(v) {
|
|
16
19
|
if (v === undefined || v === null)
|
|
17
20
|
return true;
|
|
@@ -40,8 +43,7 @@ function isJsonObject(v) {
|
|
|
40
43
|
if (v === undefined || v === null)
|
|
41
44
|
return true;
|
|
42
45
|
try {
|
|
43
|
-
|
|
44
|
-
return typeof p === "object" && !Array.isArray(p) && p !== null;
|
|
46
|
+
return isRecord(JSON.parse(v));
|
|
45
47
|
}
|
|
46
48
|
catch {
|
|
47
49
|
return false;
|
|
@@ -83,7 +85,7 @@ Use this for GET request filters, search terms, pagination, sorting — any para
|
|
|
83
85
|
|
|
84
86
|
CRITICAL: For search/filter/list endpoints (e.g., GET /products/search?q=bear&limit=10), parameters MUST go here, NOT in requestBody. GET request bodies are non-standard and may be ignored or rejected by servers and frameworks.
|
|
85
87
|
|
|
86
|
-
Workspace-configured api.defaultQueryParams (if set) are merged into every step automatically — no need to repeat them here. An explicit value for the same key in a step overrides
|
|
88
|
+
Workspace-configured api.defaultQueryParams (if set) are merged into every step automatically, and any api.queryParamOverrides entry whose pathPattern matches that step's path is layered on top — no need to repeat them here. An explicit value for the same key in a step overrides both, for that step only.`),
|
|
87
89
|
responseBody: z
|
|
88
90
|
.string()
|
|
89
91
|
.optional()
|
|
@@ -253,13 +255,31 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
253
255
|
logger.warning("Could not resolve auth from workspace config");
|
|
254
256
|
}
|
|
255
257
|
}
|
|
256
|
-
|
|
258
|
+
// Workspace-declared query params (SKYR-4050, path-scoped in SKYR-4127).
|
|
259
|
+
// Loaded once here; resolved per step below, since each step has its own
|
|
260
|
+
// path and may match a different override.
|
|
261
|
+
let scopedQueryParams;
|
|
262
|
+
// The request ScenarioGenerationService actually emits is basePath +
|
|
263
|
+
// step.path, so pathPattern matching has to see that same composed path —
|
|
264
|
+
// otherwise a prefix-inclusive pattern (as the workspace.yml schema docs
|
|
265
|
+
// instruct authors to write) never matches a scenario step (SKYR-4127
|
|
266
|
+
// final review). deriveBasePath is the one definition of that prefix,
|
|
267
|
+
// shared with the service itself so the two cannot drift.
|
|
268
|
+
let basePrefix = "";
|
|
257
269
|
try {
|
|
258
270
|
const repoPath = params.outputDir || process.cwd();
|
|
259
|
-
|
|
260
|
-
if (
|
|
261
|
-
|
|
262
|
-
|
|
271
|
+
scopedQueryParams = await getWorkspaceScopedQueryParams(repoPath);
|
|
272
|
+
if (scopedQueryParams && contributesNothing(scopedQueryParams)) {
|
|
273
|
+
// Skip the merge entirely rather than setting queryParams to "{}"
|
|
274
|
+
// where the old code left it untouched (Finding 4).
|
|
275
|
+
scopedQueryParams = undefined;
|
|
276
|
+
}
|
|
277
|
+
if (scopedQueryParams) {
|
|
278
|
+
basePrefix = deriveBasePath(params.baseURL);
|
|
279
|
+
logger.info("Loaded workspace query params for scenario steps", {
|
|
280
|
+
baseKeys: Object.keys(scopedQueryParams.base),
|
|
281
|
+
overrideCount: scopedQueryParams.overrides.length,
|
|
282
|
+
basePrefix,
|
|
263
283
|
});
|
|
264
284
|
}
|
|
265
285
|
}
|
|
@@ -410,8 +430,8 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
410
430
|
method: step.method,
|
|
411
431
|
path: step.path,
|
|
412
432
|
requestBody: step.requestBody,
|
|
413
|
-
queryParams:
|
|
414
|
-
? mergeQueryParamsJson(step.queryParams,
|
|
433
|
+
queryParams: scopedQueryParams
|
|
434
|
+
? mergeQueryParamsJson(step.queryParams, resolveQueryParamsForPath(scopedQueryParams, basePrefix + ensureLeadingSlash(step.path)))
|
|
415
435
|
: step.queryParams,
|
|
416
436
|
responseBody: step.responseBody,
|
|
417
437
|
statusCode: step.statusCode,
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { PlanMatchQuery } from "../../utils/planMatchKeys.js";
|
|
2
2
|
export type PlanGuardQuery = PlanMatchQuery;
|
|
3
|
-
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
export
|
|
3
|
+
/** Re-exported for the generation tools that already import it from here.
|
|
4
|
+
* The implementation moved to utils/urlPath.ts (SKYR-4127) once workspace
|
|
5
|
+
* query-param resolution needed the same derivation — see that module for why
|
|
6
|
+
* it must not be copied. */
|
|
7
|
+
export { pathFromEndpointURL } from "../../utils/urlPath.js";
|
|
8
8
|
/**
|
|
9
9
|
* Returns null when the generation call is allowed, or a human-readable error
|
|
10
10
|
* message (for `toolError`) when it must be rejected because it doesn't match
|
|
@@ -7,23 +7,11 @@
|
|
|
7
7
|
import { resolveRunStatePath, StateManager } from "../../utils/AnalysisStateManager.js";
|
|
8
8
|
import { matchesApprovedPlan } from "../../utils/planMatchKeys.js";
|
|
9
9
|
import { TOOL_REGISTER_TEST_PLAN } from "../../toolNames.js";
|
|
10
|
-
/**
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
export
|
|
15
|
-
if (!endpointURL)
|
|
16
|
-
return undefined;
|
|
17
|
-
try {
|
|
18
|
-
// new URL() percent-encodes {param} placeholders ({id} → %7Bid%7D), while
|
|
19
|
-
// register-time matchKeys are computed from raw scenario paths with braces
|
|
20
|
-
// intact — restore them or parameterized endpoints never match the plan.
|
|
21
|
-
return new URL(endpointURL).pathname.replace(/%7B/gi, "{").replace(/%7D/gi, "}");
|
|
22
|
-
}
|
|
23
|
-
catch {
|
|
24
|
-
return endpointURL.startsWith("/") ? endpointURL : undefined;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
10
|
+
/** Re-exported for the generation tools that already import it from here.
|
|
11
|
+
* The implementation moved to utils/urlPath.ts (SKYR-4127) once workspace
|
|
12
|
+
* query-param resolution needed the same derivation — see that module for why
|
|
13
|
+
* it must not be copied. */
|
|
14
|
+
export { pathFromEndpointURL } from "../../utils/urlPath.js";
|
|
27
15
|
/**
|
|
28
16
|
* Returns null when the generation call is allowed, or a human-readable error
|
|
29
17
|
* message (for `toolError`) when it must be rejected because it doesn't match
|
|
@@ -1,6 +1,20 @@
|
|
|
1
1
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { TestType, HttpMethod } from "../types/TestTypes.js";
|
|
4
|
+
/**
|
|
5
|
+
* Filename of the report, written beside the state file. SKYR-4147: the report path is
|
|
6
|
+
* derived here rather than accepted as a parameter. The caller builds the state file and
|
|
7
|
+
* the report path from a single temp dir (testbot's `skyrampTempDir()`), so the state
|
|
8
|
+
* file's directory IS the directory the caller reads the report back from. Taking the
|
|
9
|
+
* path from the agent instead meant the agent had to retype it out of a 600-character
|
|
10
|
+
* encoded prompt URI; one wrong character wrote the report somewhere nobody reads, and
|
|
11
|
+
* the tool still reported success, throwing away the whole run without a word.
|
|
12
|
+
*
|
|
13
|
+
* Must stay in sync with testbot, which hardcodes the same string in main.ts, post.ts and
|
|
14
|
+
* eval-entry.ts — the same duplication across the two repos that already exists for
|
|
15
|
+
* `RUN_STATE_FILE_NAME` / testbot's `STATE_FILE_NAME`.
|
|
16
|
+
*/
|
|
17
|
+
export declare const REPORT_FILE_NAME = "testbot-result.txt";
|
|
4
18
|
export declare const targetElementSchema: z.ZodObject<{
|
|
5
19
|
role: z.ZodString;
|
|
6
20
|
accessibleName: z.ZodString;
|
|
@@ -36,7 +50,7 @@ export declare const pageContextSchema: z.ZodObject<{
|
|
|
36
50
|
url: string;
|
|
37
51
|
pageHash?: string | undefined;
|
|
38
52
|
}>;
|
|
39
|
-
export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
53
|
+
export declare const newTestSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
40
54
|
testId: z.ZodString;
|
|
41
55
|
testType: z.ZodNativeEnum<typeof TestType>;
|
|
42
56
|
category: z.ZodEffects<z.ZodEnum<["business_rule", "security_boundary", "data_integrity", "breaking_change", "auth", "error_handling", "workflow", "data_validation", "crud"]>, "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud", unknown>;
|
|
@@ -94,7 +108,6 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
94
108
|
repository?: string | undefined;
|
|
95
109
|
traceFile?: string | undefined;
|
|
96
110
|
scenarioFile?: string | undefined;
|
|
97
|
-
frontendTrace?: string | undefined;
|
|
98
111
|
targetElements?: {
|
|
99
112
|
role: string;
|
|
100
113
|
accessibleName: string;
|
|
@@ -108,6 +121,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
108
121
|
url: string;
|
|
109
122
|
pageHash?: string | undefined;
|
|
110
123
|
} | undefined;
|
|
124
|
+
frontendTrace?: string | undefined;
|
|
111
125
|
}, {
|
|
112
126
|
description: string;
|
|
113
127
|
testType: TestType;
|
|
@@ -119,7 +133,6 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
119
133
|
category?: unknown;
|
|
120
134
|
traceFile?: string | undefined;
|
|
121
135
|
scenarioFile?: string | undefined;
|
|
122
|
-
frontendTrace?: string | undefined;
|
|
123
136
|
targetElements?: {
|
|
124
137
|
role: string;
|
|
125
138
|
accessibleName: string;
|
|
@@ -133,6 +146,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
133
146
|
url: string;
|
|
134
147
|
pageHash?: string | undefined;
|
|
135
148
|
} | undefined;
|
|
149
|
+
frontendTrace?: string | undefined;
|
|
136
150
|
}>, {
|
|
137
151
|
description: string;
|
|
138
152
|
testType: TestType;
|
|
@@ -144,7 +158,6 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
144
158
|
repository?: string | undefined;
|
|
145
159
|
traceFile?: string | undefined;
|
|
146
160
|
scenarioFile?: string | undefined;
|
|
147
|
-
frontendTrace?: string | undefined;
|
|
148
161
|
targetElements?: {
|
|
149
162
|
role: string;
|
|
150
163
|
accessibleName: string;
|
|
@@ -158,6 +171,7 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
158
171
|
url: string;
|
|
159
172
|
pageHash?: string | undefined;
|
|
160
173
|
} | undefined;
|
|
174
|
+
frontendTrace?: string | undefined;
|
|
161
175
|
}, {
|
|
162
176
|
description: string;
|
|
163
177
|
testType: TestType;
|
|
@@ -169,7 +183,31 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
169
183
|
category?: unknown;
|
|
170
184
|
traceFile?: string | undefined;
|
|
171
185
|
scenarioFile?: string | undefined;
|
|
186
|
+
targetElements?: {
|
|
187
|
+
role: string;
|
|
188
|
+
accessibleName: string;
|
|
189
|
+
testId: string | null;
|
|
190
|
+
stableId: string | null;
|
|
191
|
+
contextText: string[] | null;
|
|
192
|
+
mutability?: "unknown" | "mutable" | "immutable" | undefined;
|
|
193
|
+
widgetType?: "unknown" | "custom" | "native" | undefined;
|
|
194
|
+
}[] | null | undefined;
|
|
195
|
+
pageContext?: {
|
|
196
|
+
url: string;
|
|
197
|
+
pageHash?: string | undefined;
|
|
198
|
+
} | undefined;
|
|
172
199
|
frontendTrace?: string | undefined;
|
|
200
|
+
}>, {
|
|
201
|
+
description: string;
|
|
202
|
+
testType: TestType;
|
|
203
|
+
category: "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
|
|
204
|
+
fileName: string;
|
|
205
|
+
endpoint: string;
|
|
206
|
+
testId: string;
|
|
207
|
+
reasoning: string;
|
|
208
|
+
repository?: string | undefined;
|
|
209
|
+
traceFile?: string | undefined;
|
|
210
|
+
scenarioFile?: string | undefined;
|
|
173
211
|
targetElements?: {
|
|
174
212
|
role: string;
|
|
175
213
|
accessibleName: string;
|
|
@@ -183,8 +221,9 @@ export declare const newTestSchema: z.ZodEffects<z.ZodObject<{
|
|
|
183
221
|
url: string;
|
|
184
222
|
pageHash?: string | undefined;
|
|
185
223
|
} | undefined;
|
|
186
|
-
|
|
187
|
-
|
|
224
|
+
frontendTrace?: string | undefined;
|
|
225
|
+
}, unknown>;
|
|
226
|
+
export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
188
227
|
testId: z.ZodString;
|
|
189
228
|
testType: z.ZodNativeEnum<typeof TestType>;
|
|
190
229
|
category: z.ZodEffects<z.ZodEnum<["business_rule", "security_boundary", "data_integrity", "breaking_change", "auth", "error_handling", "workflow", "data_validation", "crud"]>, "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud", unknown>;
|
|
@@ -271,7 +310,6 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
271
310
|
reasoning: string;
|
|
272
311
|
repository?: string | undefined;
|
|
273
312
|
scenarioName?: string | undefined;
|
|
274
|
-
frontendTrace?: string | undefined;
|
|
275
313
|
targetElements?: {
|
|
276
314
|
role: string;
|
|
277
315
|
accessibleName: string;
|
|
@@ -285,6 +323,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
285
323
|
url: string;
|
|
286
324
|
pageHash?: string | undefined;
|
|
287
325
|
} | undefined;
|
|
326
|
+
frontendTrace?: string | undefined;
|
|
288
327
|
primaryEndpoint?: string | undefined;
|
|
289
328
|
openApiSpec?: string | undefined;
|
|
290
329
|
backendTrace?: string | undefined;
|
|
@@ -305,7 +344,6 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
305
344
|
repository?: string | undefined;
|
|
306
345
|
scenarioName?: string | undefined;
|
|
307
346
|
category?: unknown;
|
|
308
|
-
frontendTrace?: string | undefined;
|
|
309
347
|
targetElements?: {
|
|
310
348
|
role: string;
|
|
311
349
|
accessibleName: string;
|
|
@@ -319,6 +357,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
319
357
|
url: string;
|
|
320
358
|
pageHash?: string | undefined;
|
|
321
359
|
} | undefined;
|
|
360
|
+
frontendTrace?: string | undefined;
|
|
322
361
|
primaryEndpoint?: string | undefined;
|
|
323
362
|
openApiSpec?: string | undefined;
|
|
324
363
|
backendTrace?: string | undefined;
|
|
@@ -339,7 +378,6 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
339
378
|
reasoning: string;
|
|
340
379
|
repository?: string | undefined;
|
|
341
380
|
scenarioName?: string | undefined;
|
|
342
|
-
frontendTrace?: string | undefined;
|
|
343
381
|
targetElements?: {
|
|
344
382
|
role: string;
|
|
345
383
|
accessibleName: string;
|
|
@@ -353,6 +391,7 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
353
391
|
url: string;
|
|
354
392
|
pageHash?: string | undefined;
|
|
355
393
|
} | undefined;
|
|
394
|
+
frontendTrace?: string | undefined;
|
|
356
395
|
primaryEndpoint?: string | undefined;
|
|
357
396
|
openApiSpec?: string | undefined;
|
|
358
397
|
backendTrace?: string | undefined;
|
|
@@ -373,7 +412,40 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
373
412
|
repository?: string | undefined;
|
|
374
413
|
scenarioName?: string | undefined;
|
|
375
414
|
category?: unknown;
|
|
415
|
+
targetElements?: {
|
|
416
|
+
role: string;
|
|
417
|
+
accessibleName: string;
|
|
418
|
+
testId: string | null;
|
|
419
|
+
stableId: string | null;
|
|
420
|
+
contextText: string[] | null;
|
|
421
|
+
mutability?: "unknown" | "mutable" | "immutable" | undefined;
|
|
422
|
+
widgetType?: "unknown" | "custom" | "native" | undefined;
|
|
423
|
+
}[] | null | undefined;
|
|
424
|
+
pageContext?: {
|
|
425
|
+
url: string;
|
|
426
|
+
pageHash?: string | undefined;
|
|
427
|
+
} | undefined;
|
|
376
428
|
frontendTrace?: string | undefined;
|
|
429
|
+
primaryEndpoint?: string | undefined;
|
|
430
|
+
openApiSpec?: string | undefined;
|
|
431
|
+
backendTrace?: string | undefined;
|
|
432
|
+
}>, {
|
|
433
|
+
description: string;
|
|
434
|
+
priority: "high" | "medium" | "low";
|
|
435
|
+
testType: TestType;
|
|
436
|
+
category: "business_rule" | "security_boundary" | "data_integrity" | "breaking_change" | "auth" | "error_handling" | "workflow" | "data_validation" | "crud";
|
|
437
|
+
steps: {
|
|
438
|
+
description: string;
|
|
439
|
+
path?: string | undefined;
|
|
440
|
+
method?: HttpMethod | undefined;
|
|
441
|
+
requestBody?: Record<string, any> | undefined;
|
|
442
|
+
responseBody?: Record<string, any> | undefined;
|
|
443
|
+
expectedStatusCode?: number | undefined;
|
|
444
|
+
}[];
|
|
445
|
+
testId: string;
|
|
446
|
+
reasoning: string;
|
|
447
|
+
repository?: string | undefined;
|
|
448
|
+
scenarioName?: string | undefined;
|
|
377
449
|
targetElements?: {
|
|
378
450
|
role: string;
|
|
379
451
|
accessibleName: string;
|
|
@@ -387,8 +459,9 @@ export declare const additionalRecommendationSchema: z.ZodEffects<z.ZodObject<{
|
|
|
387
459
|
url: string;
|
|
388
460
|
pageHash?: string | undefined;
|
|
389
461
|
} | undefined;
|
|
462
|
+
frontendTrace?: string | undefined;
|
|
390
463
|
primaryEndpoint?: string | undefined;
|
|
391
464
|
openApiSpec?: string | undefined;
|
|
392
465
|
backendTrace?: string | undefined;
|
|
393
|
-
}>;
|
|
466
|
+
}, unknown>;
|
|
394
467
|
export declare function registerSubmitReportTool(server: McpServer): void;
|