@skyramp/mcp 0.5.0-rc.1 → 0.5.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/entitlements/entitlementGate.d.ts +8 -0
- package/build/entitlements/entitlementGate.js +57 -0
- package/build/entitlements/runCodes.d.ts +18 -0
- package/build/entitlements/runCodes.js +15 -0
- package/build/index.js +18 -1
- package/build/playwright/essentialTools.d.ts +1 -0
- package/build/playwright/essentialTools.js +43 -0
- package/build/playwright/registerPlaywrightTools.js +1 -43
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +22 -196
- package/build/services/FleetExecutionService.d.ts +87 -16
- package/build/services/FleetExecutionService.js +487 -48
- package/build/services/FleetRunRegistry.d.ts +49 -0
- package/build/services/FleetRunRegistry.js +178 -0
- package/build/services/fleetLimitConfirmation.d.ts +19 -0
- package/build/services/fleetLimitConfirmation.js +126 -0
- package/build/tool-phases.d.ts +14 -8
- package/build/tool-phases.js +88 -43
- package/build/tools/executeFleetTool.d.ts +6 -4
- package/build/tools/executeFleetTool.js +67 -23
- package/build/tools/fleetLifecycleTools.d.ts +20 -0
- package/build/tools/fleetLifecycleTools.js +124 -0
- package/build/tools/generate-tests/generateUILoadRestTool.d.ts +19 -1
- package/build/tools/generate-tests/generateUILoadRestTool.js +316 -50
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/workspace/workspace.d.ts +42 -120
- package/build/workspace/workspace.js +11 -45
- package/build/workspace/workspaceVerifier.d.ts +4 -10
- package/build/workspace/workspaceVerifier.js +3 -245
- package/package.json +2 -2
- package/plugin/skills/setup-testbot-target-sut/SKILL.md +5 -5
- package/plugin/skills/setup-testbot-target-sut/reference/deployed-environment.md +55 -0
- package/plugin/skills/setup-testbot-target-sut/reference/existing-tests.md +1 -1
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { McpServer } from "@modelcontextprotocol/server";
|
|
2
|
+
import { type RunCodes } from "./runCodes.js";
|
|
3
|
+
/**
|
|
4
|
+
* Makes every tool registered on `server` after this call check the run's
|
|
5
|
+
* entitlement codes before it runs. Without a run token it does nothing, so
|
|
6
|
+
* every tool behaves as before. Must be called before any tool is registered.
|
|
7
|
+
*/
|
|
8
|
+
export declare function installEntitlementGate(server: McpServer, readCodes?: () => Promise<RunCodes>, token?: string | undefined): void;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { readRunCodes } from "./runCodes.js";
|
|
2
|
+
import { NOT_GATED, TOOLS } from "../tool-phases.js";
|
|
3
|
+
const FEATURE_NAMES = {
|
|
4
|
+
analyze: "change analysis",
|
|
5
|
+
recommend: "test recommendations",
|
|
6
|
+
generate: "test generation",
|
|
7
|
+
execute: "test execution",
|
|
8
|
+
};
|
|
9
|
+
function refusal(text) {
|
|
10
|
+
return { content: [{ type: "text", text }], isError: true };
|
|
11
|
+
}
|
|
12
|
+
/** Returns why `tool` may not run with these codes, or undefined if it may. */
|
|
13
|
+
function refusalReason(tool, code, run) {
|
|
14
|
+
if ("refused" in run) {
|
|
15
|
+
return `Tool ${tool} is not available: Skyramp could not confirm what this run is allowed to do. ${run.refused}`;
|
|
16
|
+
}
|
|
17
|
+
if (run.codes.has(code))
|
|
18
|
+
return undefined;
|
|
19
|
+
return `Tool ${tool} is not available: your Skyramp plan does not include ${FEATURE_NAMES[code]}.`;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Makes every tool registered on `server` after this call check the run's
|
|
23
|
+
* entitlement codes before it runs. Without a run token it does nothing, so
|
|
24
|
+
* every tool behaves as before. Must be called before any tool is registered.
|
|
25
|
+
*/
|
|
26
|
+
export function installEntitlementGate(server, readCodes = readRunCodes, token = process.env.SKYRAMP_RUN_TOKEN) {
|
|
27
|
+
if (!token?.trim())
|
|
28
|
+
return;
|
|
29
|
+
let codes;
|
|
30
|
+
const runCodes = () => (codes ??= readCodes()
|
|
31
|
+
.catch((err) => ({
|
|
32
|
+
refused: err instanceof Error ? err.message : String(err),
|
|
33
|
+
}))
|
|
34
|
+
.then((run) => {
|
|
35
|
+
// A refusal can be temporary, so the next tool call reads again.
|
|
36
|
+
if ("refused" in run)
|
|
37
|
+
codes = undefined;
|
|
38
|
+
return run;
|
|
39
|
+
}));
|
|
40
|
+
// Typed loosely: the gate forwards every argument as it came, whatever
|
|
41
|
+
// overload the tool used.
|
|
42
|
+
const target = server;
|
|
43
|
+
const register = target.registerTool.bind(target);
|
|
44
|
+
target.registerTool = (name, config, cb) => {
|
|
45
|
+
const code = TOOLS[name]?.entitlement;
|
|
46
|
+
if (code === NOT_GATED)
|
|
47
|
+
return register(name, config, cb);
|
|
48
|
+
const handler = cb;
|
|
49
|
+
return register(name, config, async (...args) => {
|
|
50
|
+
if (code === undefined) {
|
|
51
|
+
return refusal(`Tool ${name} is not available in this run.`);
|
|
52
|
+
}
|
|
53
|
+
const reason = refusalReason(name, code, await runCodes());
|
|
54
|
+
return reason ? refusal(reason) : handler(...args);
|
|
55
|
+
});
|
|
56
|
+
};
|
|
57
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export type RunCodes = {
|
|
2
|
+
codes: ReadonlySet<string>;
|
|
3
|
+
}
|
|
4
|
+
/** Core refused the run token; `refused` is its reason. */
|
|
5
|
+
| {
|
|
6
|
+
refused: string;
|
|
7
|
+
};
|
|
8
|
+
type Entitlements = {
|
|
9
|
+
codes: string[];
|
|
10
|
+
} | {
|
|
11
|
+
error: string;
|
|
12
|
+
};
|
|
13
|
+
/** The part of SkyrampClient this file uses. Core releases before getEntitlements lack it. */
|
|
14
|
+
export interface Core {
|
|
15
|
+
getEntitlements?: (repository: string) => Promise<Entitlements>;
|
|
16
|
+
}
|
|
17
|
+
export declare function readRunCodes(core?: Core, env?: NodeJS.ProcessEnv): Promise<RunCodes>;
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The only file that may ask Skyramp core for the run's entitlement codes.
|
|
3
|
+
* Everything else in MCP reads them through `readRunCodes`, so a change on the
|
|
4
|
+
* core side changes this file and nothing else.
|
|
5
|
+
*/
|
|
6
|
+
import { SkyrampClient } from "@skyramp/skyramp";
|
|
7
|
+
export async function readRunCodes(core = new SkyrampClient(), env = process.env) {
|
|
8
|
+
if (!core.getEntitlements) {
|
|
9
|
+
return { refused: "This Skyramp version does not support it yet." };
|
|
10
|
+
}
|
|
11
|
+
const answer = await core.getEntitlements(env.GITHUB_REPOSITORY ?? "");
|
|
12
|
+
return "error" in answer
|
|
13
|
+
? { refused: answer.error }
|
|
14
|
+
: { codes: new Set(answer.codes) };
|
|
15
|
+
}
|
package/build/index.js
CHANGED
|
@@ -7,9 +7,12 @@ import { registerTraceTool } from "./tools/trace/startTraceCollectionTool.js";
|
|
|
7
7
|
import { registerTraceStopTool } from "./tools/trace/stopTraceCollectionTool.js";
|
|
8
8
|
import { registerExecuteSkyrampTestTool } from "./tools/executeSkyrampTestTool.js";
|
|
9
9
|
import { registerExecuteFleetTool } from "./tools/executeFleetTool.js";
|
|
10
|
+
import { registerFleetLifecycleTools } from "./tools/fleetLifecycleTools.js";
|
|
11
|
+
import { fleetRuns } from "./services/FleetRunRegistry.js";
|
|
10
12
|
import { AUTH_PLACEHOLDER_TOKEN } from "./types/TestTypes.js";
|
|
11
13
|
import { AUTH_CONFLICT_ERROR_MSG } from "./prompts/test-recommendation/recommendationSections.js";
|
|
12
14
|
import { logger } from "./utils/logger.js";
|
|
15
|
+
import { installEntitlementGate } from "./entitlements/entitlementGate.js";
|
|
13
16
|
import { registerUITestTool } from "./tools/generate-tests/generateUIRestTool.js";
|
|
14
17
|
import { registerUILoadTestTool } from "./tools/generate-tests/generateUILoadRestTool.js";
|
|
15
18
|
import { registerSmokeTestTool } from "./tools/generate-tests/generateSmokeRestTool.js";
|
|
@@ -155,6 +158,8 @@ async function createServer({ era }) {
|
|
|
155
158
|
8. The user can always override workspace defaults by explicitly specifying values in their request.
|
|
156
159
|
`,
|
|
157
160
|
});
|
|
161
|
+
// Before any tool registers, so every tool goes through the check.
|
|
162
|
+
installEntitlementGate(server);
|
|
158
163
|
// Register prompts
|
|
159
164
|
logger.info("Starting prompt registration process");
|
|
160
165
|
const prompts = [
|
|
@@ -239,6 +244,7 @@ async function createServer({ era }) {
|
|
|
239
244
|
registerLogoutTool,
|
|
240
245
|
registerExecuteSkyrampTestTool,
|
|
241
246
|
registerExecuteFleetTool,
|
|
247
|
+
registerFleetLifecycleTools,
|
|
242
248
|
registerTraceTool,
|
|
243
249
|
registerTraceStopTool,
|
|
244
250
|
];
|
|
@@ -325,11 +331,18 @@ async function createServer({ era }) {
|
|
|
325
331
|
runInitTrigger();
|
|
326
332
|
return server;
|
|
327
333
|
}
|
|
334
|
+
// A crashing process must not stay alive anywhere near the full graceful
|
|
335
|
+
// shutdown's 30s grace — it still gives workers a moment to cancel, but caps
|
|
336
|
+
// it low enough that a crash can't hang the process.
|
|
337
|
+
const CRASH_CLEANUP_WAIT_MS = 3_000;
|
|
338
|
+
/** Interrupt running fleets (bounded wait) so an exit does not leave workers loading. */
|
|
339
|
+
const cancelFleets = () => fleetRuns.cancelAll(CRASH_CLEANUP_WAIT_MS).catch(() => { });
|
|
328
340
|
// Global error handlers for crash telemetry
|
|
329
341
|
process.on("uncaughtException", async (error) => {
|
|
330
342
|
// Don't send analytics for EPIPE errors
|
|
331
343
|
const isEpipeError = "code" in error && error.code === "EPIPE";
|
|
332
344
|
if (isEpipeError) {
|
|
345
|
+
await cancelFleets();
|
|
333
346
|
process.exit(0);
|
|
334
347
|
}
|
|
335
348
|
logger.critical("Uncaught exception - MCP server crashing", {
|
|
@@ -342,6 +355,7 @@ process.on("uncaughtException", async (error) => {
|
|
|
342
355
|
catch (telemetryError) {
|
|
343
356
|
logger.error("Failed to send crash telemetry", { telemetryError });
|
|
344
357
|
}
|
|
358
|
+
await cancelFleets();
|
|
345
359
|
process.exit(1);
|
|
346
360
|
});
|
|
347
361
|
// Handle unhandled promise rejections — without this handler Node.js 15+
|
|
@@ -351,6 +365,7 @@ process.on("unhandledRejection", async (reason) => {
|
|
|
351
365
|
const error = reason instanceof Error ? reason : new Error(String(reason));
|
|
352
366
|
// EPIPE means the client already disconnected — exit cleanly
|
|
353
367
|
if ("code" in error && error.code === "EPIPE") {
|
|
368
|
+
await cancelFleets();
|
|
354
369
|
process.exit(0);
|
|
355
370
|
}
|
|
356
371
|
// gRPC/Docker connectivity errors — log and survive.
|
|
@@ -397,7 +412,9 @@ async function main() {
|
|
|
397
412
|
const cleanupPlaywright = playwrightService
|
|
398
413
|
? playwrightService.shutdown().catch(() => { })
|
|
399
414
|
: Promise.resolve();
|
|
400
|
-
|
|
415
|
+
// A running fleet is interrupted so it can cancel its workers, not left loading them.
|
|
416
|
+
const cleanupFleets = fleetRuns.cancelAll().catch(() => { });
|
|
417
|
+
Promise.all([cleanupPlaywright, cleanupFleets]).finally(() => {
|
|
401
418
|
handle.close().finally(() => {
|
|
402
419
|
process.exit(0);
|
|
403
420
|
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare const ESSENTIAL_TOOLS: ReadonlySet<string>;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// Only expose essential browser tools to reduce LLM confusion.
|
|
2
|
+
// The full set (23 tools) overwhelms the LLM and causes it to use
|
|
3
|
+
// browser_evaluate/browser_run_code instead of the simpler tools.
|
|
4
|
+
//
|
|
5
|
+
// browser_blueprint is the only DOM Analyzer tool registered — it handles
|
|
6
|
+
// both full captures and same-URL deltas via the URL-cache (isFullCapture).
|
|
7
|
+
// browser_blueprint_diff, browser_sitemap_build, browser_sitemap_query fire
|
|
8
|
+
// 0 times in production evals and have no active call path in the testbot prompt.
|
|
9
|
+
//
|
|
10
|
+
// browser_widget_contract_lookup / browser_widget_contract_cache are
|
|
11
|
+
// intentionally excluded: the curated fingerprint library only covers
|
|
12
|
+
// test fixtures (not production widgets), and the LLM can handle custom
|
|
13
|
+
// widgets via snapshot-driven trial-and-error without the tool.
|
|
14
|
+
// The widgetType field on blueprint elements still signals "custom" to
|
|
15
|
+
// the agent — no guidance is lost.
|
|
16
|
+
export const ESSENTIAL_TOOLS = new Set([
|
|
17
|
+
"browser_navigate",
|
|
18
|
+
"browser_snapshot",
|
|
19
|
+
"browser_click",
|
|
20
|
+
"browser_type",
|
|
21
|
+
"browser_press_key",
|
|
22
|
+
"browser_select_option",
|
|
23
|
+
"browser_hover",
|
|
24
|
+
"browser_drag",
|
|
25
|
+
"browser_file_upload",
|
|
26
|
+
"browser_evaluate",
|
|
27
|
+
"browser_tabs",
|
|
28
|
+
"browser_navigate_back",
|
|
29
|
+
"browser_wait_for",
|
|
30
|
+
"browser_take_screenshot",
|
|
31
|
+
"browser_assert",
|
|
32
|
+
"browser_assert_hidden",
|
|
33
|
+
"browser_assert_api_request",
|
|
34
|
+
"browser_assert_table_cell",
|
|
35
|
+
"browser_mouse_action",
|
|
36
|
+
"browser_visual_snapshot",
|
|
37
|
+
"browser_gojs_add_node",
|
|
38
|
+
"browser_gojs_link",
|
|
39
|
+
"skyramp_export_zip",
|
|
40
|
+
"skyramp_load_trace",
|
|
41
|
+
// DOM Analyzer: blueprint capture (capture-act-capture pattern)
|
|
42
|
+
"browser_blueprint",
|
|
43
|
+
]);
|
|
@@ -8,6 +8,7 @@ import { PlaywrightTraceService, } from "./PlaywrightTraceService.js";
|
|
|
8
8
|
import { blueprintJsonSpan } from "./blueprintDigest.js";
|
|
9
9
|
import { recordBlueprintCapture, recordBrowserNavigation, } from "./blueprintDigestStore.js";
|
|
10
10
|
import { recordTraceExport } from "./traceExportStore.js";
|
|
11
|
+
import { ESSENTIAL_TOOLS } from "./essentialTools.js";
|
|
11
12
|
let _service = null;
|
|
12
13
|
export function getPlaywrightTraceService() {
|
|
13
14
|
return _service;
|
|
@@ -169,49 +170,6 @@ export async function registerPlaywrightTools(server, options) {
|
|
|
169
170
|
}
|
|
170
171
|
const tools = _service.getTools();
|
|
171
172
|
logger.info(`Registering ${tools.length} Playwright browser tools`);
|
|
172
|
-
// Only expose essential browser tools to reduce LLM confusion.
|
|
173
|
-
// The full set (23 tools) overwhelms the LLM and causes it to use
|
|
174
|
-
// browser_evaluate/browser_run_code instead of the simpler tools.
|
|
175
|
-
//
|
|
176
|
-
// browser_blueprint is the only DOM Analyzer tool registered — it handles
|
|
177
|
-
// both full captures and same-URL deltas via the URL-cache (isFullCapture).
|
|
178
|
-
// browser_blueprint_diff, browser_sitemap_build, browser_sitemap_query fire
|
|
179
|
-
// 0 times in production evals and have no active call path in the testbot prompt.
|
|
180
|
-
//
|
|
181
|
-
// browser_widget_contract_lookup / browser_widget_contract_cache are
|
|
182
|
-
// intentionally excluded: the curated fingerprint library only covers
|
|
183
|
-
// test fixtures (not production widgets), and the LLM can handle custom
|
|
184
|
-
// widgets via snapshot-driven trial-and-error without the tool.
|
|
185
|
-
// The widgetType field on blueprint elements still signals "custom" to
|
|
186
|
-
// the agent — no guidance is lost.
|
|
187
|
-
const ESSENTIAL_TOOLS = new Set([
|
|
188
|
-
"browser_navigate",
|
|
189
|
-
"browser_snapshot",
|
|
190
|
-
"browser_click",
|
|
191
|
-
"browser_type",
|
|
192
|
-
"browser_press_key",
|
|
193
|
-
"browser_select_option",
|
|
194
|
-
"browser_hover",
|
|
195
|
-
"browser_drag",
|
|
196
|
-
"browser_file_upload",
|
|
197
|
-
"browser_evaluate",
|
|
198
|
-
"browser_tabs",
|
|
199
|
-
"browser_navigate_back",
|
|
200
|
-
"browser_wait_for",
|
|
201
|
-
"browser_take_screenshot",
|
|
202
|
-
"browser_assert",
|
|
203
|
-
"browser_assert_hidden",
|
|
204
|
-
"browser_assert_api_request",
|
|
205
|
-
"browser_assert_table_cell",
|
|
206
|
-
"browser_mouse_action",
|
|
207
|
-
"browser_visual_snapshot",
|
|
208
|
-
"browser_gojs_add_node",
|
|
209
|
-
"browser_gojs_link",
|
|
210
|
-
"skyramp_export_zip",
|
|
211
|
-
"skyramp_load_trace",
|
|
212
|
-
// DOM Analyzer: blueprint capture (capture-act-capture pattern)
|
|
213
|
-
"browser_blueprint",
|
|
214
|
-
]);
|
|
215
173
|
const filteredTools = tools.filter((t) => ESSENTIAL_TOOLS.has(t.name));
|
|
216
174
|
logger.info(`Filtering to ${filteredTools.length} essential tools (from ${tools.length} total)`);
|
|
217
175
|
for (const tool of filteredTools) {
|
|
@@ -1,194 +1,21 @@
|
|
|
1
1
|
import { getPersonaPrefix } from "../personas.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
]
|
|
20
|
-
\`\`\`
|
|
21
|
-
|
|
22
|
-
### List all top-level directories
|
|
23
|
-
<list_directories>
|
|
24
|
-
Run a directory listing of the workspace root. Every top-level directory is a potential service. Common layouts:
|
|
25
|
-
| Layout | Example dirs | Expect |
|
|
26
|
-
|--------|-------------|--------|
|
|
27
|
-
| Monorepo | apps/web, apps/api, packages/shared | 1 service per app |
|
|
28
|
-
| Microservices | services/auth, services/orders | 1 service per service dir |
|
|
29
|
-
| Single service | src/, lib/ | 1 service (the root) |
|
|
30
|
-
|
|
31
|
-
If the repo is a monorepo, check for workspace manifests BEFORE scanning individual directories:
|
|
32
|
-
1. JavaScript/TypeScript monorepos:
|
|
33
|
-
- If pnpm-workspace.yaml exists, read the packages array to find all package directories (for example, ["apps/*", "packages/*"]).
|
|
34
|
-
- If turbo.json exists, read the root package.json workspaces field to enumerate all packages.
|
|
35
|
-
- If lerna.json exists, read the packages array in lerna.json (for example, ["packages/*"]).
|
|
36
|
-
- If the root package.json has a "workspaces" field, use the glob patterns to identify all packages.
|
|
37
|
-
- If nx.json exists, scan the projects in nx.json or project.json files in subdirectories.
|
|
38
|
-
2. Java monorepos:
|
|
39
|
-
- If a parent pom.xml has a <modules> section, each listed module is a separate service (Maven multi-module project).
|
|
40
|
-
- If settings.gradle or settings.gradle.kts has include statements, each included project is a separate service (Gradle multi-project).
|
|
41
|
-
3. Python monorepos:
|
|
42
|
-
- If a pyproject.toml has a [tool.poetry.packages] or [tool.setuptools.packages] section with multiple paths, each is a potential service.
|
|
43
|
-
- If multiple directories each contain their own pyproject.toml or requirements.txt, treat each as an independent service.
|
|
44
|
-
When any of these exist, use the workspace manifest to enumerate ALL packages and apps. Each package with its own server entry point, API routes, or UI framework is a separate service. Do NOT skip frontend packages. A Next.js app, React SPA, or Vue app in the monorepo is a service.
|
|
45
|
-
</list_directories>
|
|
46
|
-
|
|
47
|
-
### Inspect every candidate directory
|
|
48
|
-
<inspect_directories>
|
|
49
|
-
For each top-level directory, check for service indicator files.
|
|
50
|
-
|
|
51
|
-
Language indicators (presence of any means an independent service):
|
|
52
|
-
1. package.json indicates typescript or javascript
|
|
53
|
-
2. requirements.txt, pyproject.toml, or Pipfile indicates python
|
|
54
|
-
3. pom.xml or build.gradle indicates java
|
|
55
|
-
|
|
56
|
-
Test framework (look inside the service directory):
|
|
57
|
-
1. playwright.config.* indicates playwright
|
|
58
|
-
2. pytest.ini, conftest.py, or pyproject.toml [tool.pytest] indicates pytest
|
|
59
|
-
3. junit in pom.xml indicates junit
|
|
60
|
-
|
|
61
|
-
API schemas (look inside the service directory and check known framework defaults):
|
|
62
|
-
1. openapi.json/yaml or swagger.json/yaml provides the schema file path
|
|
63
|
-
2. FastAPI projects serve the schema at http://localhost:{port}/openapi.json
|
|
64
|
-
3. Express with swagger-ui serves at http://localhost:{port}/api-docs
|
|
65
|
-
4. Spring Boot serves at http://localhost:{port}/v3/api-docs
|
|
66
|
-
</inspect_directories>
|
|
67
|
-
|
|
68
|
-
### Check root-level runtime config
|
|
69
|
-
<runtime_config>
|
|
70
|
-
Inspect the repo root (and subdirectories like .devcontainer/) for shared runtime configuration:
|
|
71
|
-
1. CLAUDE.md or AGENTS.md: if either file exists at the repo root, check it for dev/test/CI setup instructions including how to start services, required environment variables, and runtime configuration.
|
|
72
|
-
2. Docker Compose files: scan for ALL compose files including docker-compose.yml, docker-compose.yaml, docker-compose.*.yml (such as docker-compose.testbot.yml or docker-compose.dev.yml), compose.yml, and compose.*.yaml in the repo root and subdirectories. Distinguish between application compose files and infrastructure-only compose files. Infrastructure compose files contain only supporting services like databases, Redis, Minio, mail servers, or message queues and do NOT run the application itself. Application compose files contain the actual application services that build from the repo source code (look for "build:" directives pointing to the repo, or services that map to discovered application directories). Only use application compose files for determining runtime, serverStartCommand, and serverStopCommand. When a non-default compose file is found, the serverStartCommand and serverStopCommand must reference it explicitly (such as "docker compose -f docker-compose.testbot.yml up -d --build <service-name>" and "docker compose -f docker-compose.testbot.yml stop <service-name>"). Docker Compose ALWAYS prefixes the network name with the project name. If compose has "networks: { my-net: ... }", the actual network name is "<project-name>_my-net". If there is no explicit networks section, the default network is "<project-name>_default". The project name is the basename of the working directory where docker compose runs.
|
|
73
|
-
3. Makefile: extract start and dev targets.
|
|
74
|
-
4. Root package.json scripts: extract workspace-level commands.
|
|
75
|
-
</runtime_config>
|
|
76
|
-
|
|
77
|
-
### Build the complete services array
|
|
78
|
-
<build_services>
|
|
79
|
-
Create one service entry per deployable unit. You MUST include every backend/API service (Python, Java, Go, Node.js) and every frontend service (React, Vue, Angular, Next.js). Set runtime fields from docker-compose.yml if present.
|
|
80
|
-
</build_services>
|
|
81
|
-
|
|
82
|
-
### Basic fields
|
|
83
|
-
<basic_fields>
|
|
84
|
-
1. serviceName (required): A unique identifier for the service. Use the format <repoName>-<serviceDirName>-<frontend or backend> for consistency. For example: directus-api-backend, directus-app-frontend. If the directory name already ends with "backend" or "frontend", do not duplicate the suffix (use "demoshop-fullstack-backend" not "demoshop-fullstack-backend-backend"). For single-service repos where the service directory is the root, omit the directory portion such as onlineBoutique-frontend.
|
|
85
|
-
2. language (required): One of python, typescript, javascript, or java. Detect from package.json (typescript or javascript), requirements.txt or pyproject.toml (python), or pom.xml or build.gradle (java).
|
|
86
|
-
3. framework (required): One of playwright, pytest, robot, or junit. Detect from pytest.ini, playwright.config, jest.config, or junit in pom.xml. The framework MUST match the language: python uses pytest or robot, typescript or javascript uses playwright, and java uses junit.
|
|
87
|
-
4. testDirectory (required): A stable path relative to the repo root where generated tests for this service will be placed. For each service, use the test directory configured by that service's test framework when one is discoverable:
|
|
88
|
-
- Playwright: Read playwright.config.ts (or .js/.mjs) and extract the testDir value.
|
|
89
|
-
- pytest: Read pytest.ini, pyproject.toml [tool.pytest.ini_options], or setup.cfg [tool:pytest] for testpaths.
|
|
90
|
-
- JUnit: Usually src/test/java. Check pom.xml or build.gradle for custom test source directories.
|
|
91
|
-
If no framework-configured test directory is available, use the Skyramp deterministic fallback:
|
|
92
|
-
- Single service: set testDirectory to tests/skyramp.
|
|
93
|
-
- Multiple services or monorepos: set testDirectory to tests/skyramp/<serviceDirName>, where <serviceDirName> is the service's directory relative to its repository root with path separators and whitespace replaced by hyphens, or the repository's name for a service at the root.
|
|
94
|
-
Framework config takes precedence. Use the Skyramp deterministic fallback only when no framework-configured test directory is available.
|
|
95
|
-
NEVER set testDirectory to \`.skyramp\`, to any path inside it, or to \`.\` or the repo root, whatever the framework config or existing test files say. \`.skyramp\` is the executor's own working area: it holds the run videos and executor artefacts. skyramp_init_workspace rejects all of these, and the generation tools refuse to write there.
|
|
96
|
-
</basic_fields>
|
|
97
|
-
|
|
98
|
-
### API fields
|
|
99
|
-
<api_fields>
|
|
100
|
-
1. api.schemaPath: Path or URL to an OpenAPI or Swagger schema. Search for openapi.json, openapi.yaml, swagger.json, or swagger.yaml files. Framework defaults are: FastAPI serves /openapi.json, Express serves /api-docs, and Spring serves /v3/api-docs. For locally-run services, use a localhost URL. For cloud or externally hosted services (such as Salesforce, Vercel, or Cloudflare), use the actual deployment URL found in config or documentation.
|
|
101
|
-
2. api.baseUrl (required): The base URL where the service is reachable, such as "http://localhost:3000" or "https://api.example.com". Derive from docker-compose ports, app config, README, or environment variables. Use localhost for services run locally and the actual deployment URL for cloud or externally hosted services. NEVER fabricate a URL. Only use URLs found in config files, README, or environment variables.
|
|
102
|
-
Include the API path prefix (e.g. \`/api\`, \`/api/v1\`, \`/v1\`) when one applies — not just \`host:port\`. A bare host means every test request hits the root and 404s. Derive the prefix from the strongest available signal: OpenAPI \`servers[].url\` or \`basePath\`; the app's routing config (FastAPI \`root_path\`, Express \`app.use('/api/v1', ...)\`, Rails routes prefix, Django \`urlpatterns\` mount); or the longest common leading path across the existing test suite's endpoints. If none of those yield a prefix, leave it bare AND flag it in your summary so the user can review.
|
|
103
|
-
3. api.authType (required): The authentication type. Valid values are: ${AUTH_TYPES_PROMPT_LIST}
|
|
104
|
-
Detect by checking in the following order (language-agnostic, apply whichever signals match):
|
|
105
|
-
a. Dependencies and packages (package.json, requirements.txt, go.mod, Gemfile, composer.json, pom.xml, build.gradle):
|
|
106
|
-
- jsonwebtoken, passport-jwt, @nestjs/jwt, jose, fastapi[security], PyJWT, python-jose, github.com/golang-jwt/jwt, jjwt, or spring-security-oauth2 indicates bearer. Spring exception: if spring-security is present but the security config uses HttpSecurity.formLogin() or sessionManagement() without a JwtDecoder bean, the actual auth is session cookies so use cookie. Check the SecurityConfig or WebSecurityConfigurerAdapter class before assigning bearer to any Spring service.
|
|
107
|
-
- passport-http or Spring basic auth indicates basic.
|
|
108
|
-
- passport-oauth2, openid-client, doorkeeper, keycloak-connect, spring-security-oauth2-resource-server, or github.com/coreos/go-oidc indicates oauth.
|
|
109
|
-
- rest_framework with TokenAuthentication or djangorestframework-simplejwt (Token scheme) indicates token.
|
|
110
|
-
- express-session, cookie-session, iron-session, next-auth, gorilla/sessions, or laravel/session indicates cookie.
|
|
111
|
-
- laravel/sanctum or laravel/passport on API routes indicates bearer; on frontend web routes indicates cookie.
|
|
112
|
-
b. Environment variables (.env, docker-compose, README):
|
|
113
|
-
- JWT_SECRET, ACCESS_TOKEN_SECRET, FIREBASE_SECRET, or SUPABASE_JWT_SECRET indicates bearer.
|
|
114
|
-
- API_KEY, X_API_KEY, ADMIN_KEY, or SERVICE_KEY (used as header value, not JWT signing) indicates apiKey.
|
|
115
|
-
- CLIENT_ID paired with CLIENT_SECRET, OAUTH_CLIENT_*, or OIDC_* indicates oauth.
|
|
116
|
-
- SESSION_SECRET, COOKIE_SECRET, or NEXTAUTH_SECRET indicates cookie.
|
|
117
|
-
c. Source code and middleware patterns (auth, middleware, or security config files):
|
|
118
|
-
- Node/Express: req.headers.authorization split "Bearer" indicates bearer. "Authorization: Token" indicates token. Custom header check indicates apiKey. Session or cookie middleware indicates cookie.
|
|
119
|
-
- FastAPI/Starlette: HTTPBearer, OAuth2PasswordBearer, or Depends(get_current_user) indicates bearer. SessionMiddleware indicates cookie.
|
|
120
|
-
- Django/DRF: TokenAuthentication indicates token. JWTAuthentication indicates bearer. SessionAuthentication indicates cookie.
|
|
121
|
-
- Spring: JwtDecoder or JwtAuthenticationFilter indicates bearer. HttpSecurity.formLogin() with sessions indicates cookie.
|
|
122
|
-
- Go: ParseWithClaims or GetHeader("Authorization") indicates bearer. gorilla/sessions indicates cookie.
|
|
123
|
-
- Rails: authenticate_or_request_with_http_token indicates bearer. before_action :authenticate_user! (Devise) indicates cookie.
|
|
124
|
-
- Laravel: auth:sanctum on API indicates bearer. middleware("auth") on web indicates cookie.
|
|
125
|
-
- Rust/Axum: Authorization<Bearer> extraction indicates bearer.
|
|
126
|
-
d. Query-parameter auth: Some APIs pass credentials as a URL query param rather than a header (such as ?key=<key>, ?api_key=<key>, or ?access_token=<token>). Signals include source code reading credentials from req.query.key, request.query_params["api_key"], or @RequestParam("token"), and API docs or README showing auth examples like /endpoint?key=<your-key>.
|
|
127
|
-
e. Fallback: when none of the signals above applies, use none, for any service. An API that answers without credentials is unauthenticated; never assume bearer without a signal.
|
|
128
|
-
4. api.authHeader: The exact HTTP header name carrying the credential.
|
|
129
|
-
- For bearer, basic, oauth, or token: always "Authorization" (inferred automatically, so you may omit it).
|
|
130
|
-
- For cookie or session: always "Cookie" (also inferred automatically).
|
|
131
|
-
- For apiKey: this is required. Set to the actual custom header name such as "X-API-Key" or "X-Admin-Key".
|
|
132
|
-
- For none: set authHeader to "".
|
|
133
|
-
5. api.authScheme: The Authorization header prefix (the word before the token). Standard types derive this automatically: bearer uses "Bearer", token uses "Token", and basic uses "Basic". Set explicitly only for custom or non-standard schemes (for example, Hawk uses "Hawk" and Digest uses "Digest"). Omit for cookie, session, or apiKey since they do not use the Authorization header.
|
|
134
|
-
</api_fields>
|
|
135
|
-
|
|
136
|
-
### Runtime fields
|
|
137
|
-
<runtime_fields>
|
|
138
|
-
1. runtimeDetails.runtime (required): One of local, docker, or k8s. Detect per service:
|
|
139
|
-
- If the service is listed in an application docker compose file (one that builds or runs the application, not infrastructure-only files containing only databases, caches, or queues) or has a Dockerfile as its intended run method, use "docker".
|
|
140
|
-
- If k8s manifests exist (charts/, k8s/, deploy/) AND the service is intended to run in K8s for local development, use "k8s".
|
|
141
|
-
- If the service has no Docker or k8s configuration and is run directly via a language runtime (npm, python, java, etc.), use "local".
|
|
142
|
-
A repo may have MIXED runtimes. A backend in docker-compose.yml uses "docker" while a frontend run with pnpm or npm locally uses "local". Include ALL services regardless of runtime.
|
|
143
|
-
If the frontend is bundled inside the API Docker container (no separate frontend service in the compose file), both frontend and backend services should use runtime "docker", share the same baseUrl (the container's exposed URL), and use the same serverStartCommand since they run in the same container.
|
|
144
|
-
IMPORTANT — proxy parameters are NOT runtime indicators: If proxyKubeconfig, proxyNamespace, or proxyUrl appear in the conversation context, they are for a Skyramp proxy worker that collects traffic samples for mock generation. They do NOT indicate the repo's runtime. Do not use proxy K8s params to set runtime, k8sNamespace, or k8sContext. Determine the runtime solely from the repo's own files (docker-compose.yml, Dockerfile, k8s manifests that are part of the repo's own local dev workflow). Note that many repos have K8s deployment manifests for staging/production but use Docker Compose for local development — check which setup the repo actually uses for local dev.
|
|
145
|
-
2. runtimeDetails.serverStartCommand: The command to start or deploy the service. It MUST match the runtime:
|
|
146
|
-
- For "docker" runtime: Use a Docker command that rebuilds application services from the working tree before starting them, such as "docker compose up -d --build <service-name>" for the default docker-compose.yml, or "docker compose -f <compose-file> up -d --build <service-name>" when using a non-default compose file. This is always derivable from the application docker compose file and service name. If the compose service has no build context and only references an image, note that the command cannot rebuild local code.
|
|
147
|
-
- For "k8s" runtime: Use a deploy command such as "kubectl apply -f deploy/", "helm install myrelease .", or "skaffold run". This is always derivable from the manifests or charts present in the repo.
|
|
148
|
-
- For "local" runtime: Use an application command such as "uvicorn main:app", "npm run dev", or "java -jar app.jar". Derive from Makefile, package.json scripts, or README. If no start command is discoverable, omit this field entirely.
|
|
149
|
-
NEVER mix runtime types with incompatible commands (for example, using "uvicorn" with runtime "docker" will cause errors). For "local" runtime, NEVER fabricate a command. Only use commands found in Makefile, package.json scripts, or README.
|
|
150
|
-
When the SUT recipe (.skyramp/sut/setup.sh) is what brings a service up and no compose service in the repository runs it, set serverStartCommand to "bash .skyramp/sut/setup.sh" and serverStopCommand to "bash .skyramp/sut/teardown.sh". Never invent a container name or a hand-built docker build or docker run command.
|
|
151
|
-
3. runtimeDetails.serverStopCommand: Optional command to stop the service after test execution. ONLY set when runtime is "docker" and serverStartCommand starts persistent Docker infrastructure, or to "bash .skyramp/sut/teardown.sh" when serverStartCommand runs the SUT recipe. For docker compose starts, prefer matching stop/down commands from the same compose file. Omit this field for "local" and "k8s" runtimes, or when teardown is unsafe or not discoverable.
|
|
152
|
-
4. runtimeDetails.dockerNetwork: Docker network name. ONLY set when runtime is "docker". NEVER set for "local" or "k8s".
|
|
153
|
-
5. runtimeDetails.k8sNamespace: Kubernetes namespace. ONLY set when runtime is "k8s". NEVER set for "local" or "docker".
|
|
154
|
-
6. runtimeDetails.k8sContext: Kubernetes context. ONLY set when runtime is "k8s". NEVER set for "local" or "docker".
|
|
155
|
-
</runtime_fields>
|
|
156
|
-
|
|
157
|
-
### Existing test suites
|
|
158
|
-
<test_suites>
|
|
159
|
-
runtimeDetails.testSuites records the service's own existing tests, which later runs execute and maintain. They cannot find a suite missing from this list. Each distinct (runner, test directory) pair is one suite: a Mocha suite under packages/api/test/ and a Playwright suite under e2e/ are two. Find them in two places and record the union: every test-runner config in the test tree (playwright.config.*, cypress.config.*, vitest.config.*, pytest.ini, test projects, and so on), and every CI step that runs tests (npm test, pytest, go test, dotnet test, mvn test or verify, cypress run, and so on), including a separate integration-test step. Record each suite under the service it tests, even when there is only one, and every suite you run yourself. Omit testSuites only when the service has no existing tests. Each entry sets:
|
|
160
|
-
1. framework: the suite's runner, one of ${SUPPORTED_FRAMEWORKS.join(", ")}. Leave it out when the runner is none of these; record the suite all the same.
|
|
161
|
-
2. testRunCommand: the command that runs the suite, from the root of the repository the suite lives in, taken from its CI job or package scripts. A machine-readable reporter flag and the test file paths, relative to that root, are appended to the end of it, so it must accept appended arguments and must not cd into a subdirectory: point the runner at the suite instead, with the repository's own config file or a package filter (for example "npx playwright test --config e2e/playwright.config.ts" or "pnpm --filter @acme/api test").
|
|
162
|
-
3. pathGlobs: the directories the suite owns, as globs relative to that same root (for example ["e2e/tests/**"]). Set it on every suite. Scope by directory, not by filename, and keep the suites' globs disjoint, so each test file belongs to exactly one suite.
|
|
163
|
-
</test_suites>
|
|
164
|
-
|
|
165
|
-
### Verification
|
|
166
|
-
<verification>
|
|
167
|
-
Before calling skyramp_init_workspace, confirm all of the following:
|
|
168
|
-
1. Always scan the repo and find services. A repo should have at least one service.
|
|
169
|
-
2. ALL services are included, both backend and frontend. The workspace config is a complete registry of the entire repo, not just the service relevant to your current task. A fullstack or monorepo MUST have multiple services. If you found only one, re-scan every top-level directory before proceeding.
|
|
170
|
-
3. Services NOT in docker-compose.yml (such as a frontend run with pnpm or npm locally) MUST still be included with runtime "local".
|
|
171
|
-
4. Every service has api.baseUrl set to a valid, discoverable URL. Use localhost for local services or the actual deployment URL for cloud or external services. Never fabricate a URL.
|
|
172
|
-
5. Every service with authType apiKey has authHeader explicitly set to the actual custom header name (such as "X-API-Key" or "X-Admin-Key"). If you cannot find the header name in the source code, env vars, or README, do NOT use authType apiKey. Use authType none instead and add a YAML comment explaining auth is unresolved.
|
|
173
|
-
6. framework matches language (python uses pytest or robot, typescript or javascript uses playwright, java uses junit).
|
|
174
|
-
7. testDirectory follows the stable resolution rules above: framework config file when present (Playwright testDir in playwright.config.ts, pytest testpaths in pytest.ini or pyproject.toml, JUnit test source dir in pom.xml or build.gradle); otherwise the deterministic default (tests/skyramp for a single service, tests/skyramp/<serviceDirName> for multiple services). No service uses \`.skyramp\`, \`.\` or the repo root.
|
|
175
|
-
8. If serverStartCommand is provided, it matches the runtime. If serverStopCommand is provided, runtime is "docker" and the command is a Docker command.
|
|
176
|
-
9. For services in docker-compose.yml: runtime MUST be "docker" and the command MUST be a docker command such as "docker compose up -d --build <service-name>" when the service has a build context. Always include it since it is derivable from the service name.
|
|
177
|
-
10. NEVER use application-level commands (uvicorn, npm, node, python, java, etc.) with runtime "docker".
|
|
178
|
-
11. For "local" runtime: if no start command is discoverable from Makefile, package.json scripts, or README, omit serverStartCommand rather than guessing.
|
|
179
|
-
12. dockerNetwork is set only when runtime is "docker".
|
|
180
|
-
13. k8sNamespace and k8sContext are set only when runtime is "k8s".
|
|
181
|
-
14. Every existing test suite, from the test tree and from CI, is recorded under runtimeDetails.testSuites with its testRunCommand and pathGlobs.
|
|
182
|
-
15. If force-recreating a workspace.yml due to a schema validation error, correct the schema error but keep any valid system-under-test setup (serviceName, baseUrl, auth config, runtime details) as it is.
|
|
183
|
-
</verification>
|
|
184
|
-
|
|
185
|
-
Once verified, call skyramp_init_workspace with:
|
|
186
|
-
- workspacePath: the repository root path
|
|
187
|
-
- services: the array built above
|
|
188
|
-
- scanToken: the token returned by the first call to skyramp_init_scan
|
|
189
|
-
- force: defaults to false. Set to true only if the user explicitly asks to overwrite an existing .skyramp/workspace.yml, or if the existing file must be regenerated due to a validation failure such as a schema mismatch, unknown fields, or a parse error.
|
|
190
|
-
|
|
191
|
-
skyramp_init_workspace checks the services against these rules before writing anything. When it refuses, it lists each problem: fix every one and call it again with the full corrected services array and the same scanToken, which a refusal leaves valid.`;
|
|
2
|
+
export const INIT_WORKSPACE_INSTRUCTIONS = `${getPersonaPrefix()}Scan this repository and register every component of the system under test in .skyramp/workspace.yml.
|
|
3
|
+
|
|
4
|
+
### Find the components
|
|
5
|
+
A component runs on its own: a backend API, a frontend app, or a worker with its own entry point. Shared libraries are not components. Read the workspace manifests, compose and container files, CI workflows, README, CLAUDE.md and AGENTS.md, then check every top-level directory. Register every component, frontends included; a monorepo usually has several.
|
|
6
|
+
|
|
7
|
+
### Fields
|
|
8
|
+
Take every value from a file you read or a rule below; never invent one. For each component:
|
|
9
|
+
- serviceName: always \`<repoName>-<componentDir>\`, or \`<repoName>\` for a component at the root.
|
|
10
|
+
- language: the language of its tests, or of its code when it has none, in lowercase.
|
|
11
|
+
- framework: the framework of its API or UI tests; when it has none, the one its repository or language commonly uses. Never leave it out.
|
|
12
|
+
- testDirectory: where generated tests go, relative to the repository root. Use the directory its test framework config names; otherwise \`tests/skyramp\` for a single component, or \`tests/skyramp/<componentDir>\` for several, with \`/\` in the directory replaced by \`-\`. Never \`.skyramp\`, a path inside it, or the root.
|
|
13
|
+
- api.baseUrl: the URL its API paths are appended to: scheme, host and port where it runs locally (or its hosted URL), plus a base path only when its OpenAPI \`servers\` URL or \`basePath\` holds one the paths leave out. A frontend the backend serves shares the backend's. api.schemaPath: its OpenAPI file or URL, if any.
|
|
14
|
+
- api.authType: from its auth code, dependencies or environment variables; \`none\` without a signal. For \`apiKey\`, set api.authHeader to the header name.
|
|
15
|
+
- runtimeDetails.runtime: \`docker\` when it runs in containers, \`k8s\` when it runs on Kubernetes, else \`local\`, as the repository runs it. dockerNetwork: the network its containers join, if any.
|
|
16
|
+
|
|
17
|
+
### Call the tool
|
|
18
|
+
First write a Discovery Summary: the services array, and for each value the file or rule that set it. Then call skyramp_init_workspace with workspacePath (the repository root), services and scanToken. Set force only when the user asks to overwrite, or the existing file fails validation; then keep its valid values. When the tool refuses, fix every problem it lists and call it again with the full array and the same scanToken.`;
|
|
192
19
|
/**
|
|
193
20
|
* The scan instructions for one workspace. With related repositories, every
|
|
194
21
|
* repository is scanned into the one workspace.yml at `workspacePath`, each
|
|
@@ -200,19 +27,18 @@ export function buildInitWorkspaceInstructions(target) {
|
|
|
200
27
|
const deliveryRoot = target.testRepositoryPath ?? target.workspacePath;
|
|
201
28
|
const blocks = [];
|
|
202
29
|
if (related.length > 0) {
|
|
203
|
-
const count = related.length + 1;
|
|
204
30
|
blocks.push(`### Target repositories
|
|
205
31
|
<target_repositories>
|
|
206
|
-
|
|
207
|
-
- ${target.workspacePath}: this workspace's own
|
|
208
|
-
${related.map((r) => `- ${r.repositoryPath}: set repository to "${r.repository}" on
|
|
209
|
-
|
|
32
|
+
Scan each repository below as above; pass the components of all ${related.length + 1} to one skyramp_init_workspace call with workspacePath "${target.workspacePath}":
|
|
33
|
+
- ${target.workspacePath}: this workspace's own, with no repository.
|
|
34
|
+
${related.map((r) => `- ${r.repositoryPath}: set repository to "${r.repository}" on each of its components.`).join("\n")}
|
|
35
|
+
Write a related repository's paths from its own root, except testDirectory: every testDirectory is relative to ${deliveryRoot}, and a related repository's fallback is \`tests/skyramp/<repoName>\`, or \`tests/skyramp/<repoName>-<componentDir>\` with several components.
|
|
210
36
|
</target_repositories>`);
|
|
211
37
|
}
|
|
212
38
|
if (target.testRepositoryPath) {
|
|
213
39
|
blocks.push(`### Test repository
|
|
214
40
|
<test_repository>
|
|
215
|
-
Generated tests
|
|
41
|
+
Generated tests go to the test repository at ${target.testRepositoryPath}. Read testDirectory from its framework config, relative to its root, with the same fallback. Register no component for it.
|
|
216
42
|
</test_repository>`);
|
|
217
43
|
}
|
|
218
44
|
return blocks.length
|