@skyramp/mcp 0.2.1 → 0.2.150-rc.sut
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +7 -2
- package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +82 -0
- package/build/prompts/sut-setup/registerSetupSutPrompt.js +49 -0
- package/build/prompts/sut-setup/shared.js +80 -0
- package/build/resources/sutSetupResource.js +47 -0
- package/build/utils/docker.test.js +1 -1
- package/build/utils/versions.js +1 -1
- package/package.json +2 -2
package/build/index.js
CHANGED
|
@@ -26,6 +26,8 @@ import { registerBatchScenarioTestTool } from "./tools/generate-tests/generateBa
|
|
|
26
26
|
import { registerMockTool } from "./tools/generate-tests/generateMockRestTool.js";
|
|
27
27
|
import { registerAnalyzeChangesTool, registerUiAnalyzeChangesTool, registerAnalyzeTestHealthTool, registerActionsTool, } from "./tools/test-management/index.js";
|
|
28
28
|
import { registerTestbotPrompt } from "./prompts/testbot/testbot-prompts.js";
|
|
29
|
+
import { registerSetupSutPrompt } from "./prompts/sut-setup/registerSetupSutPrompt.js";
|
|
30
|
+
import { registerSutSetupResource } from "./resources/sutSetupResource.js";
|
|
29
31
|
import { registerTestbotResource } from "./resources/testbotResource.js";
|
|
30
32
|
import { registerSubmitReportTool } from "./tools/submitReportTool.js";
|
|
31
33
|
import { registerInitializeWorkspaceTool } from "./tools/workspace/initializeWorkspaceTool.js";
|
|
@@ -123,7 +125,10 @@ const prompts = [
|
|
|
123
125
|
if (isTestbotEnabled()) {
|
|
124
126
|
prompts.push(registerTestbotPrompt);
|
|
125
127
|
registerTestbotResource(server);
|
|
126
|
-
logger.info("
|
|
128
|
+
logger.info("Testbot prompt enabled via SKYRAMP_FEATURE_TESTBOT");
|
|
129
|
+
prompts.push(registerSetupSutPrompt);
|
|
130
|
+
registerSutSetupResource(server);
|
|
131
|
+
logger.info("Testbot SUT setup prompt and resource enabled via SKYRAMP_FEATURE_TESTBOT");
|
|
127
132
|
}
|
|
128
133
|
prompts.forEach((registerPrompt) => registerPrompt(server));
|
|
129
134
|
logger.info("All prompts registered successfully");
|
|
@@ -174,7 +179,7 @@ const infrastructureTools = [
|
|
|
174
179
|
];
|
|
175
180
|
if (isTestbotEnabled()) {
|
|
176
181
|
infrastructureTools.push(registerSubmitReportTool);
|
|
177
|
-
logger.info("
|
|
182
|
+
logger.info("Testbot tools enabled via SKYRAMP_FEATURE_TESTBOT");
|
|
178
183
|
}
|
|
179
184
|
infrastructureTools.forEach((registerTool) => registerTool(server));
|
|
180
185
|
// Register Playwright browser tools (trace recording via browser automation)
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { TESTBOT_WORKFLOW_PATH, buildContextBlock, buildCommonSutErrorsSection, buildLocalValidationSection, buildTestbotLifecycleInputsSection, getPersonaPrefix, } from "../shared.js";
|
|
2
|
+
export function getAdaptWorkflowPrompt(args) {
|
|
3
|
+
const sourceLine = args.sutSourceWorkflowFile
|
|
4
|
+
? `The user pointed at \`${args.sutSourceWorkflowFile}\` as the source workflow whose setup/teardown steps should be copied into the Testbot workflow. Use it as your primary source of setup steps — but still read the rest of the repo (other workflows, docker-compose, Kubernetes, Helm) to understand the full SUT shape.`
|
|
5
|
+
: `No specific source workflow was provided. Read ALL available GitHub workflows in this repository to learn how services are built, started, and authenticated, and pick the source workflow whose setup steps best match a SUT for testing (typically an e2e/integration workflow, not lint/release/security workflows).`;
|
|
6
|
+
const scanReadStep = args.sutSourceWorkflowFile
|
|
7
|
+
? `Read the given source GitHub workflow file \`${args.sutSourceWorkflowFile}\` (the user's hint about which steps to lift), then ALSO read the rest of \`.github/workflows/\` and all infrastructure files (docker-compose, Kubernetes manifests, Helm charts) — the full SUT shape often lives across multiple files, so do NOT tunnel into the source workflow alone. Workflows show *how* the SUT is started; infrastructure files show *what* the SUT is. When a workflow step references an infra file (e.g., \`docker compose -f infra/compose.yml up\`, \`helm upgrade <chart>\`, \`kubectl apply -f k8s/\`), open that file too — services, ports, env vars, and dependencies live there.`
|
|
8
|
+
: `Read all GitHub workflow files in \`.github/workflows/\` AND all infrastructure files (docker-compose, Kubernetes manifests, Helm charts). Workflows show *how* the SUT is started; infrastructure files show *what* the SUT is. When a workflow step references an infra file (e.g., \`docker compose -f infra/compose.yml up\`, \`helm upgrade <chart>\`, \`kubectl apply -f k8s/\`), open that file too — services, ports, env vars, and dependencies live there. Pick the source workflow (the one whose SETUP/TEST/TEARDOWN steps will be lifted) once you have the full picture.`;
|
|
9
|
+
const thinkingSourceClause = args.sutSourceWorkflowFile
|
|
10
|
+
? `confirms you are using \`${args.sutSourceWorkflowFile}\` as the source of setup steps (or explains why a different workflow is a better fit)`
|
|
11
|
+
: `names the workflow you selected as the source of setup steps and explains why`;
|
|
12
|
+
return `${getPersonaPrefix()}
|
|
13
|
+
### Goal
|
|
14
|
+
1. Generate working System Under Test (SUT) files that enable the Skyramp Testbot workflow (\`${TESTBOT_WORKFLOW_PATH}\`) to test any PR-specific code in this repository end-to-end for the services supported (REST or gRPC service APIs, queues, frontend UIs).
|
|
15
|
+
2. Source workflow: ${sourceLine}
|
|
16
|
+
3. Required output:
|
|
17
|
+
a. Output a <thinking> block that: (i) ${thinkingSourceClause}, (ii) summarizes what you learned about the SUT — services, secrets, auth, runner size — pulling from ALL workflows and infrastructure files, and (iii) decides how the SUT will be brought up — prefer Testbot lifecycle commands alone (targetSetupCommand, targetReadyCheckCommand, targetTeardownCommand) when the setup can be expressed as a single shell command; add GHA pre-steps only for tooling that cannot run in a shell (Buildx, cache, registry login) while still using lifecycle commands for service start; escalate to full GHA steps replacing service startup only when a single command cannot express it; use fully wrapped GHA steps for both setup and teardown only as a last resort. State clearly which approach was chosen and why.
|
|
18
|
+
b. Adapt the Testbot workflow file (\`${TESTBOT_WORKFLOW_PATH}\`) for bringing up the system under test.
|
|
19
|
+
${buildContextBlock(args)}
|
|
20
|
+
|
|
21
|
+
### Scan repo and understand the SUT
|
|
22
|
+
1. ${scanReadStep}
|
|
23
|
+
2. From the source workflow, classify each step by its *purpose* — not by its name or surface mechanism (\`env:\`, \`with:\`, and \`\${{ secrets.* }}\` can appear on any category and do not determine it):
|
|
24
|
+
a. SETUP — prepares the SUT (install, build, start services, migrate, seed, configure env or inject secrets for the SUT). Common attributes: \`env:\` blocks (e.g. \`DATABASE_URL\`, build flags), \`with:\` parameters (e.g. \`actions/cache\`, \`actions/setup-node\`), \`\${{ secrets.* }}\` references (e.g. registry login, deploy keys), dependency installs (e.g. \`npm ci\` before build).
|
|
25
|
+
b. TEST — runs the test suite (will be replaced by \`skyramp/testbot\`). Common attributes: \`env:\` blocks (e.g. \`API_BASE_URL\`, \`AUTH_TOKEN\`), \`with:\` parameters (e.g. \`actions/cache\`, custom test-runner actions), \`\${{ secrets.* }}\` references (e.g. test credentials, live-integration keys), dependency installs (e.g. \`npm ci\` right before \`npm test\` in the same step).
|
|
26
|
+
c. TEARDOWN — stops or cleans up.
|
|
27
|
+
d. UNRELATED — does none of those (lint, format, notifications, badges).
|
|
28
|
+
|
|
29
|
+
Output as:
|
|
30
|
+
\`\`\`
|
|
31
|
+
Step: "<step name>" → SETUP | TEST | TEARDOWN | UNRELATED
|
|
32
|
+
\`\`\`
|
|
33
|
+
3. Summarize the SUT, pulling from all sources:
|
|
34
|
+
a. Setup commands needed to bring up all services (may come from multiple workflows / infra files).
|
|
35
|
+
b. Required secrets and env vars (with their \`\${{ secrets.* }}\` references and pass-through style — \`env:\` blocks vs. \`secrets: inherit\`).
|
|
36
|
+
c. Runner needed (\`runs-on\` value; large-ubuntu for heavy builds).
|
|
37
|
+
d. How an auth token is obtained (script, step output, or secret).
|
|
38
|
+
|
|
39
|
+
### Decide SUT pattern
|
|
40
|
+
The SUT must always be built from the PR's source code — Testbot reuses this workflow for every future PR, so it must validate the code in the PR being tested, not a stale snapshot. Use build: blocks in compose files (or docker build against the PR checkout) so images come from PR source. Pull-only image references and pinned upstream tags are fine for sidecars (databases, queues, caches), but the application services under test must be built locally.
|
|
41
|
+
Evaluate the four patterns in strict priority order — always start with Pattern A and only escalate when it genuinely cannot express the setup. Full semantics for each lifecycle input are in the reference section below.
|
|
42
|
+
1. Pattern A — Testbot lifecycle commands only (try this first, always)
|
|
43
|
+
Use when the entire setup — build, start, seed — can be expressed as one or two shell commands (e.g., docker compose up -d --build, make start, ./scripts/start.sh). Do NOT add any GHA steps. Set targetSetupCommand, targetReadyCheckCommand, targetReadyCheckTimeout, and targetTeardownCommand on the Testbot action; leave skipTargetSetup unset.
|
|
44
|
+
2. Pattern B — Hybrid: GHA pre-steps for tool/env setup + Testbot lifecycle commands for service start
|
|
45
|
+
Use when the source workflow requires GHA-specific steps purely for environment or tooling (e.g., docker/setup-buildx-action, actions/cache, private registry login) but the actual service start can still be expressed as a single command. Copy only those tool/env pre-steps BEFORE the Testbot action. Set targetSetupCommand, targetReadyCheckCommand, targetReadyCheckTimeout, and targetTeardownCommand on the Testbot action; leave skipTargetSetup unset. Preserve every env:, with:, and \${{ secrets.* }} reference from the pre-steps exactly as written.
|
|
46
|
+
3. Pattern C — GHA steps replace service startup + Testbot lifecycle validation
|
|
47
|
+
Use when the service startup itself requires multiple GHA steps that cannot be collapsed into a single command. Copy the source workflow's SETUP steps verbatim BEFORE the Testbot action. Set skipTargetSetup: 'true' and do not set targetSetupCommand. Set targetReadyCheckCommand and targetReadyCheckTimeout on the Testbot action. Add TEARDOWN steps AFTER the Testbot action with if: always(), or set targetTeardownCommand if a single command suffices — pick one, not both. Preserve every env:, with:, and \${{ secrets.* }} reference exactly as written.
|
|
48
|
+
4. Pattern D — Fully wrapped by GHA steps
|
|
49
|
+
Use when teardown is also too complex for a single command. Copy the SETUP steps BEFORE and TEARDOWN steps AFTER the Testbot action (with if: always()). Set skipTargetSetup: 'true'. Do not set targetSetupCommand or targetTeardownCommand. Preserve every env:, with:, and \${{ secrets.* }} reference exactly as written.
|
|
50
|
+
|
|
51
|
+
${buildTestbotLifecycleInputsSection()}
|
|
52
|
+
|
|
53
|
+
### Adapt Testbot workflow
|
|
54
|
+
#### What to do
|
|
55
|
+
1. Edit \`${TESTBOT_WORKFLOW_PATH}\` (already created by the Testbot installer):
|
|
56
|
+
a. Apply the chosen pattern:
|
|
57
|
+
- Pattern A: set lifecycle inputs on the Testbot action, no GHA steps.
|
|
58
|
+
- Pattern B: add GHA tool/env pre-steps BEFORE the Testbot action, set lifecycle inputs (\`targetSetupCommand\`, \`targetReadyCheckCommand\`, \`targetTeardownCommand\`), leave \`skipTargetSetup\` unset.
|
|
59
|
+
- Pattern C: add GHA SETUP steps BEFORE the Testbot action, set \`skipTargetSetup: 'true'\`, set \`targetReadyCheckCommand\`, add TEARDOWN steps AFTER or set \`targetTeardownCommand\` — do NOT set \`targetSetupCommand\`.
|
|
60
|
+
- Pattern D: add GHA SETUP steps BEFORE and TEARDOWN steps AFTER the Testbot action, set \`skipTargetSetup: 'true'\`, do NOT set \`targetSetupCommand\` or \`targetTeardownCommand\`.
|
|
61
|
+
b. Update \`runs-on\` to match the runner needed by the source workflow's setup (some repos need large-ubuntu).
|
|
62
|
+
c. Pass all required secrets through — reuse the same \`\${{ secrets.* }}\` references or \`secrets: inherit\`. For GHA steps (Patterns B, C, D), set secrets via \`env:\` on the individual steps that need them; for lifecycle inputs (Patterns A, B), set them on the Testbot action or at the job level.
|
|
63
|
+
d. The basic file already contains the \`skyramp/testbot\` action — REPLACE the original workflow's test-run step with it, do NOT add a duplicate test runner step.
|
|
64
|
+
2. Configure auth — use the lifecycle input reference above for full semantics of \`authTokenCommand\` and \`uiCredentials\`:
|
|
65
|
+
a. If the source workflow exports or seeds an auth token, set \`authTokenCommand\` on the Testbot action (or create \`.skyramp/sut/get-auth-token.sh\`).
|
|
66
|
+
b. If any service requires browser login, set \`uiCredentials\` on the Testbot action.
|
|
67
|
+
c. Omit both when the SUT is unauthenticated.
|
|
68
|
+
3. Handle source-workflow shape:
|
|
69
|
+
a. If the original workflow uses \`matrix\` builds, pick ONE configuration for the Testbot workflow (matrix runs would multiply Testbot invocations).
|
|
70
|
+
b. If the original workflow uses \`needs:\` (job dependencies), inline the dependent job's steps into the Testbot job — Testbot runs as a single job.
|
|
71
|
+
|
|
72
|
+
#### What not to do
|
|
73
|
+
1. Do not modify any other GitHub workflow files referenced. The only github workflow file you may edit is \`${TESTBOT_WORKFLOW_PATH}\`; every other workflow under \`.github/workflows/\` (including the source workflow) is read-only reference material.
|
|
74
|
+
2. Do not change the \`sutSetupMode\` input in \`${TESTBOT_WORKFLOW_PATH}\`. Testbot manages this value automatically — it must stay as provided so the next CI run knows to validate the adapted workflow rather than run tests. Changing it to \`none\` or any other value will break the bootstrap cycle.
|
|
75
|
+
3. Do not point the SUT at a prebuilt upstream image (e.g. \`image: org/app:latest\` or a fixed commit SHA from \`main\`) for any application service under test. The image will lag the PR and Testbot will validate stale code.
|
|
76
|
+
4. Do not point the SUT at a remote staging or production environment for application services under test. Staging code drifts from PR source and turns Testbot's results into noise. External infra is acceptable only for sidecar dependencies the PR does not change (e.g. a managed test database).
|
|
77
|
+
|
|
78
|
+
### Verify
|
|
79
|
+
${buildCommonSutErrorsSection()}
|
|
80
|
+
${buildLocalValidationSection()}
|
|
81
|
+
`;
|
|
82
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { logger } from "../../utils/logger.js";
|
|
3
|
+
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
4
|
+
import { getAdaptWorkflowPrompt } from "./modes/adaptWorkflowPrompt.js";
|
|
5
|
+
export var SutSetupMode;
|
|
6
|
+
(function (SutSetupMode) {
|
|
7
|
+
SutSetupMode["None"] = "none";
|
|
8
|
+
SutSetupMode["AdaptWorkflow"] = "adapt_workflow";
|
|
9
|
+
})(SutSetupMode || (SutSetupMode = {}));
|
|
10
|
+
export function registerSetupSutPrompt(server) {
|
|
11
|
+
logger.info("Registering SUT setup prompt");
|
|
12
|
+
server.registerPrompt("skyramp_testbot_sut", {
|
|
13
|
+
description: "Scan a repository and generate SUT (System Under Test) setup files for Skyramp Testbot. " +
|
|
14
|
+
"First milestone: adapts an existing CI/e2e workflow into the Testbot workflow.",
|
|
15
|
+
argsSchema: {
|
|
16
|
+
repositoryPath: z
|
|
17
|
+
.string()
|
|
18
|
+
.describe("Absolute path to the repository root"),
|
|
19
|
+
sutSetupMode: z
|
|
20
|
+
.nativeEnum(SutSetupMode)
|
|
21
|
+
.default(SutSetupMode.None)
|
|
22
|
+
.describe("SUT setup mode: none | adapt_workflow"),
|
|
23
|
+
sutSourceWorkflowFile: z
|
|
24
|
+
.string()
|
|
25
|
+
.default("")
|
|
26
|
+
.describe("Path to an existing workflow file to adapt. Example: .github/workflows/e2e-tests.yml"),
|
|
27
|
+
},
|
|
28
|
+
}, async (args) => {
|
|
29
|
+
const prompt = buildSutPrompt(args);
|
|
30
|
+
AnalyticsService.pushMCPToolEvent("skyramp_testbot_sut_prompt", undefined, { mode: args.sutSetupMode }).catch(() => { });
|
|
31
|
+
return {
|
|
32
|
+
messages: [
|
|
33
|
+
{
|
|
34
|
+
role: "user",
|
|
35
|
+
content: { type: "text", text: prompt },
|
|
36
|
+
},
|
|
37
|
+
],
|
|
38
|
+
};
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
function buildSutPrompt(args) {
|
|
42
|
+
switch (args.sutSetupMode) {
|
|
43
|
+
case SutSetupMode.AdaptWorkflow:
|
|
44
|
+
return getAdaptWorkflowPrompt(args);
|
|
45
|
+
case SutSetupMode.None:
|
|
46
|
+
default:
|
|
47
|
+
return "sutSetupMode is 'none'. No SUT setup required.";
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { getPersonaPrefix } from "../personas.js";
|
|
2
|
+
/**
|
|
3
|
+
* Path of the Testbot workflow file relative to the repo root. Created by the
|
|
4
|
+
* Testbot installer and edited in-place by every SUT setup mode.
|
|
5
|
+
*/
|
|
6
|
+
export const TESTBOT_WORKFLOW_PATH = ".github/workflows/skyramp-testbot.yml";
|
|
7
|
+
export function buildContextBlock(args) {
|
|
8
|
+
const lines = [];
|
|
9
|
+
lines.push(`<context>`);
|
|
10
|
+
lines.push(`Repository path: ${args.repositoryPath}`);
|
|
11
|
+
lines.push(`Setup mode: ${args.sutSetupMode}`);
|
|
12
|
+
if (args.sutSourceWorkflowFile)
|
|
13
|
+
lines.push(`Source workflow file: ${args.sutSourceWorkflowFile}`);
|
|
14
|
+
lines.push(`</context>`);
|
|
15
|
+
return lines.join("\n");
|
|
16
|
+
}
|
|
17
|
+
export function buildLocalValidationSection() {
|
|
18
|
+
return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
|
|
19
|
+
Run commands one at a time in your shell — individual execution pinpoints failures far faster than running the whole workflow at once. Any non-zero exit is a fix-needed signal: adjust the workflow, re-run the failing command, and do not proceed until it passes.
|
|
20
|
+
1. Validate build scripts and helper programs first — run these before anything else so build and auth failures surface cheaply:
|
|
21
|
+
a. Every helper script the workflow calls (e.g., \`./scripts/*.sh\`, \`make <target>\`, \`./gradlew <task>\`) — run each and confirm exit 0.
|
|
22
|
+
b. \`.skyramp/sut/get-auth-token.sh\` if present — run it and confirm it prints a non-empty token to stdout.
|
|
23
|
+
c. Every Dockerfile referenced by build steps — run \`docker build\` against each.
|
|
24
|
+
d. Every docker-compose file the workflow references — run \`docker compose -f <path> config\` to validate the YAML, then \`docker compose -f <path> build\`.
|
|
25
|
+
2. Validate the SUT lifecycle — choose the branch that matches the chosen pattern:
|
|
26
|
+
a. If lifecycle commands are set on the Testbot action (skipTargetSetup is unset):
|
|
27
|
+
i. Run \`targetSetupCommand\` — confirm exit 0.
|
|
28
|
+
ii. Poll \`targetReadyCheckCommand\` until it exits 0 within \`targetReadyCheckTimeout\` seconds.
|
|
29
|
+
iii. Run \`targetTeardownCommand\` — confirm cleanup and exit 0.
|
|
30
|
+
b. If GHA steps wrap the Testbot action (skipTargetSetup: 'true'):
|
|
31
|
+
i. Extract the shell body of each SETUP step and run them in order. If a step depends entirely on a \${{ secrets.* }} value your shell cannot resolve, note it in <thinking> and skip — do not fabricate a value.
|
|
32
|
+
ii. Poll \`targetReadyCheckCommand\` until it exits 0.
|
|
33
|
+
iii. Extract and run each TEARDOWN step's body in order so the next iteration starts clean.
|
|
34
|
+
Only proceed to the success report once setup → health check → teardown all pass.`;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Returns a reference block describing every Testbot action input that controls
|
|
38
|
+
* the SUT lifecycle and auth. Embed this wherever a prompt needs the LLM to
|
|
39
|
+
* understand what knobs are available before choosing how to wire up the SUT.
|
|
40
|
+
*/
|
|
41
|
+
export function buildTestbotLifecycleInputsSection() {
|
|
42
|
+
return `#### Testbot action inputs — target lifecycle and auth
|
|
43
|
+
Target lifecycle inputs (control how Testbot starts, validates, and stops the SUT):
|
|
44
|
+
1. \`targetSetupCommand\` — shell command that builds and starts all SUT services. Testbot runs this before tests.
|
|
45
|
+
a. Default when unset: \`docker compose up -d\` (starts existing images, does NOT build).
|
|
46
|
+
b. For PR-source builds use \`docker compose up -d --build\` or the equivalent for helm/script-based SUTs.
|
|
47
|
+
c. Override whenever the source workflow's start sequence differs from the default.
|
|
48
|
+
2. \`targetReadyCheckCommand\` — polling command (e.g., \`curl -sf http://localhost:8080/health\`) that Testbot runs repeatedly after setup until it exits 0 (service is ready) or the timeout is reached.
|
|
49
|
+
a. Always set this — without it Testbot proceeds immediately and tests may hit a not-yet-ready service.
|
|
50
|
+
3. \`targetReadyCheckTimeout\` — seconds to wait for \`targetReadyCheckCommand\` to succeed.
|
|
51
|
+
a. Default: \`30\`. Cold Docker builds routinely take 60–180 s; always set this to \`'120'\` or higher for compose-based SUTs.
|
|
52
|
+
4. \`targetTeardownCommand\` — shell command that stops and cleans up the SUT after tests (e.g., \`docker compose down -v\`).
|
|
53
|
+
a. Always set this so each run starts from a clean state.
|
|
54
|
+
b. Omit only when teardown is handled by surrounding GHA steps with \`if: always()\`.
|
|
55
|
+
5. \`skipTargetSetup\` — when \`'true'\`, Testbot skips running \`targetSetupCommand\` entirely.
|
|
56
|
+
a. Set this only when GHA steps surrounding the Testbot action already bring the SUT up.
|
|
57
|
+
b. Leave it unset (or \`'false'\`) when Testbot should run the full lifecycle via the inputs above.
|
|
58
|
+
Auth inputs (used by Testbot to authenticate against the running SUT during test recording and execution):
|
|
59
|
+
6. \`authTokenCommand\` — shell command whose stdout is the auth credential for REST/gRPC API testing (e.g., a Bearer token, API key, or session cookie). Testbot captures the output and injects it as the \`Authorization\` header.
|
|
60
|
+
a. Set this when the SUT requires authentication for API calls.
|
|
61
|
+
b. If the source workflow exports a token via a step output or script, wire that same script here (e.g., \`bash .skyramp/sut/get-auth-token.sh\`).
|
|
62
|
+
c. If the workflow seeds a test user during setup, create \`.skyramp/sut/get-auth-token.sh\` that logs in with those credentials and prints the token.
|
|
63
|
+
d. Omit when the SUT APIs are unauthenticated.
|
|
64
|
+
7. \`uiCredentials\` — \`username:password\` pair typed into the browser login form during UI test recording (format: \`myuser:mypassword\`).
|
|
65
|
+
a. Set for any frontend service that requires browser-based login.
|
|
66
|
+
b. Use \`\${{ secrets.SKYRAMP_UI_CREDENTIALS }}\` if the secret exists, otherwise use credentials seeded during SUT setup.`;
|
|
67
|
+
}
|
|
68
|
+
export function buildCommonSutErrorsSection() {
|
|
69
|
+
return `Before finishing, verify the adapted Testbot workflow setup against common errors seen during SUT setup:
|
|
70
|
+
1. port_conflict — another container is already using the port. Pick a free host port or stop the conflicting container.
|
|
71
|
+
2. image_not_found / build_failed — the referenced image does not exist for this SHA, or the build step has the wrong context/Dockerfile path. Build from PR source, do not rely on upstream commit-SHA tags.
|
|
72
|
+
3. healthcheck_timeout — the service is slow to come up. Use \`targetReadyCheckTimeout >= 120\` and ensure the health endpoint actually exists.
|
|
73
|
+
4. connection_refused — the service is not listening on the expected host/port (often binding to 127.0.0.1 inside the container; bind to 0.0.0.0).
|
|
74
|
+
5. dependency_error — required service (DB, cache, queue) is missing from compose or not healthy before the app starts. Add \`depends_on\` with \`condition: service_healthy\`.
|
|
75
|
+
6. permission_denied — script is not executable (\`chmod +x\`) or volume mount has wrong ownership.
|
|
76
|
+
7. auth_endpoint_404 / auth_credentials_invalid — the auth script hits a wrong path, or the database has no seeded user. The auth script must create the user before login when starting from a fresh DB.
|
|
77
|
+
8. missing env vars / secrets — required env vars are unset on the runner. Either default them in the script or pass them through workflow \`env:\`.
|
|
78
|
+
Do not add an in-prompt fix loop — Testbot's external validator will retry with concrete error context if any of these occur.`;
|
|
79
|
+
}
|
|
80
|
+
export { getPersonaPrefix };
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { ResourceTemplate, } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
+
import { logger } from "../utils/logger.js";
|
|
3
|
+
import { AnalyticsService } from "../services/AnalyticsService.js";
|
|
4
|
+
import { getAdaptWorkflowPrompt } from "../prompts/sut-setup/modes/adaptWorkflowPrompt.js";
|
|
5
|
+
import { SutSetupMode } from "../prompts/sut-setup/registerSetupSutPrompt.js";
|
|
6
|
+
export function registerSutSetupResource(server) {
|
|
7
|
+
logger.info("Registering SUT setup resource");
|
|
8
|
+
// RFC 6570 {+rest} (reserved expansion) captures the entire query string
|
|
9
|
+
// including the leading "?". This avoids the SDK's per-param regex which
|
|
10
|
+
// fails on empty query-param values.
|
|
11
|
+
const template = new ResourceTemplate("skyramp://prompts/skyramp_testbot_sut{+rest}", { list: undefined });
|
|
12
|
+
server.registerResource("skyramp_testbot_sut", template, {
|
|
13
|
+
title: "Skyramp Testbot SUT Setup",
|
|
14
|
+
description: "Returns mode-specific instructions for SUT (System Under Test) setup. " +
|
|
15
|
+
"Modes: none, adapt_workflow.",
|
|
16
|
+
mimeType: "text/plain",
|
|
17
|
+
}, async (uri) => {
|
|
18
|
+
const param = (name, fallback) => uri.searchParams.get(name) ?? fallback;
|
|
19
|
+
const mode = param("sutSetupMode", SutSetupMode.None);
|
|
20
|
+
const args = {
|
|
21
|
+
repositoryPath: param("repositoryPath", "."),
|
|
22
|
+
sutSetupMode: mode,
|
|
23
|
+
sutSourceWorkflowFile: param("sutSourceWorkflowFile", ""),
|
|
24
|
+
};
|
|
25
|
+
let prompt;
|
|
26
|
+
switch (mode) {
|
|
27
|
+
case SutSetupMode.AdaptWorkflow:
|
|
28
|
+
prompt = getAdaptWorkflowPrompt(args);
|
|
29
|
+
break;
|
|
30
|
+
case SutSetupMode.None:
|
|
31
|
+
prompt = "sutSetupMode is 'none'. No SUT setup required.";
|
|
32
|
+
break;
|
|
33
|
+
default:
|
|
34
|
+
prompt = `Unknown SUT setup mode: '${mode}'. Valid modes: none, adapt_workflow.`;
|
|
35
|
+
}
|
|
36
|
+
AnalyticsService.pushMCPToolEvent("skyramp_testbot_sut_resource", undefined, { mode }).catch(() => { });
|
|
37
|
+
return {
|
|
38
|
+
contents: [
|
|
39
|
+
{
|
|
40
|
+
uri: uri.toString(),
|
|
41
|
+
mimeType: "text/plain",
|
|
42
|
+
text: prompt,
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
};
|
|
46
|
+
});
|
|
47
|
+
}
|
|
@@ -54,7 +54,7 @@ describe("dockerImageExistsLocally", () => {
|
|
|
54
54
|
});
|
|
55
55
|
});
|
|
56
56
|
describe("pullDockerImage", () => {
|
|
57
|
-
const IMAGE = "skyramp/executor:v1.3.
|
|
57
|
+
const IMAGE = "skyramp/executor:v1.3.25";
|
|
58
58
|
beforeEach(() => jest.clearAllMocks());
|
|
59
59
|
describe("on amd64 host", () => {
|
|
60
60
|
const originalArch = process.arch;
|
package/build/utils/versions.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skyramp/mcp",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.150-rc.sut",
|
|
4
4
|
"main": "build/index.js",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./build/index.js",
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@modelcontextprotocol/sdk": "^1.24.3",
|
|
57
57
|
"@playwright/test": "^1.55.0",
|
|
58
|
-
"@skyramp/skyramp": "1.3.
|
|
58
|
+
"@skyramp/skyramp": "1.3.25",
|
|
59
59
|
"dockerode": "^5.0.0",
|
|
60
60
|
"fast-glob": "^3.3.3",
|
|
61
61
|
"js-yaml": "^4.1.1",
|