@skyramp/mcp 0.2.150-rc.ldw → 0.2.150-rc.ldw-4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/commands/commandLibrary.d.ts +0 -1
- package/build/commands/commandLibrary.js +0 -7
- package/build/commands/localDevTestChangesCommand.js +4 -4
- package/build/index.js +3 -2
- package/build/playwright/registerPlaywrightTools.js +2 -0
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +28 -1
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +1 -0
- package/build/prompts/local-dev/local-dev-plan.js +26 -44
- package/build/prompts/local-dev/local-dev-prompts.js +2 -1
- package/build/prompts/personas.d.ts +2 -2
- package/build/prompts/personas.js +2 -2
- package/build/prompts/sut-setup/modes/dockerComposePrompt.js +3 -0
- package/build/prompts/sut-setup/shared.js +1 -3
- package/build/prompts/test-maintenance/actionsInstructions.d.ts +3 -1
- package/build/prompts/test-maintenance/actionsInstructions.js +10 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +15 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +63 -3
- package/build/prompts/test-maintenance/driftAnalysisSections.d.ts +0 -2
- package/build/prompts/test-maintenance/driftAnalysisSections.js +7 -31
- package/build/prompts/test-maintenance/driftAnalysisShared.d.ts +20 -0
- package/build/prompts/test-maintenance/driftAnalysisShared.js +80 -0
- package/build/prompts/test-maintenance/uiDriftAnalysisSections.d.ts +32 -0
- package/build/prompts/test-maintenance/uiDriftAnalysisSections.js +212 -0
- package/build/prompts/test-recommendation/recommendationShared.js +1 -1
- package/build/prompts/test-recommendation/scopeAssessment.js +12 -2
- package/build/prompts/testbot/testbot-prompts.js +9 -14
- package/build/resources/testbotResource.js +1 -1
- package/build/services/AnalyticsService.d.ts +0 -4
- package/build/services/AnalyticsService.js +0 -16
- package/build/services/ScenarioGenerationService.d.ts +0 -2
- package/build/services/ScenarioGenerationService.js +0 -116
- package/build/services/ScenarioGenerationService.test.js +0 -30
- package/build/services/TestDiscoveryService.d.ts +12 -10
- package/build/services/TestDiscoveryService.js +137 -83
- package/build/services/TestDiscoveryService.test.js +370 -15
- package/build/tool-phases.js +1 -0
- package/build/tools/executeSkyrampTestTool.js +42 -96
- package/build/tools/generateEnrichedIntegrationTestTool.d.ts +1 -0
- package/build/tools/generateEnrichedIntegrationTestTool.js +10 -1
- package/build/tools/localDevWorkerComposeTool.d.ts +23 -0
- package/build/tools/localDevWorkerComposeTool.js +191 -0
- package/build/tools/localDevWorkerComposeTool.test.d.ts +1 -0
- package/build/tools/localDevWorkerComposeTool.test.js +121 -0
- package/build/tools/submitReportTool.d.ts +11 -11
- package/build/tools/submitReportTool.js +123 -34
- package/build/tools/submitReportTool.test.js +413 -94
- package/build/tools/test-management/actionsTool.js +213 -80
- package/build/tools/test-management/actionsTool.test.d.ts +1 -0
- package/build/tools/test-management/actionsTool.test.js +370 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +9 -0
- package/build/tools/test-management/analyzeChangesTool.js +171 -190
- package/build/tools/test-management/analyzeChangesTool.test.js +55 -3
- package/build/tools/test-management/analyzeTestHealthTool.js +63 -22
- package/build/tools/test-management/analyzeTestHealthTool.test.d.ts +1 -0
- package/build/tools/test-management/analyzeTestHealthTool.test.js +268 -0
- package/build/tools/trace/startTraceCollectionTool.js +2 -15
- package/build/tools/workspace/initializeWorkspaceTool.js +35 -15
- package/build/types/FrontendIntegration.d.ts +25 -0
- package/build/types/FrontendIntegration.js +19 -0
- package/build/types/RepositoryAnalysis.d.ts +12 -12
- package/build/types/TestAnalysis.d.ts +18 -49
- package/build/types/TestAnalysis.js +0 -28
- package/build/types/TestTypes.d.ts +3 -1
- package/build/types/TestTypes.js +3 -1
- package/build/types/TestbotReport.d.ts +1 -1
- package/build/utils/AnalysisStateManager.d.ts +16 -35
- package/build/utils/AnalysisStateManager.js +0 -37
- package/build/utils/docker.test.js +1 -1
- package/build/utils/featureFlags.d.ts +2 -5
- package/build/utils/featureFlags.js +3 -7
- package/build/utils/featureFlags.test.js +2 -2
- package/build/utils/frontendIntegration.d.ts +9 -0
- package/build/utils/frontendIntegration.js +222 -0
- package/build/utils/frontendIntegration.test.d.ts +1 -0
- package/build/utils/frontendIntegration.test.js +229 -0
- package/build/utils/importerHop.d.ts +123 -0
- package/build/utils/importerHop.js +482 -0
- package/build/utils/importerHop.test.d.ts +1 -0
- package/build/utils/importerHop.test.js +399 -0
- package/build/utils/initAgent.d.ts +0 -2
- package/build/utils/initAgent.js +0 -1
- package/build/utils/pathAffinityClassification.d.ts +49 -0
- package/build/utils/pathAffinityClassification.js +180 -0
- package/build/utils/pathAffinityClassification.test.d.ts +1 -0
- package/build/utils/pathAffinityClassification.test.js +208 -0
- package/build/utils/pathSignatures.d.ts +6 -0
- package/build/utils/pathSignatures.js +33 -0
- package/build/utils/pr-comment-parser.d.ts +3 -3
- package/build/utils/pr-comment-parser.js +6 -6
- package/build/utils/pr-comment-parser.test.js +3 -3
- package/build/utils/progress.d.ts +16 -0
- package/build/utils/progress.js +25 -0
- package/build/utils/progress.test.d.ts +1 -0
- package/build/utils/progress.test.js +37 -0
- package/build/utils/projectMetadata.js +294 -83
- package/build/utils/projectMetadata.test.js +91 -0
- package/build/utils/proxy-terminal.js +0 -9
- package/build/utils/repoScanner.d.ts +7 -0
- package/build/utils/repoScanner.js +14 -6
- package/build/utils/repoScanner.test.js +9 -9
- package/build/utils/routeParsers.d.ts +40 -0
- package/build/utils/routeParsers.js +292 -2
- package/build/utils/routeParsers.test.js +394 -29
- package/build/utils/scenarioDrafting.d.ts +21 -0
- package/build/utils/scenarioDrafting.js +84 -1
- package/build/utils/scenarioDrafting.test.js +59 -1
- package/build/utils/sourceRouteExtractor.js +174 -5
- package/build/utils/sourceRouteExtractor.test.js +173 -0
- package/build/utils/utils.d.ts +9 -6
- package/build/utils/utils.js +16 -9
- package/build/utils/utils.test.js +23 -1
- package/build/utils/versions.d.ts +4 -3
- package/build/utils/versions.js +3 -1
- package/node_modules/playwright/._ThirdPartyNotices.txt +0 -0
- package/node_modules/playwright/._cli.js +0 -0
- package/node_modules/playwright/._index.d.ts +0 -0
- package/node_modules/playwright/._index.js +0 -0
- package/node_modules/playwright/._index.mjs +0 -0
- package/node_modules/playwright/._jsx-runtime.js +0 -0
- package/node_modules/playwright/._jsx-runtime.mjs +0 -0
- package/node_modules/playwright/._lib +0 -0
- package/node_modules/playwright/._node_modules +0 -0
- package/node_modules/playwright/._package.json +0 -0
- package/node_modules/playwright/._test.d.ts +0 -0
- package/node_modules/playwright/._test.js +0 -0
- package/node_modules/playwright/._test.mjs +0 -0
- package/node_modules/playwright/._types +0 -0
- package/node_modules/playwright/lib/mcp/browser/tab.js +16 -1
- package/node_modules/playwright/lib/mcp/browser/tools/__stubs__/mcpBundle.js +79 -0
- package/node_modules/playwright/lib/mcp/browser/tools/files.js +28 -10
- package/node_modules/playwright/lib/mcp/browser/tools/files.test.js +132 -0
- package/node_modules/playwright/lib/mcp/browser/tools/wait.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/exportTool.js +10 -1
- package/node_modules/playwright/lib/mcp/skyramp/tableSelectorBuilder.js +2 -1
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +52 -11
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +12 -1
- package/node_modules/playwright/lib/mcp/test/skyRampExport.test.js +44 -0
- package/node_modules/playwright/node_modules/playwright-core/._.DS_Store +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._.npmignore +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._README.md +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._ThirdPartyNotices.txt +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._bin +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._browsers.json +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._bundles +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._cli.js +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._index.d.ts +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._index.js +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._index.mjs +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._lib +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._package.json +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._src +0 -0
- package/node_modules/playwright/node_modules/playwright-core/._types +0 -0
- package/node_modules/playwright/node_modules/playwright-core/ThirdPartyNotices.txt +126 -23
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.B_7ywgmr.js → index.d4gkVSou.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +3 -3
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import type { OneClickCommandId, OneClickCommandDef } from "../types/OneClickCommands.js";
|
|
2
|
-
export declare function getCommand(id: OneClickCommandId): OneClickCommandDef;
|
|
3
2
|
export declare function getAllCommands(): OneClickCommandDef[];
|
|
4
3
|
export declare function getCommandIds(): OneClickCommandId[];
|
|
5
4
|
/**
|
|
@@ -18,13 +18,6 @@ function getCommandLibrary() {
|
|
|
18
18
|
}
|
|
19
19
|
return commands;
|
|
20
20
|
}
|
|
21
|
-
export function getCommand(id) {
|
|
22
|
-
const cmd = getCommandLibrary()[id];
|
|
23
|
-
if (!cmd) {
|
|
24
|
-
throw new Error(`Unknown one-click command: ${id}`);
|
|
25
|
-
}
|
|
26
|
-
return cmd;
|
|
27
|
-
}
|
|
28
21
|
export function getAllCommands() {
|
|
29
22
|
return Object.values(getCommandLibrary()).filter((cmd) => Boolean(cmd));
|
|
30
23
|
}
|
|
@@ -48,7 +48,7 @@ const localDevTestChangesSteps = [
|
|
|
48
48
|
File to check/create only if the tool did not already install it:
|
|
49
49
|
- Cursor: \`.cursor/skills/local-dev/SKILL.md\`
|
|
50
50
|
- Claude Code: \`.claude/skills/local-dev/SKILL.md\`
|
|
51
|
-
Agents without repo-local skill support skip skill creation. Also ensure the MCP config has the required env vars set (SKYRAMP_FEATURE_LOCAL_DEV=1
|
|
51
|
+
Agents without repo-local skill support skip skill creation. Also ensure the MCP config has the required env vars set (SKYRAMP_FEATURE_LOCAL_DEV=1).`,
|
|
52
52
|
conditionalGuidance: `Create the skill file for the calling agent ONLY if it doesn't already exist — never overwrite.
|
|
53
53
|
|
|
54
54
|
**Skill path for the calling agent only:**
|
|
@@ -65,7 +65,7 @@ ${LOCAL_DEV_SKILL_CONTENT}
|
|
|
65
65
|
- For VS Code: add to the Skyramp MCP server's VS Code MCP config env block:
|
|
66
66
|
\`"SKYRAMP_FEATURE_LOCAL_DEV": "1"\`
|
|
67
67
|
|
|
68
|
-
|
|
68
|
+
After updating the config, the user must restart the IDE or reload MCP servers for the change to take effect.
|
|
69
69
|
|
|
70
70
|
After verifying or creating files, inform the user what was installed.`,
|
|
71
71
|
},
|
|
@@ -85,7 +85,7 @@ After verifying or creating files, inform the user what was installed.`,
|
|
|
85
85
|
{
|
|
86
86
|
stepIndex: 2,
|
|
87
87
|
title: "Invoke the skyramp_local_dev prompt",
|
|
88
|
-
description: "Call the `skyramp_local_dev` MCP prompt to run the full local-dev testing workflow.\n\nPass these parameters:\n- `baseCommit`: the one-click `baseBranch` value (default `main`)\n- `repositoryPath`: absolute path to the repo\n- `servicesToMock`: comma-separated list only when the user explicitly named services to mock\n- `realServices`: comma-separated list only when the user explicitly named services that should receive real traffic\n- `mockProtocols`: comma-separated list only when the user explicitly constrained mocked protocols (for example, `rest`)\n\nThe one-click tool exposes `baseBranch` because that matches other workflow inputs; the `skyramp_local_dev` prompt argument is named `baseCommit`. Always map `baseBranch` → `baseCommit` when invoking the prompt.\n\nThe workflow generates the highest-value tests regardless of type. For integration scenarios with selected downstream mocks: skyramp_mock_generation → skyramp_integration_test_generation → skyramp_enrich_test_with_mocks. For contract scenarios: skyramp_contract_test_generation only;
|
|
88
|
+
description: "Call the `skyramp_local_dev` MCP prompt to run the full local-dev testing workflow.\n\nPass these parameters:\n- `baseCommit`: the one-click `baseBranch` value (default `main`)\n- `repositoryPath`: absolute path to the repo\n- `servicesToMock`: comma-separated list only when the user explicitly named services to mock\n- `realServices`: comma-separated list only when the user explicitly named services that should receive real traffic\n- `mockProtocols`: comma-separated list only when the user explicitly constrained mocked protocols (for example, `rest`)\n\nThe one-click tool exposes `baseBranch` because that matches other workflow inputs; the `skyramp_local_dev` prompt argument is named `baseCommit`. Always map `baseBranch` → `baseCommit` when invoking the prompt.\n\nThe workflow generates the highest-value tests regardless of type. For integration scenarios with selected downstream mocks: skyramp_mock_generation → skyramp_integration_test_generation → skyramp_enrich_test_with_mocks. For provider contract scenarios: skyramp_contract_test_generation only; no mocks needed. Integration enrich is MANDATORY when the selected scenario has downstream mocks — it embeds MOCK_SERVICES into the test file.\n\nFollow the returned prompt instructions EXACTLY — do not skip steps or freelance.",
|
|
89
89
|
conditionalGuidance: renderFallbackGuidance(),
|
|
90
90
|
},
|
|
91
91
|
];
|
|
@@ -135,7 +135,7 @@ Read <REPOSITORY_PATH>/.skyramp/workspace.yml before classifying dependencies. U
|
|
|
135
135
|
## Key Principles
|
|
136
136
|
- ALL output paths absolute inside repo. NEVER /tmp.
|
|
137
137
|
- For **integration tests**: call \`skyramp_enrich_test_with_mocks\` after generating each test that uses selected downstream mocks. Each integration test with mocked dependencies MUST have MOCK_SERVICES embedded for third-party mocks or user-requested mocks.
|
|
138
|
-
- For **contract tests**: mocks
|
|
138
|
+
- For **provider contract tests**: no mocks needed — test hits the real endpoint directly. Do NOT call \`skyramp_enrich_test_with_mocks\` for contract tests.
|
|
139
139
|
- Prioritize tests that catch real bugs and edge cases; test type is secondary to coverage value.
|
|
140
140
|
- Present results directly to the user (no skyramp_submit_report).
|
|
141
141
|
- **Prefer batch operations**: Use \`skyramp_batch_mock_generation\` over individual mock calls.
|
package/build/index.js
CHANGED
|
@@ -37,6 +37,7 @@ import { registerOneClickTool } from "./tools/one-click/oneClickTool.js";
|
|
|
37
37
|
import { registerEnrichTestWithMocksTool } from "./tools/enrichTestWithMocksTool.js";
|
|
38
38
|
import { registerPreflightMockCheckTool } from "./tools/preflightMockCheckTool.js";
|
|
39
39
|
import { registerGenerateEnrichedIntegrationTestTool } from "./tools/generateEnrichedIntegrationTestTool.js";
|
|
40
|
+
import { registerLocalDevWorkerComposeTool } from "./tools/localDevWorkerComposeTool.js";
|
|
40
41
|
import { registerAnalysisResources } from "./resources/analysisResources.js";
|
|
41
42
|
import { registerProgressResource } from "./resources/progressResource.js";
|
|
42
43
|
import { AnalyticsService } from "./services/AnalyticsService.js";
|
|
@@ -116,7 +117,7 @@ Before calling ANY test generation tool, you MUST follow this flow:
|
|
|
116
117
|
4. **CRITICAL — endpointURL**: The \`endpointURL\` parameter MUST be the full URL to the specific endpoint being tested, NOT just the base URL. Construct it by combining \`api.baseUrl\` with the endpoint path. Example: if \`api.baseUrl\` is \`http://localhost:8000\` and the endpoint is \`/api/v1/products\`, pass \`endpointURL: "http://localhost:8000/api/v1/products"\`. NEVER pass just the base URL (e.g. \`http://localhost:8000\`) as \`endpointURL\`.
|
|
117
118
|
5. **CRITICAL — scenario generation**: When calling \`skyramp_batch_scenario_test_generation\`, ALWAYS pass:
|
|
118
119
|
- \`baseURL\`: The full base URL from \`api.baseUrl\` (e.g., \`http://localhost:3000\`). This determines the scheme, host, and port in the generated trace. Without it, the trace defaults to https:443 which is almost always wrong for local development.
|
|
119
|
-
- \`authHeader\`: Which HTTP header carries the auth credential. Get it from \`api.authHeader\` in workspace config. Examples: \`Authorization\` (Bearer/Token auth), \`X-Api-Key\` (API key auth), \`Cookie\` (session/cookie auth like NextAuth). Pass \`""\` to
|
|
120
|
+
- \`authHeader\`: Which HTTP header carries the auth credential. Get it from \`api.authHeader\` in workspace config. Examples: \`Authorization\` (Bearer/Token auth), \`X-Api-Key\` (API key auth), \`Cookie\` (session/cookie auth like NextAuth). Pass \`""\` to document the endpoint as unauthenticated in the generated test's assertions (unauthenticated endpoints or \`api.authType: "none"\`).
|
|
120
121
|
- \`authScheme\`: Only when \`authHeader\` is \`Authorization\`. The prefix before the token (e.g., \`"Bearer"\` → \`Authorization: Bearer <token>\`). **Derive from**: (1) OpenAPI spec \`securitySchemes\`/\`securityDefinitions\`, (2) source code auth middleware, (3) workspace \`api.authType\`. **Do NOT guess.**
|
|
121
122
|
- \`authToken\`: The full header value, used verbatim. When omitted, \`SKYRAMP_PLACEHOLDER_TOKEN\` is auto-generated. Only provide when the header needs a specific format (e.g., \`"session=${AUTH_PLACEHOLDER_TOKEN}"\` for Cookie). **Do NOT fabricate token values.**
|
|
122
123
|
- \`apiSchema\` is OPTIONAL — omit it for code-first apps without OpenAPI specs.
|
|
@@ -198,7 +199,7 @@ const infrastructureTools = [
|
|
|
198
199
|
registerTraceStopTool,
|
|
199
200
|
];
|
|
200
201
|
if (isLocalDevEnabled()) {
|
|
201
|
-
infrastructureTools.push(registerEnrichTestWithMocksTool, registerPreflightMockCheckTool, registerGenerateEnrichedIntegrationTestTool);
|
|
202
|
+
infrastructureTools.push(registerEnrichTestWithMocksTool, registerPreflightMockCheckTool, registerGenerateEnrichedIntegrationTestTool, registerLocalDevWorkerComposeTool);
|
|
202
203
|
logger.info("Local-dev tools enabled via SKYRAMP_FEATURE_LOCAL_DEV");
|
|
203
204
|
}
|
|
204
205
|
if (isTestbotEnabled()) {
|
|
@@ -57,6 +57,8 @@ export async function registerPlaywrightTools(server, options) {
|
|
|
57
57
|
'browser_assert_table_cell',
|
|
58
58
|
'browser_mouse_action',
|
|
59
59
|
'browser_visual_snapshot',
|
|
60
|
+
'browser_gojs_add_node',
|
|
61
|
+
'browser_gojs_link',
|
|
60
62
|
'skyramp_export_zip',
|
|
61
63
|
'skyramp_load_trace',
|
|
62
64
|
// DOM Analyzer: blueprint capture (capture-act-capture pattern)
|
|
@@ -4,6 +4,33 @@ const UI_ASSERTION_CATEGORIES = [
|
|
|
4
4
|
{
|
|
5
5
|
name: "Critical UI Assertions",
|
|
6
6
|
rules: [
|
|
7
|
+
{
|
|
8
|
+
title: "Exercise the changed interaction (dynamic behavior)",
|
|
9
|
+
description: "When the PR introduces or changes interactive behavior (toggle, theme switch, expand/collapse, dialog open/close, focus management, optimistic update), the test MUST perform the triggering action and assert BOTH the state before AND after the action — not only the default or resting state. WHY: the PR's change IS the transition; a test that asserts only the initial/resting state would still pass even if the interaction were completely broken, so it gives zero coverage of what changed. If the test asserts only the resting state, add the action that triggers the change and assert the resulting transition — never mark this not applicable merely because the generated test lacks an action.",
|
|
10
|
+
subPoints: [
|
|
11
|
+
"Toggle / switch: click the control and assert the observable flips (e.g. data-theme 'light' -> 'dark'), then click again and assert it reverses.",
|
|
12
|
+
"Dialog / modal: open it and assert focus moves into the dialog; close via ESC or Cancel and assert focus returns to the trigger.",
|
|
13
|
+
"The after-state must differ from the before-state — a single assertion of the unchanged default value does not verify the change.",
|
|
14
|
+
],
|
|
15
|
+
examples: [
|
|
16
|
+
{
|
|
17
|
+
language: "javascript",
|
|
18
|
+
code: `await expect(page.locator('html')).toHaveAttribute('data-theme', 'light');
|
|
19
|
+
await page.getByTestId('theme-toggle').click();
|
|
20
|
+
await expect(page.locator('html')).toHaveAttribute('data-theme', 'dark');`,
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
// Python (playwright-python) — same before/after transition pattern
|
|
24
|
+
language: "python",
|
|
25
|
+
code: `trigger = page.get_by_test_id('delete-recipe')
|
|
26
|
+
trigger.click()
|
|
27
|
+
expect(page.get_by_role('dialog')).to_be_focused()
|
|
28
|
+
page.keyboard.press('Escape')
|
|
29
|
+
# focus returns to the trigger
|
|
30
|
+
expect(trigger).to_be_focused()`,
|
|
31
|
+
},
|
|
32
|
+
],
|
|
33
|
+
},
|
|
7
34
|
{
|
|
8
35
|
title: "Selector constraints",
|
|
9
36
|
description: "Every assertion uses a selector already in the file. Never invent `data-testid`, role names, or classes.",
|
|
@@ -240,7 +267,7 @@ Before editing the given test file, you must output a \`<thinking>\` block. The
|
|
|
240
267
|
3. Classify each in-scope action / selector / captured response by its applicable assertion category, marking each category APPLICABLE or NOT APPLICABLE.
|
|
241
268
|
- Critical UI Assertions applies when there is a collection / repeated element (\`toHaveCount\`), a pageerror handler, a captured network response, a negative-only or URL-only test needing a positive-path companion, or a tautological locator-by-text + assert-text to replace.
|
|
242
269
|
- Computed Values applies when there is a knowable exact text / value / attribute, including a definition-page displayed value or an image \`src\`.
|
|
243
|
-
- Post-edit State applies
|
|
270
|
+
- Post-edit State applies whenever EITHER (a) the test contains a state-changing action (form fill+submit, save / delete / create / toggle, checkbox click, hover / mouseout, refresh, reload, JS update, form edit), OR (b) the PR changes interactive behavior — in case (b), add the triggering action and assert the before/after transition (see "Exercise the changed interaction"). Mark NOT APPLICABLE only when the PR changes no interactive behavior and the test has no state-changing action.
|
|
244
271
|
4. For each in-scope item, output one JSON object using the template below. The output is an array — repeat the object template below once per in-scope item.
|
|
245
272
|
- \`action_or_selector_or_response\`: the selector, action, or captured network response this entry covers.
|
|
246
273
|
- \`assertion_categories\`: an object that MUST contain every category name below as a key. The value of each category is itself an object that MUST contain every rule title under that category as a key. For each rule, the value is an array of assertion lines you will add for this item under that rule. Use \`[]\` only when the rule does not apply to this item — every category key and every rule key must still be present. This forces you to consider every rule for every item.
|
|
@@ -97,6 +97,7 @@ Create one service entry per deployable unit. You MUST include every backend/API
|
|
|
97
97
|
<api_fields>
|
|
98
98
|
1. api.schemaPath: Path or URL to an OpenAPI or Swagger schema. Search for openapi.json, openapi.yaml, swagger.json, or swagger.yaml files. Framework defaults are: FastAPI serves /openapi.json, Express serves /api-docs, and Spring serves /v3/api-docs. For locally-run services, use a localhost URL. For cloud or externally hosted services (such as Salesforce, Vercel, or Cloudflare), use the actual deployment URL found in config or documentation.
|
|
99
99
|
2. api.baseUrl (required): The base URL where the service is reachable, such as "http://localhost:3000" or "https://api.example.com". Derive from docker-compose ports, app config, README, or environment variables. Use localhost for services run locally and the actual deployment URL for cloud or externally hosted services. NEVER fabricate a URL. Only use URLs found in config files, README, or environment variables.
|
|
100
|
+
Include the API path prefix (e.g. \`/api\`, \`/api/v1\`, \`/v1\`) when one applies — not just \`host:port\`. A bare host means every test request hits the root and 404s. Derive the prefix from the strongest available signal: OpenAPI \`servers[].url\` or \`basePath\`; the app's routing config (FastAPI \`root_path\`, Express \`app.use('/api/v1', ...)\`, Rails routes prefix, Django \`urlpatterns\` mount); or the longest common leading path across the existing test suite's endpoints. If none of those yield a prefix, leave it bare AND flag it in your summary so the user can review.
|
|
100
101
|
3. api.authType (required): The authentication type. Valid values are: ${AUTH_TYPES_PROMPT_LIST}
|
|
101
102
|
Detect by checking in the following order (language-agnostic, apply whichever signals match):
|
|
102
103
|
a. Dependencies and packages (package.json, requirements.txt, go.mod, Gemfile, composer.json, pom.xml, build.gradle):
|
|
@@ -99,7 +99,6 @@ For each endpoint under test:
|
|
|
99
99
|
|
|
100
100
|
5. **For API and downstream contracts**, include scenarios only when they catch a realistic bug:
|
|
101
101
|
- Provider contract: response field added/renamed/removed, type/nullable mismatch, status-code semantic drift.
|
|
102
|
-
- Consumer contract: our code sends the wrong downstream request body/header/path, or parses a response field that may be missing/renamed.
|
|
103
102
|
|
|
104
103
|
6. **For full API flows**, treat API E2E coverage as integration coverage:
|
|
105
104
|
- **API full-flow coverage is integration coverage** in this workflow. Do NOT label API-only workflows as E2E.
|
|
@@ -122,9 +121,8 @@ For each endpoint under test:
|
|
|
122
121
|
| 1 | Order creation applies downstream pricing rules | integration | happy-path-with-pricing | All required downstream success | 201 + calculated totals | high |
|
|
123
122
|
| 2 | Payment rejection rolls back order creation | integration | payment-rejects | Payment service → 422 | expected error + no persisted order | high |
|
|
124
123
|
| 3 | Public order response shape does not regress | provider-contract | order-response-shape | N/A | required fields typed | medium |
|
|
125
|
-
| 4 |
|
|
126
|
-
| 5 |
|
|
127
|
-
| 6 | Frontend checkout flow still works after UI changes | ui | checkout-ui-flow | Browser trace + applicable API mocks | visible success/error state | high when frontend changed |
|
|
124
|
+
| 4 | Complete API order workflow remains coherent | integration | order-creation-full-flow | All downstreams for the cross-service workflow | final response + side effects observed | high |
|
|
125
|
+
| 5 | Frontend checkout flow still works after UI changes | ui | checkout-ui-flow | Browser trace + applicable API mocks | visible success/error state | high when frontend changed |
|
|
128
126
|
| ...| ... | ... | ... | ... | ... | ... |
|
|
129
127
|
|
|
130
128
|
**IMPORTANT:** Only proceed to the next phase once you have the complete scenario list. The total number should reflect distinct bug-catching outcomes and API boundaries — skip redundant tests even if they are a different type.`)
|
|
@@ -146,7 +144,7 @@ For each endpoint under test:
|
|
|
146
144
|
|
|
147
145
|
Work through the scenario list from Phase 2 in **bug-catching priority order**. The budget is ${ctx.maxGenerate} total tests across all types.
|
|
148
146
|
|
|
149
|
-
**Selection rule:** Generate the tests most likely to catch real regressions and edge-case bugs. Choose integration, provider contract,
|
|
147
|
+
**Selection rule:** Generate the tests most likely to catch real regressions and edge-case bugs. Choose integration, provider contract, or UI based on which one best proves the behavior. There is no required quota by test type. If the highest-value ${ctx.maxGenerate} scenarios are all integration tests, generate integrations; if focused provider contracts catch the riskiest issues more directly, generate contracts. For API-only full-flow coverage, choose integration and model the whole scenario in one scenario trace; reserve UI tests for frontend changes or browser-observable flows.
|
|
150
148
|
|
|
151
149
|
**Coverage floor:** If Phase 2 identifies multiple high-priority scenarios, generate and execute each high-priority scenario up to ${ctx.maxGenerate}. Do not stop after the first passing test while other high-priority scenarios remain ungenerated or unexecuted.
|
|
152
150
|
|
|
@@ -229,23 +227,6 @@ Provider contracts assert: status codes, required fields present, field types co
|
|
|
229
227
|
|
|
230
228
|
---
|
|
231
229
|
|
|
232
|
-
**For \`consumer-contract\` scenarios:**
|
|
233
|
-
|
|
234
|
-
Call \`skyramp_contract_test_generation\` with:
|
|
235
|
-
- \`endpointURL\`: the downstream service URL using the original Docker service hostname (e.g., "http://identity-service:4000/api/internal/v1/users/validate")
|
|
236
|
-
- \`method\`: what OUR code sends
|
|
237
|
-
- \`requestData\`: the request body OUR code constructs (from reading the implementation)
|
|
238
|
-
- \`responseData\`: what OUR code expects back (the shape it parses)
|
|
239
|
-
- \`outputDir\`: "${ctx.repositoryPath}/tests/skyramp/"
|
|
240
|
-
|
|
241
|
-
Consumer contract tests have mocks wired **inline by default** — the generated test file includes \`MockV2\` creation and \`client.apply_mock(mock)\` directly in the test function. **No separate \`skyramp_enrich_test_with_mocks\` call is needed for contract tests.** The mock defines the expected downstream response shape and the test validates the contract against it.
|
|
242
|
-
|
|
243
|
-
Consumer contract tests are incomplete until they invoke real consumer/SUT code that triggers the outbound request. A generated stub that calls \`client.send_request\` directly against the mock only proves the mock responds; it does not prove the application sends the right body/header/path. If you cannot wire real consumer code, do not execute or report the contract as PASS — mark the scenario **Blocked — consumer wiring unavailable**.
|
|
244
|
-
|
|
245
|
-
Consumer contracts catch: wrong outbound request body/header/path, field renamed downstream, new required header added, response shape changed, status code semantics changed.
|
|
246
|
-
|
|
247
|
-
---
|
|
248
|
-
|
|
249
230
|
**For \`ui\` scenarios:**
|
|
250
231
|
|
|
251
232
|
Generate UI tests only when the diff includes frontend changes, a changed UI route, or a browser-observable user flow:
|
|
@@ -280,7 +261,7 @@ Keep advancing in bug-catching priority order until the highest-value scenarios
|
|
|
280
261
|
|
|
281
262
|
This injects import statements from the generated mock files — no data duplication. The test imports directly from the mock files.
|
|
282
263
|
|
|
283
|
-
**Skip contract tests** —
|
|
264
|
+
**Skip contract tests** — provider contract tests don't use mocks at all.`)
|
|
284
265
|
.done()
|
|
285
266
|
// ─── Phase 4: Verify ───────────────────────────────────────────────────────
|
|
286
267
|
.addPhase("verify", "Verify Tests", { headerLevel: "##", stepFormat: "hash" })
|
|
@@ -288,13 +269,12 @@ This injects import statements from the generated mock files — no data duplica
|
|
|
288
269
|
|
|
289
270
|
1. Every scenario from Phase 2 has a corresponding test file in \`${ctx.repositoryPath}/tests/skyramp/\`
|
|
290
271
|
2. For integration tests with selected downstream mocks: mock files exist in \`mocks/\` for each selected service/protocol and test files import them via a non-empty \`MOCK_SERVICES\` plus the language-specific apply helper (\`apply_all_mocks(client)\` for Python or \`applyAllMocks(client)\` for TypeScript/JavaScript/Java); if \`MOCK_SERVICES = {}\` or any expected mock import is missing for a mocked dependency, return to \`GENERATE_MOCKS_AND_TESTS\` or \`ENRICH_TESTS\` before any edits or execution
|
|
291
|
-
3. For
|
|
292
|
-
4.
|
|
293
|
-
5.
|
|
294
|
-
6.
|
|
295
|
-
7. **
|
|
296
|
-
8. **
|
|
297
|
-
9. **No unresolved stubs** — no generated test file contains \`TODO\`, a standalone \`pass\` statement in a test body, \`NotImplemented\`, "replace with real consumer code", or an executable consumer-contract test that only calls the mock directly. A skipped or strict xfail direct-mock reference is allowed only when the file also contains a separate executable real consumer/SUT test. These are blocking defects, not passing tests.
|
|
272
|
+
3. For provider contract tests: no mocks — test hits the real endpoint directly
|
|
273
|
+
4. Test files use idempotent data (UUID-suffixed emails, random values — not hardcoded strings)
|
|
274
|
+
5. Auth headers are present in all test requests
|
|
275
|
+
6. **Integration tests have scenario-driven assertions** — each test asserts the specific response fields that prove the "What this verifies" column from the scenario table, not only the status code. If a test only asserts \`response.status_code == 201\` (or equivalent) without any field-level assertion on the behavior being tested, it is incomplete. Add \`skyramp.get_response_value()\` assertions for the key fields before executing.
|
|
276
|
+
7. **Outbound side effects are verified** — for every selected REST/gRPC/Kafka mock whose request body/header/path/event payload is the behavior under test, the mock spec or test includes a request/event assertion. If the toolchain cannot express that assertion, mark the scenario blocked instead of treating a response-only test as coverage.
|
|
277
|
+
8. **No unresolved stubs** — no generated test file contains \`TODO\`, a standalone \`pass\` statement in a test body, \`NotImplemented\`, or "replace with real consumer code". These are blocking defects, not passing tests.
|
|
298
278
|
|
|
299
279
|
**If any mock file is missing for integration tests:** re-run \`skyramp_mock_generation\` for that service/endpoint.`)
|
|
300
280
|
.step("ENHANCE_ASSERTIONS", "Enhance assertions for each test file", (ctx) => `For each test file in \`${ctx.repositoryPath}/tests/skyramp/\`, call \`skyramp_enhance_assertions\` with:
|
|
@@ -318,23 +298,24 @@ The tool returns instructions — it does NOT edit the file automatically. Apply
|
|
|
318
298
|
.done()
|
|
319
299
|
// ─── Phase 5: Deploy ────────────────────────────────────────────────────────
|
|
320
300
|
.addPhase("deploy", "Deploy", { headerLevel: "##", stepFormat: "hash" })
|
|
321
|
-
.step("START_ENVIRONMENT", "Bring up the service under test", (ctx) => `
|
|
301
|
+
.step("START_ENVIRONMENT", "Bring up the service under test and the Skyramp worker", (ctx) => `Bring up the service under test and its real infrastructure (DB, cache, real broker, real first-party services) together with the Skyramp mock worker — **but mocked downstream services must NOT be running.**
|
|
322
302
|
|
|
323
|
-
**Why this matters:** The Skyramp worker intercepts traffic via Docker DNS alias hijacking — it
|
|
303
|
+
**Why this matters:** The Skyramp worker runs as a Docker Compose service on the SUT's network and intercepts traffic via Docker DNS alias hijacking at mock-apply time — it takes over the DNS alias for each mocked service's hostname. If the real service container is also up on the same hostname and port, the two conflict: traffic may reach the real service instead of the mock, or port binding fails. This is the most common cause of mocks not being applied correctly. Running the worker as a compose service (rather than spawning it ad hoc) guarantees it joins the SUT's network and is torn down with the stack.
|
|
324
304
|
|
|
325
305
|
Steps:
|
|
326
|
-
1.
|
|
327
|
-
2.
|
|
328
|
-
3.
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
306
|
+
1. Call \`skyramp_setup_local_dev_worker\` with \`repositoryPath\` = \`${ctx.repositoryPath}\` (pass \`composeFile\` if the base docker-compose file is not at the repo root). It writes a worker compose override and returns \`composeFilePrefix\` (the \`-f <base> -f <override>\` flags), the resolved \`dockerNetwork\`, and the worker port. **Use the returned \`composeFilePrefix\` for every \`docker compose\` command below, and pass the returned \`dockerNetwork\` to test/mock generation (see the Generate phase).**
|
|
307
|
+
2. Run \`docker compose <composeFilePrefix> ps\` to check current container state.
|
|
308
|
+
3. Identify which docker-compose service names correspond to services selected for mocking (e.g., if \`ams\` is selected for mocking, do not start the \`ams\` container).
|
|
309
|
+
4. Stop any mocked service containers that are already running:
|
|
310
|
+
\`docker compose <composeFilePrefix> stop <mocked-service-1> <mocked-service-2> ...\`
|
|
311
|
+
5. Rebuild and bring up only the SUT, its real dependencies, and the worker — start selectively:
|
|
312
|
+
\`docker compose <composeFilePrefix> up -d --build <sut-service> <db> <cache> <broker> <real-first-party-services> skyramp\`
|
|
313
|
+
Always include the \`skyramp\` service. Do **not** run plain \`docker compose up -d\` when it would start mocked downstream services or reuse a stale SUT image. If the compose file cannot start selectively, start the minimum required profile/services plus \`skyramp\`, then re-check that mocked service containers are stopped before executing tests. If the SUT service has no compose \`build\` context, record that local code cannot be rebuilt by compose before execution.
|
|
314
|
+
6. Wait for health checks to pass and verify (a) the service under test responds (e.g., curl the health endpoint) and (b) the \`skyramp\` worker container is running.
|
|
315
|
+
|
|
316
|
+
The \`skyramp_execute_test\` tool runs the enriched test in the language runner. Its generated Skyramp client connects to the worker — emitted with \`runtime="docker"\`, \`docker_network\`, and \`docker_skyramp_port\` (from the \`dockerNetwork\` and \`dockerWorkerPort\` passed to generation) — and applies the selected mocks before exercising the service.
|
|
317
|
+
|
|
318
|
+
**Do NOT proceed to Execute until: (a) the service under test responds, (b) the \`skyramp\` worker is running, and (c) all mocked service containers are confirmed stopped.**`)
|
|
338
319
|
.subStep("DEPLOY_GRPC", "Handle gRPC dependencies", (_ctx) => `If the dependency map from Phase 1 includes gRPC services selected for mocking:
|
|
339
320
|
|
|
340
321
|
To regenerate a single missing gRPC mock, call \`skyramp_mock_generation\` with \`protocol: "grpc"\`.
|
|
@@ -414,6 +395,7 @@ That's it. Pass \`token: ""\` so the executor injects \`SKYRAMP_TEST_TOKEN\` fro
|
|
|
414
395
|
- **Generated test issue** — failure from wrong auth, missing field, or incorrect assertion in the generated test itself.
|
|
415
396
|
- **Timeout** — execution did not complete; pass/fail is unknown.
|
|
416
397
|
- **Issues found**: unmockable deps, generation failures, connection errors, gaps
|
|
398
|
+
- **Teardown**: remind the user that, when finished, tearing the stack down with \`docker compose <composeFilePrefix> down -v --remove-orphans\` (the same \`-f\` prefix returned by \`skyramp_setup_local_dev_worker\`) removes the SUT, real dependencies, **and the Skyramp worker** together — so no worker container or network is left behind. A plain \`docker compose down\` *without* the override prefix leaves the worker running (it is defined only in the override, so a base-only \`down\` treats it as an orphan) and the network fails to delete with "resource still in use"; always include the prefix. If the prefix is ever unavailable at teardown, \`docker compose down --remove-orphans\` against the base file still sweeps the worker as an orphan.
|
|
417
399
|
|
|
418
400
|
Do NOT write results to a file — report inline to the user.`)
|
|
419
401
|
.done();
|
|
@@ -58,11 +58,12 @@ ${sandboxWorkerBlock ? sandboxWorkerBlock + "\n" : ""}${serviceContext ? service
|
|
|
58
58
|
- **Mock files are code in the same language as tests** — \`skyramp_mock_generation\` produces mock files (language from workspace config) in \`tests/skyramp/mocks/\` with language-native mock factory functions (\`get_all_mocks()\` for Python, \`getAllMocks()\` for TypeScript/JavaScript/Java) that return MockV2 objects. The enrichment tool imports these, builds a collective \`MOCK_SERVICES\` collection, reuses the integration test client, clears stale mocks, and applies the full mock set in one atomic call. Do NOT create YAML mock files.
|
|
59
59
|
- **NO Skyramp CLI** — do NOT use \`skyramp mocker apply\`, \`skyramp mocker generate\`, or any other Skyramp CLI command. Use ONLY MCP tools for Skyramp operations.
|
|
60
60
|
- **NO duplicate mock data** — the generated mock files are the single source of truth. Do NOT duplicate response bodies or status codes in the test file. The test references mock files, not copies of them.
|
|
61
|
-
- **Mock generation → enrich flow (integration tests only)** — use \`skyramp_mock_generation\` to generate mock files, then call \`skyramp_enrich_test_with_mocks\` to wire mock references into the test. The enrichment uses the same data from generation — never invent different values.
|
|
61
|
+
- **Mock generation → enrich flow (integration tests only)** — use \`skyramp_mock_generation\` to generate mock files, then call \`skyramp_enrich_test_with_mocks\` to wire mock references into the test. The enrichment uses the same data from generation — never invent different values.
|
|
62
62
|
- **MANDATORY enrichment gate** — after every \`skyramp_integration_test_generation\` call for a scenario with any generated mock files, immediately call \`skyramp_enrich_test_with_mocks\` before reading, editing, enhancing, or executing that test. Do not batch all enrichment for later.
|
|
63
63
|
- **Enrichment validation** — an integration test that has downstream mocks but lacks \`MOCK_SERVICES\` and the language-specific apply helper (\`apply_all_mocks(client)\` for Python or \`applyAllMocks(client)\` for TypeScript/JavaScript/Java) is incomplete. Stop and enrich it; never call \`skyramp_execute_test\` on an incomplete integration test.
|
|
64
64
|
- **Do not delete mocks to pass** — if mock deployment fails, do NOT remove mock imports, empty \`MOCK_SERVICES\`, switch mocked downstreams to live Compose stubs, or report the resulting test as passed. Fix the mock wiring once; if still blocked, report the scenario as failed/blocked with the mock deployment error.
|
|
65
65
|
- **Mocked services must not be running** — before execution, stop every Docker container for a service selected for mocking (\`docker compose stop <mocked-service>\`). The Skyramp worker takes over that service's DNS alias on the Docker network; a real container running on the same hostname and port will conflict, causing traffic to reach the real service instead of the mock or causing port binding failure. Bring up only the SUT and its real infrastructure (DB, cache, real broker, real first-party services); do NOT run \`docker compose up -d\` without immediately stopping mocked service containers.
|
|
66
|
+
- **Worker runs as a compose service** — the Deploy phase calls \`skyramp_setup_local_dev_worker\`, which adds the Skyramp worker to the SUT's Docker network via a compose override and returns the resolved \`dockerNetwork\` plus the \`-f\` \`composeFilePrefix\`. Pass that \`dockerNetwork\` (and \`dockerWorkerPort\`, default 35142) to \`skyramp_generate_enriched_integration_test\` so the generated client connects to that worker — generation emits the client with \`runtime="docker"\`, \`docker_network\`, and \`docker_skyramp_port\`. Use the \`composeFilePrefix\` for every \`docker compose\` command, and tear the stack down with the same prefix (\`docker compose <composeFilePrefix> down -v --remove-orphans\`) so the worker is removed with the stack — a base-only \`docker compose down\` leaves the worker (it lives only in the override) holding the network open. Never spawn the worker manually.
|
|
66
67
|
- **Mock URL format** — the \`endpointURL\` in \`skyramp_mock_generation\` MUST use the original Docker service hostname (e.g., \`http://identity-service:4000\`, \`http://profile-service:50052\`). The Skyramp executor uses DNS alias hijacking — it takes over the service's DNS name on the Docker network so the service under test's requests are intercepted transparently. NEVER use \`localhost\`, \`127.0.0.1\`, \`0.0.0.0\`, \`host.docker.internal\`, or the worker address as the mock URL. For REST mocks, \`skyramp_preflight_mock_check\` returns blocking \`REST_LOOPBACK_URL\` when a deployable mock uses these hosts.
|
|
67
68
|
- **REST mock routing** — Generated REST mock files may contain a fallback \`URL\` value such as \`http://localhost:8080\`; do not manually edit those files. The enrichment tool uses the mock generation \`# Command\` target to set each REST mock's \`mock.url\` back to the original Docker service origin before \`apply_mock()\`. If the command target is loopback, regenerate the mock with the original service hostname.
|
|
68
69
|
- **gRPC mock routing** — gRPC mocks may target the real downstream service port (e.g., \`partner-accounts:50051\`). Apply gRPC mocks before the SUT starts or restart the SUT after applying them, and verify the Skyramp worker has the original endpoint host alias from \`endpointURL\` (\`partner-accounts\`, not the protobuf service name like \`PartnerAccountsService\`).
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Skyramp personas injected into tool descriptions and prompts.
|
|
3
3
|
*
|
|
4
|
-
* In
|
|
4
|
+
* In Testbot environments (SKYRAMP_FEATURE_TESTBOT=1), the persona is injected
|
|
5
5
|
* once via `claude --append-system-prompt` from the testbot GitHub Action
|
|
6
6
|
* (see testbot.git `src/agents/claude.ts` `loadQaPersona()`) rather than
|
|
7
7
|
* repeating it in every tool description. In that case getPersonaPrefix()
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
export declare const SKYRAMP_QA_PERSONA = "You are acting as a Skyramp QA Automation Engineer. Your responsibility is to translate user test intent into precise, deterministic test artifacts \u2014 whether generating API tests from specs, recording browser interactions for UI flows, or maintaining existing test suites. Derive all parameters strictly from the codebase, workspace config, API schemas, and page snapshots. Never guess or hallucinate values.";
|
|
15
15
|
/**
|
|
16
16
|
* Returns the persona prefix for use in tool descriptions.
|
|
17
|
-
* Returns an empty string when running inside
|
|
17
|
+
* Returns an empty string when running inside Testbot — the testbot action
|
|
18
18
|
* appends the persona via `claude --append-system-prompt` instead, so we
|
|
19
19
|
* avoid duplicating it in every tool description.
|
|
20
20
|
*/
|
|
@@ -2,7 +2,7 @@ import { isTestbotEnabled } from "../utils/featureFlags.js";
|
|
|
2
2
|
/**
|
|
3
3
|
* Skyramp personas injected into tool descriptions and prompts.
|
|
4
4
|
*
|
|
5
|
-
* In
|
|
5
|
+
* In Testbot environments (SKYRAMP_FEATURE_TESTBOT=1), the persona is injected
|
|
6
6
|
* once via `claude --append-system-prompt` from the testbot GitHub Action
|
|
7
7
|
* (see testbot.git `src/agents/claude.ts` `loadQaPersona()`) rather than
|
|
8
8
|
* repeating it in every tool description. In that case getPersonaPrefix()
|
|
@@ -15,7 +15,7 @@ import { isTestbotEnabled } from "../utils/featureFlags.js";
|
|
|
15
15
|
export const SKYRAMP_QA_PERSONA = `You are acting as a Skyramp QA Automation Engineer. Your responsibility is to translate user test intent into precise, deterministic test artifacts — whether generating API tests from specs, recording browser interactions for UI flows, or maintaining existing test suites. Derive all parameters strictly from the codebase, workspace config, API schemas, and page snapshots. Never guess or hallucinate values.`;
|
|
16
16
|
/**
|
|
17
17
|
* Returns the persona prefix for use in tool descriptions.
|
|
18
|
-
* Returns an empty string when running inside
|
|
18
|
+
* Returns an empty string when running inside Testbot — the testbot action
|
|
19
19
|
* appends the persona via `claude --append-system-prompt` instead, so we
|
|
20
20
|
* avoid duplicating it in every tool description.
|
|
21
21
|
*/
|
|
@@ -93,6 +93,9 @@ Steps:
|
|
|
93
93
|
- The \`--project-directory .\` means build context is the repository root (NOT .skyramp/sut/)
|
|
94
94
|
- Healthchecks for ALL services (testbot relies on \`--wait\`)
|
|
95
95
|
- Simple testbot credentials in environment (admin@testbot.com / testbot)
|
|
96
|
+
- Never paste cryptographic material into the \`environment:\` section of the compose file. This rule covers RSA and EC private keys, PEM blobs, JWK JSON, OAuth client secrets, and any other signing keypair. In practice, refuse anything that looks like a PEM block (text starting with \`-----BEGIN \` followed by \`PRIVATE KEY\`, \`RSA PRIVATE KEY\`, \`EC PRIVATE KEY\`, \`OPENSSH PRIVATE KEY\`, or \`ENCRYPTED PRIVATE KEY\` and the matching \`-----\`), a JWK in base64url-encoded JSON form (strings starting with \`eyJrdHkiOi\`, which decodes to \`{"kty":\`), or any base64 string longer than about 200 characters that looks like a key. The compose file is committed to the customer's repository, so anything written here lands in their PR diff and stays in git history — including the private half of any keypair. Two cases to handle:
|
|
97
|
+
1. **Symmetric secrets** — when the app accepts a plain string (\`JWT_SECRET\`, \`SESSION_SECRET\`, signing salts, HMAC keys), use a clearly-marked dummy literal so a code reviewer immediately recognizes it as fake. For example: \`JWT_SECRET: "testbot-dummy-do-not-use-in-production"\`.
|
|
98
|
+
2. **Asymmetric keys** — when the app requires a real keypair (RS256 JWT signing, gRPC TLS, signing keypairs), do not commit the keys. Instead, generate them inside the container at startup so they exist only for the container's lifetime. First look at the app's source or config to determine what format it expects: PEM, JWK, or raw base64. Then override the service's \`entrypoint\` with a short script that generates a keypair in that format, exports the key bodies into the env vars the app reads, and finally \`exec\`s the app's original command. Pick the tool that matches the format: \`openssl genpkey\` for PEM, \`python -m jwcrypto\` / \`node -e "require('jose')..."\` / \`step crypto jwk create\` for JWK, \`openssl rand -base64\` for a raw secret. Prefer a tool that already exists in the app's runtime image — for instance, a Django service usually has \`jwcrypto\` or \`josepy\` installed already — rather than installing one at startup. Testbot fetches its bearer token via \`get-auth-token.sh\`, which calls the app's login endpoint; the app signs the token with its in-container private key and returns it as an opaque string. The test client never reads the private key directly, so per-container ephemeral keys are enough — no GitHub repo secrets, no out-of-band key sharing.
|
|
96
99
|
- Generous \`start_period\` (30-120s) for CI cold builds
|
|
97
100
|
- For monorepos: may need a builder service that runs first
|
|
98
101
|
|
|
@@ -80,9 +80,7 @@ export function buildWorkspaceInitSection() {
|
|
|
80
80
|
1. Call \`skyramp_init_scan\` with \`workspacePath\` set to that repository root → follow the returned instructions to discover all services.
|
|
81
81
|
2. Call \`skyramp_init_workspace\` with the same \`workspacePath\`, \`services\`, and the \`scanToken\` from step 1.
|
|
82
82
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
Also verify each service's \`api.baseUrl\` includes the API path prefix (e.g. \`/api\`, \`/api/v1\`, \`/v1\`) — not just \`host:port\`. A bare host means every test request hits the root and 404s. Derive the prefix from the strongest available signal: OpenAPI \`servers[].url\` or \`basePath\`; the app's routing config (FastAPI \`root_path\`, Express \`app.use('/api/v1', ...)\`, Rails routes prefix, Django \`urlpatterns\` mount); or the longest common leading path across the existing test suite's endpoints. If none of those yield a prefix, leave it bare AND flag it in your summary so the user can review.`;
|
|
83
|
+
Later steps in this SUT bootstrap run write \`.skyramp/sut/*\` files alongside the workspace file. If you need to update per-service \`runtimeDetails.serverStartCommand\` to match the new SUT files, call \`skyramp_init_workspace\` again with \`force: true\` and the full updated services array — never use Edit/Write directly on \`.skyramp/workspace.yml\`. See the \`skyramp_init_workspace\` tool description for the edit-mode contract.`;
|
|
86
84
|
}
|
|
87
85
|
export function buildLocalValidationSection() {
|
|
88
86
|
return `Before reporting success, exercise the adapted workflow locally. Testbot's external fix loop only retries when SUT lifecycle commands are set on the Testbot action. When the SUT is brought up by surrounding GHA steps (skipTargetSetup: 'true'), the fix loop is skipped — so the local check below is the only safety net before the workflow is committed.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Prompt language for skyramp_actions output — the LLM_INSTRUCTIONS block
|
|
3
|
-
* emitted to the agent that applies UPDATE and
|
|
3
|
+
* emitted to the agent that applies UPDATE, REGENERATE, and DELETE actions.
|
|
4
4
|
*
|
|
5
5
|
* Kept separate from actionsTool.ts so prompt text is co-located with other
|
|
6
6
|
* maintenance prompts and can be reviewed/edited without touching tool logic.
|
|
@@ -18,6 +18,8 @@ export declare function buildFileRenameStrategy(): string;
|
|
|
18
18
|
export declare function buildUpdateStrategy(): string;
|
|
19
19
|
/** Strategy string for REGENERATE — call generation tool to overwrite the file. */
|
|
20
20
|
export declare function buildRegenerateStrategy(): string;
|
|
21
|
+
/** Strategy string for DELETE — remove obsolete test files whose covered endpoints/UI no longer exist. */
|
|
22
|
+
export declare function buildDeleteStrategy(): string;
|
|
21
23
|
/** Per-file instruction block for a single UPDATE recommendation. */
|
|
22
24
|
export declare function buildUpdateFileInstruction(params: {
|
|
23
25
|
testFile: string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Prompt language for skyramp_actions output — the LLM_INSTRUCTIONS block
|
|
3
|
-
* emitted to the agent that applies UPDATE and
|
|
3
|
+
* emitted to the agent that applies UPDATE, REGENERATE, and DELETE actions.
|
|
4
4
|
*
|
|
5
5
|
* Kept separate from actionsTool.ts so prompt text is co-located with other
|
|
6
6
|
* maintenance prompts and can be reviewed/edited without touching tool logic.
|
|
@@ -15,12 +15,20 @@ export function buildFileRenameStrategy() {
|
|
|
15
15
|
}
|
|
16
16
|
/** Strategy string for in-place UPDATE edits. */
|
|
17
17
|
export function buildUpdateStrategy() {
|
|
18
|
-
return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context.
|
|
18
|
+
return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context.
|
|
19
|
+
|
|
20
|
+
UPDATE is an in-place edit to the existing baseline file — do not create a new test file to cover the same page or component.
|
|
21
|
+
|
|
22
|
+
After applying all edits, call skyramp_enhance_assertions with each updated file path.`;
|
|
19
23
|
}
|
|
20
24
|
/** Strategy string for REGENERATE — call generation tool to overwrite the file. */
|
|
21
25
|
export function buildRegenerateStrategy() {
|
|
22
26
|
return `For each file in regenerate_context, call the appropriate generation tool (skyramp_integration_test_generation or skyramp_contract_test_generation) with outputDir set to the file's directory and output set to the filename. Use existing_content to determine the test type, endpoint, auth pattern, and language. The generation tool will overwrite the file. Do NOT use skyramp_ui_test_generation here — UI test regeneration requires a recorded trace (playwrightInput) and must be handled separately.`;
|
|
23
27
|
}
|
|
28
|
+
/** Strategy string for DELETE — remove obsolete test files whose covered endpoints/UI no longer exist. */
|
|
29
|
+
export function buildDeleteStrategy() {
|
|
30
|
+
return `For each file in files_to_delete, remove it from the repository with 'git rm <file>' (fall back to 'rm <file>' if the file is untracked). These tests cover endpoints or UI that no longer exist and have no migration path. Do NOT edit, regenerate, or call skyramp_enhance_assertions on these files — the only action is deletion.`;
|
|
31
|
+
}
|
|
24
32
|
/** Per-file instruction block for a single UPDATE recommendation. */
|
|
25
33
|
export function buildUpdateFileInstruction(params) {
|
|
26
34
|
const { testFile, renames, suggestedNewFile, updateInstructions, rationale } = params;
|
|
@@ -1,4 +1,17 @@
|
|
|
1
|
-
export
|
|
1
|
+
export type UiDriftParams = {
|
|
2
|
+
tests: Array<{
|
|
3
|
+
testFile: string;
|
|
4
|
+
}>;
|
|
5
|
+
changedFrontendFiles: string[];
|
|
6
|
+
blueprintCaptured?: boolean;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Build the drift analysis prompt.
|
|
10
|
+
* - apiTests: tests to assess for API endpoint drift (always included unless empty and ui is provided).
|
|
11
|
+
* - ui: when provided, appends a UI drift section for component/browser tests.
|
|
12
|
+
* Omit when no frontend files changed or no UI tests exist.
|
|
13
|
+
*/
|
|
14
|
+
export declare function buildDriftAnalysisPrompt(stateFile: string | undefined, apiTests: Array<{
|
|
2
15
|
testFile: string;
|
|
3
16
|
source?: string;
|
|
4
|
-
}
|
|
17
|
+
}>, ui?: UiDriftParams, repoPaths?: string[]): string;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { buildActionDecisionTree, buildCheckAdditiveFields, buildCheckEndpointExistence, buildCheckResponseShape, buildCheckAuthAndAuthorization, buildCheckBehavioralContract, buildCheckAssignAction, buildDriftOutputChecklist, } from "./driftAnalysisSections.js";
|
|
2
|
+
import { buildUiActionDecisionTree, buildUiCheckRouteExistence, buildUiCheckSelectors, buildUiCheckPageObjects, buildUiCheckBehavioralChanges, buildUiCheckAssignAction, buildUiDriftOutputChecklist, } from "./uiDriftAnalysisSections.js";
|
|
3
|
+
import { RECOMMENDATIONS_INSTRUCTION, buildSymbolDiscoveryStep } from "./driftAnalysisShared.js";
|
|
2
4
|
import { PromptPlan } from "../test-recommendation/promptPlan.js";
|
|
3
|
-
const
|
|
5
|
+
const _apiPlan = new PromptPlan()
|
|
4
6
|
.addPhase("maintenance", "Test Maintenance Assessment", {
|
|
5
7
|
headerLevel: "##",
|
|
6
8
|
stepFormat: "hash",
|
|
@@ -14,6 +16,64 @@ const _plan = new PromptPlan()
|
|
|
14
16
|
.subStep("ASSIGN_ACTION", "Assign action", () => buildCheckAssignAction())
|
|
15
17
|
.step("CALL_TOOL", "Submit recommendations", (p) => buildDriftOutputChecklist(p.stateFile, p.existingTests))
|
|
16
18
|
.done();
|
|
17
|
-
|
|
18
|
-
|
|
19
|
+
const _uiPlan = new PromptPlan()
|
|
20
|
+
.addPhase("ui_maintenance", "UI Test Drift Assessment", {
|
|
21
|
+
headerLevel: "##",
|
|
22
|
+
stepFormat: "hash",
|
|
23
|
+
})
|
|
24
|
+
.step("UI_ASSESS", "UI Action Decision Tree — assess each UI test against the diff and blueprints", () => buildUiActionDecisionTree())
|
|
25
|
+
.subStep("UI_ROUTE_EXISTENCE", "Route existence", () => buildUiCheckRouteExistence())
|
|
26
|
+
.subStep("UI_SELECTORS", "Selector validity", () => buildUiCheckSelectors())
|
|
27
|
+
.subStep("UI_PAGE_OBJECTS", "Page object/fixture existence", () => buildUiCheckPageObjects())
|
|
28
|
+
.subStep("UI_BEHAVIOR", "Component behavior changes", () => buildUiCheckBehavioralChanges())
|
|
29
|
+
.subStep("UI_ASSIGN_ACTION", "Assign action", () => buildUiCheckAssignAction())
|
|
30
|
+
.step("UI_CALL_TOOL", "Submit recommendations", (p) => buildUiDriftOutputChecklist(p.stateFile, p.tests))
|
|
31
|
+
.done();
|
|
32
|
+
/**
|
|
33
|
+
* Build the drift analysis prompt.
|
|
34
|
+
* - apiTests: tests to assess for API endpoint drift (always included unless empty and ui is provided).
|
|
35
|
+
* - ui: when provided, appends a UI drift section for component/browser tests.
|
|
36
|
+
* Omit when no frontend files changed or no UI tests exist.
|
|
37
|
+
*/
|
|
38
|
+
export function buildDriftAnalysisPrompt(stateFile, apiTests, ui, repoPaths) {
|
|
39
|
+
const parts = [];
|
|
40
|
+
// Emit symbol discovery once at the top regardless of how many sections follow.
|
|
41
|
+
// Placing it inside each plan's checklist caused duplication on mixed diffs.
|
|
42
|
+
const hasAnyTests = apiTests.length > 0 || (ui?.tests?.length ?? 0) > 0;
|
|
43
|
+
const discovery = buildSymbolDiscoveryStep(hasAnyTests, repoPaths);
|
|
44
|
+
if (discovery)
|
|
45
|
+
parts.push(discovery);
|
|
46
|
+
// Include API drift when there are API tests, or when UI drift is not running
|
|
47
|
+
// (ensures skyramp_actions is always reachable even if apiTests is empty).
|
|
48
|
+
if (apiTests.length > 0 || !ui) {
|
|
49
|
+
parts.push(`<drift_analysis_rules>\n${_apiPlan.render({ stateFile, existingTests: apiTests })}\n</drift_analysis_rules>`);
|
|
50
|
+
}
|
|
51
|
+
if (ui) {
|
|
52
|
+
const signalNote = ui.blueprintCaptured
|
|
53
|
+
? `blueprintCaptured=true — use \`browser_blueprint_diff\` for tests that navigate pages.`
|
|
54
|
+
: `blueprintCaptured=false — use source diff only. Read each test to determine what it exercises.`;
|
|
55
|
+
const contextHeader = `<ui_analysis_context>
|
|
56
|
+
**UI Test Drift Analysis Context**
|
|
57
|
+
|
|
58
|
+
**Signal:** ${signalNote}
|
|
59
|
+
|
|
60
|
+
**Changed Frontend Files:**
|
|
61
|
+
${ui.changedFrontendFiles.length > 0
|
|
62
|
+
? ui.changedFrontendFiles.map(f => `- ${f}`).join("\n")
|
|
63
|
+
: "(none)"}
|
|
64
|
+
|
|
65
|
+
**Tests to Assess:**
|
|
66
|
+
${ui.tests.length > 0
|
|
67
|
+
? ui.tests.map(t => `- ${t.testFile}`).join("\n")
|
|
68
|
+
: "(none)"}
|
|
69
|
+
|
|
70
|
+
**Instructions:**
|
|
71
|
+
For each test above, read it to determine its type, then apply the scope gate from the decision tree. Assign the appropriate action (DELETE, REGENERATE, UPDATE, VERIFY, IGNORE).
|
|
72
|
+
</ui_analysis_context>
|
|
73
|
+
|
|
74
|
+
`;
|
|
75
|
+
parts.push(contextHeader + `<ui_drift_analysis_rules>\n${_uiPlan.render({ stateFile, tests: ui.tests, blueprintCaptured: ui.blueprintCaptured })}\n</ui_drift_analysis_rules>`);
|
|
76
|
+
}
|
|
77
|
+
parts.push(RECOMMENDATIONS_INSTRUCTION);
|
|
78
|
+
return parts.join("\n\n");
|
|
19
79
|
}
|
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
export declare function buildActionDecisionTree(): string;
|
|
2
|
-
/** @deprecated use the individual check functions; this function is no longer part of the prompt */
|
|
3
|
-
export declare function buildBreakingChangePatterns(): string;
|
|
4
2
|
export declare function buildCheckEndpointExistence(): string;
|
|
5
3
|
export declare function buildCheckResponseShape(): string;
|
|
6
4
|
export declare function buildCheckAuthAndAuthorization(): string;
|