@skyramp/mcp 0.0.63-rc.4 → 0.0.63-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/commands/commandLibrary.js +44 -0
- package/build/commands/recommendTestsAndExecuteCommand.js +118 -0
- package/build/commands/testThisEndpointCommand.js +156 -0
- package/build/index.js +11 -3
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +10 -1
- package/build/prompts/test-recommendation/recommendationSections.js +143 -29
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +2 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +50 -18
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +288 -0
- package/build/prompts/testbot/testbot-prompts.js +54 -18
- package/build/services/ScenarioGenerationService.js +16 -6
- package/build/services/ScenarioGenerationService.test.js +225 -80
- package/build/services/TestGenerationService.js +56 -3
- package/build/services/TestGenerationService.test.js +258 -0
- package/build/tools/generate-tests/generateScenarioRestTool.js +16 -11
- package/build/tools/one-click/oneClickTool.js +190 -0
- package/build/tools/submitReportTool.js +15 -2
- package/build/tools/submitReportTool.test.js +28 -3
- package/build/tools/test-management/analyzeChangesTool.js +9 -4
- package/build/tools/test-recommendation/recommendTestsTool.js +5 -5
- package/build/types/OneClickCommands.js +6 -0
- package/build/types/RepositoryAnalysis.js +4 -0
- package/build/types/TestRecommendation.js +2 -0
- package/build/types/TestTypes.js +16 -2
- package/build/utils/scenarioDrafting.js +194 -0
- package/build/utils/scenarioDrafting.test.js +128 -1
- package/build/utils/workspaceAuth.js +37 -0
- package/package.json +1 -1
- package/build/prompts/driftAnalysisPrompt.js +0 -159
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { TEST_GIVEN_ENDPOINT_COMPREHENSIVELY_COMMAND } from "./testThisEndpointCommand.js";
|
|
2
|
+
import { FULLREPO_RECOMMEND_GENERATE_EXECUTE_TOPN_TESTS_COMMAND } from "./recommendTestsAndExecuteCommand.js";
|
|
3
|
+
/**
|
|
4
|
+
* To add a new predefined workflow:
|
|
5
|
+
* 1. Add a new id to OneClickCommandId in types/OneClickCommands.ts
|
|
6
|
+
* 2. Create a new file in commands/ (e.g. myWorkflowCommand.ts) with steps and export the command
|
|
7
|
+
* 3. Import the command here and add it to COMMAND_LIBRARY
|
|
8
|
+
* 4. The generic buildWorkflowFromCommand() will render detailed instructions from your steps
|
|
9
|
+
* (no schema changes needed in oneClickTool.ts — the workflow param description is generated dynamically)
|
|
10
|
+
*/
|
|
11
|
+
/** All predefined one-click commands */
|
|
12
|
+
const COMMAND_LIBRARY = {
|
|
13
|
+
test_given_endpoint_comprehensively: TEST_GIVEN_ENDPOINT_COMPREHENSIVELY_COMMAND,
|
|
14
|
+
full_repo_scan_recommend_generate_and_execute_top_n_tests: FULLREPO_RECOMMEND_GENERATE_EXECUTE_TOPN_TESTS_COMMAND,
|
|
15
|
+
};
|
|
16
|
+
export function getCommand(id) {
|
|
17
|
+
const cmd = COMMAND_LIBRARY[id];
|
|
18
|
+
if (!cmd) {
|
|
19
|
+
throw new Error(`Unknown one-click command: ${id}`);
|
|
20
|
+
}
|
|
21
|
+
return cmd;
|
|
22
|
+
}
|
|
23
|
+
export function getAllCommands() {
|
|
24
|
+
return Object.values(COMMAND_LIBRARY);
|
|
25
|
+
}
|
|
26
|
+
export function getCommandIds() {
|
|
27
|
+
return Object.keys(COMMAND_LIBRARY);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Look up a command by its string ID. Accepts any string so the tool handler
|
|
31
|
+
* does not need to cast; throws with a helpful message listing valid IDs if
|
|
32
|
+
* the ID is not found.
|
|
33
|
+
*/
|
|
34
|
+
export function lookupCommand(id) {
|
|
35
|
+
const cmd = COMMAND_LIBRARY[id];
|
|
36
|
+
if (!cmd) {
|
|
37
|
+
const available = getCommandIds().join(", ");
|
|
38
|
+
throw new Error(`Unknown workflow: "${id}". Valid workflow IDs: ${available}.`);
|
|
39
|
+
}
|
|
40
|
+
return cmd;
|
|
41
|
+
}
|
|
42
|
+
// Re-export commands for consumers that import from commandLibrary
|
|
43
|
+
export { TEST_GIVEN_ENDPOINT_COMPREHENSIVELY_COMMAND } from "./testThisEndpointCommand.js";
|
|
44
|
+
export { FULLREPO_RECOMMEND_GENERATE_EXECUTE_TOPN_TESTS_COMMAND } from "./recommendTestsAndExecuteCommand.js";
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Predefined One-Click Command: Recommend tests and generate and execute top recommended tests
|
|
3
|
+
*
|
|
4
|
+
* Spec order:
|
|
5
|
+
* skyramp_analyze_changes (combined analyze + discover + recommend)
|
|
6
|
+
* → Generate tests for top N recommended types
|
|
7
|
+
* → Execute each via skyramp_execute_test
|
|
8
|
+
* → State cleanup
|
|
9
|
+
*/
|
|
10
|
+
const fullRepoRecommendGenerateExecuteTopNSteps = [
|
|
11
|
+
{
|
|
12
|
+
stepIndex: 0,
|
|
13
|
+
title: "Ensure workspace is initialized",
|
|
14
|
+
description: "First check whether the repositoryPath is a git repository by verifying that a .git directory exists at the repo root. If there is no .git directory, do not attempt to initialize a Skyramp workspace and proceed without one. If .git exists, check whether .skyramp/workspace.yml exists. If workspace.yml exists, read it and capture language, framework, outputDir, and api.baseUrl for use in later steps. If workspace.yml does not exist and the user has not declined initialization, call skyramp_initialize_workspace with repositoryPath to create it, then read and capture language, framework, outputDir, and api.baseUrl.",
|
|
15
|
+
toolCall: {
|
|
16
|
+
toolName: "skyramp_initialize_workspace",
|
|
17
|
+
description: "Initialize Skyramp workspace if not already present",
|
|
18
|
+
inputs: {
|
|
19
|
+
workspacePath: { source: "user", paramKey: "repositoryPath" },
|
|
20
|
+
},
|
|
21
|
+
},
|
|
22
|
+
conditionalGuidance: "Follow server-level init rules: (1) Check for .git at repositoryPath — if missing, do not call skyramp_initialize_workspace and proceed without a workspace. (2) If .git exists, check for .skyramp/workspace.yml — if it exists, read it and do not call skyramp_initialize_workspace. (3) If workspace.yml does not exist, call skyramp_initialize_workspace only if the user has not explicitly declined. If the user declines, proceed without a workspace. Always capture language, framework, outputDir, and api.baseUrl for later steps.",
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
stepIndex: 1,
|
|
26
|
+
title: "Analyze changes and get recommendations",
|
|
27
|
+
description: "Call skyramp_analyze_changes with repositoryPath and analysisScope from user. This single tool scans endpoints, discovers existing tests, and returns ranked test recommendations inline. Capture the stateFile path and the ranked recommendations for step 2.",
|
|
28
|
+
toolCall: {
|
|
29
|
+
toolName: "skyramp_analyze_changes",
|
|
30
|
+
description: "Analyze repo and generate ranked recommendations inline; returns stateFile",
|
|
31
|
+
inputs: {
|
|
32
|
+
repositoryPath: { source: "user", paramKey: "repositoryPath" },
|
|
33
|
+
scope: { source: "literal", value: "full_repo" },
|
|
34
|
+
},
|
|
35
|
+
outputs: ["stateFile", "recommendations"],
|
|
36
|
+
},
|
|
37
|
+
conditionalGuidance: "Always call skyramp_analyze_changes — do not skip this step or use a pre-existing stateFile. Capture stateFile and ranked recommendations for step 2.",
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
stepIndex: 2,
|
|
41
|
+
title: "Generate tests for top N recommended types",
|
|
42
|
+
description: "From the ranked recommendations returned by skyramp_analyze_changes in step 1, take the top N recommended test types (high priority first, then medium; N = topN from user, default 3).\n\nIMPORTANT — UI or E2E in top N: If UI or E2E test types are among the top N, treat each recommendation independently. For each such recommendation, first search the repository for existing trace files suitable for the endpoint or path associated with that recommendation (look for .zip or .json Skyramp trace files in the repo that reference the recommended endpoint path or URL). If suitable traces are found, use them directly as inputs to skyramp_ui_test_generation (playwrightInput) or skyramp_e2e_test_generation (trace + playwrightInput) — skip trace collection. If no suitable traces exist in the repo for that recommendation, collect them: call skyramp_start_trace_collection (playwright: true, outputDir: absolute path from workspace.yml), have the user interact with the recommended endpoint/path in the browser, then call skyramp_stop_trace_collection (playwrightEnabled: true, same outputDir). Use the produced trace file and Playwright zip as inputs for generation for that specific recommendation.\n\nFor all other types in the top N, for each recommendation call the corresponding generation tool using an endpoint URL derived from that recommendation (api.baseUrl + endpoint path, or a full endpoint URL included in the recommendation). If a recommendation does not specify a concrete endpoint, follow the scope indicated in the recommendation (e.g. service-wide or repo-wide) and omit endpointURL or use only api.baseUrl as appropriate. In all cases also pass apiSchema, language, framework, and outputDir from workspace.yml. Capture each generated test file path for step 3.",
|
|
43
|
+
conditionalGuidance: "Tool mapping by type — smoke: skyramp_smoke_test_generation, contract: skyramp_contract_test_generation, integration: skyramp_integration_test_generation, fuzz: skyramp_fuzz_test_generation, load: skyramp_load_test_generation, e2e: skyramp_e2e_test_generation, ui: skyramp_ui_test_generation. Limit to top N types. For UI/E2E: for each recommendation, (1) search the repo for existing traces for the endpoint or path associated with that recommendation; (2) if found, use them directly; (3) if not found, run skyramp_start_trace_collection → user interaction with the recommended endpoint/path → skyramp_stop_trace_collection, then use the produced traces. Record each generated test file path for step 3.",
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
stepIndex: 3,
|
|
47
|
+
title: "Execute generated tests",
|
|
48
|
+
description: "Before calling skyramp_execute_test, resolve the auth token using only information available to the workflow: (1) if the user provided a token param, use it directly; (2) otherwise, check .skyramp/workspace.yml or other repo config for a documented token value; (3) if no token can be resolved, proceed with empty string '' and rely on skyramp_execute_test to surface 401/403 errors — then report the auth failure to the user and ask them to provide a Bearer token to re-run. Then for each test file generated in step 2, call skyramp_execute_test with the test file path, language, testType, and the resolved token.",
|
|
49
|
+
toolCall: {
|
|
50
|
+
toolName: "skyramp_execute_test",
|
|
51
|
+
description: "Execute each generated test; resolve token before calling. For each generated test file path returned directly by the generation tools in step 2, invoke this tool with testFile set to that path.",
|
|
52
|
+
inputs: {
|
|
53
|
+
workspacePath: { source: "user", paramKey: "repositoryPath" },
|
|
54
|
+
language: { source: "literal", value: "from workspace" },
|
|
55
|
+
testType: { source: "literal", value: "from test file / recommendation" },
|
|
56
|
+
testFile: {
|
|
57
|
+
source: "literal",
|
|
58
|
+
value: "path to a single generated test file; for each test path returned by generation tools in step 2, call skyramp_execute_test separately with testFile set to that path",
|
|
59
|
+
},
|
|
60
|
+
token: { source: "user", paramKey: "token" },
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
conditionalGuidance: "Skip if step 2 generated no tests. Iterate over each generated test file path returned directly from the tools invoked in step 2 and call skyramp_execute_test once per file. Token resolution: (1) user-provided token; (2) token from .skyramp/workspace.yml or repo config; (3) empty string '' — let skyramp_execute_test surface auth errors, then ask the user for a Bearer token to re-run.",
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
stepIndex: 4,
|
|
67
|
+
title: "Clean up state files",
|
|
68
|
+
description: "Call skyramp_state_cleanup with action 'cleanup' and maxAgeHours set to 1 to remove temporary state files created by the recommendation toolset. These live in system temp (e.g. /tmp) — not in the user repo.",
|
|
69
|
+
toolCall: {
|
|
70
|
+
toolName: "skyramp_state_cleanup",
|
|
71
|
+
description: "Remove temporary state files from system temp",
|
|
72
|
+
inputs: {
|
|
73
|
+
action: { source: "literal", value: "cleanup" },
|
|
74
|
+
maxAgeHours: { source: "literal", value: 1 },
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
];
|
|
79
|
+
export const FULLREPO_RECOMMEND_GENERATE_EXECUTE_TOPN_TESTS_COMMAND = {
|
|
80
|
+
id: "full_repo_scan_recommend_generate_and_execute_top_n_tests",
|
|
81
|
+
name: "Full Repo: Recommend, Generate and Run TopN Tests",
|
|
82
|
+
description: "Run skyramp_analyze_changes to scan the repo and get ranked recommendations, generate tests for the top N recommended types, execute the generated tests, then clean up state files.",
|
|
83
|
+
intent: {
|
|
84
|
+
contextIndicators: [
|
|
85
|
+
"Use when the user wants to scan the entire repository with no specific endpoint or PR diff in mind — to get ranked test recommendations across all endpoints, generate the top N recommended test types, and execute them",
|
|
86
|
+
"Scope is always the full repository — the user has not mentioned a specific endpoint URL, path, or name",
|
|
87
|
+
"Use when the user asks to recommend, generate, and execute tests for the whole repo or says something like 'run the full repo test workflow'",
|
|
88
|
+
"Do NOT use when the user specifies a particular endpoint — use test_given_endpoint_comprehensively instead",
|
|
89
|
+
"Do NOT use when the user asks about a PR diff or branch-scoped analysis — use skyramp_analyze_changes directly instead",
|
|
90
|
+
"Do NOT use for simple single-tool requests such as 'generate a smoke test' or 'recommend tests for this PR'",
|
|
91
|
+
],
|
|
92
|
+
purpose: "Full repo scan: get recommendations → Generate top N types → Execute generated tests → Clean up (no specific endpoint, no PR diff)",
|
|
93
|
+
workflowSummary: "Full Repo Scan → Recommend → Generate top N → Execute each test → Clean up",
|
|
94
|
+
examples: {
|
|
95
|
+
use: [
|
|
96
|
+
"scan the full repo and recommend and execute top 3 tests",
|
|
97
|
+
"run the full repo test workflow",
|
|
98
|
+
"recommend generate and execute tests for the whole repo",
|
|
99
|
+
"find the best tests to write for this codebase and run them",
|
|
100
|
+
],
|
|
101
|
+
doNotUse: [
|
|
102
|
+
"comprehensively test the products endpoint → has a specific endpoint, use test_given_endpoint_comprehensively instead",
|
|
103
|
+
"recommend tests for my PR diff → branch-scoped, use skyramp_analyze_changes directly",
|
|
104
|
+
"generate a smoke test → single tool request, call skyramp_smoke_test_generation directly",
|
|
105
|
+
],
|
|
106
|
+
},
|
|
107
|
+
},
|
|
108
|
+
steps: fullRepoRecommendGenerateExecuteTopNSteps,
|
|
109
|
+
inputParams: {
|
|
110
|
+
required: ["repositoryPath"],
|
|
111
|
+
optional: [
|
|
112
|
+
"topN",
|
|
113
|
+
"token",
|
|
114
|
+
"endpointURL",
|
|
115
|
+
"apiSchema",
|
|
116
|
+
],
|
|
117
|
+
},
|
|
118
|
+
};
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Predefined One-Click Command: Test this endpoint
|
|
3
|
+
*
|
|
4
|
+
* Spec order:
|
|
5
|
+
* skyramp_analyze_changes (combined analyze + discover + recommend)
|
|
6
|
+
* → Evaluate missing (recommended minus existing)
|
|
7
|
+
* → Generate missing tests (by type)
|
|
8
|
+
* → Execute generated tests
|
|
9
|
+
* → [if existing tests found] Analyze test health → Optional batch execute → Actions
|
|
10
|
+
* → State cleanup
|
|
11
|
+
*/
|
|
12
|
+
const comprehensivelyTestGivenEndpointSteps = [
|
|
13
|
+
{
|
|
14
|
+
stepIndex: 0,
|
|
15
|
+
title: "Ensure workspace is initialized",
|
|
16
|
+
description: "First check whether the repositoryPath is a git repository by verifying that a .git directory exists at the repo root. If there is no .git directory, do not attempt to initialize a Skyramp workspace and proceed without one. If .git exists, check whether .skyramp/workspace.yml exists. If workspace.yml exists, read it and capture language, framework, outputDir, and api.baseUrl for use in later steps. If workspace.yml does not exist and the user has not declined initialization, call skyramp_initialize_workspace with repositoryPath to create it, then read and capture language, framework, outputDir, and api.baseUrl.",
|
|
17
|
+
toolCall: {
|
|
18
|
+
toolName: "skyramp_initialize_workspace",
|
|
19
|
+
description: "Initialize Skyramp workspace if not already present",
|
|
20
|
+
inputs: {
|
|
21
|
+
workspacePath: { source: "user", paramKey: "repositoryPath" },
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
conditionalGuidance: "Follow server-level init rules: (1) Check for .git at repositoryPath — if missing, do not call skyramp_initialize_workspace and proceed without a workspace. (2) If .git exists, check for .skyramp/workspace.yml — if it exists, read it and do not call skyramp_initialize_workspace. (3) If workspace.yml does not exist, call skyramp_initialize_workspace only if the user has not explicitly declined. If the user declines, proceed without a workspace. Always capture language, framework, outputDir, and api.baseUrl for later steps.",
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
stepIndex: 1,
|
|
28
|
+
title: "Analyze changes and get recommendations",
|
|
29
|
+
description: "Call skyramp_analyze_changes with repositoryPath and scope from the one-click invocation. It scans endpoints, discovers existing tests, and returns ranked test recommendations along with a stateFile path. The response includes a JSON block with stateFile, sessionId, and a summary (including existingTestCount), plus a ranked recommendations section. Capture the stateFile path and the ranked recommendations for later steps. The full list of existing tests is not in the response — it is stored in the stateFile on disk.",
|
|
30
|
+
toolCall: {
|
|
31
|
+
toolName: "skyramp_analyze_changes",
|
|
32
|
+
description: "Analyze repo, discover tests, and generate ranked recommendations; returns stateFile",
|
|
33
|
+
inputs: {
|
|
34
|
+
repositoryPath: { source: "user", paramKey: "repositoryPath" },
|
|
35
|
+
scope: { source: "literal", value: "full_repo" },
|
|
36
|
+
},
|
|
37
|
+
outputs: ["stateFile", "recommendations"],
|
|
38
|
+
},
|
|
39
|
+
conditionalGuidance: "If endpointURL is provided by the user, note it for use in generation steps. Capture the stateFile path and ranked recommendations from the response. The existing test list is not returned directly — it is embedded in the stateFile and will be used by skyramp_analyze_test_health in step 5.",
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
stepIndex: 2,
|
|
43
|
+
title: "Evaluate missing tests",
|
|
44
|
+
description: "Using the ranked recommendations from step 1, determine which test types are missing coverage for the target endpoint. The step 1 response includes an existingTestCount in its summary — if it is 0 for the target endpoint, all recommended types are missing. If existingTestCount > 0, the full existing test list is in the stateFile (it will be processed by skyramp_analyze_test_health in step 5); for now, conservatively treat all recommended types as candidates to generate unless the recommendations explicitly note coverage. Build the list of types to generate in step 3. If all recommended types are already covered, skip steps 3 and 4.",
|
|
45
|
+
conditionalGuidance: "For each recommended type (high priority first, then medium), check whether the recommendations indicate existing coverage for the target endpoint. If not indicated, add the type to the 'generate' list. If existingTestCount from step 1 is 0, all types are missing. If all recommended types have coverage, steps 3 and 4 can be skipped.",
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
stepIndex: 3,
|
|
49
|
+
title: "Generate missing tests (by type)",
|
|
50
|
+
description: "IMPORTANT — UI or E2E in missing list: If UI or E2E test types are in the 'missing' list, first search the repository for existing trace files suitable for the target endpoint (look for .zip or .json Skyramp trace files in the repo that reference the endpoint path or URL). If suitable traces are found, use them directly as inputs to skyramp_ui_test_generation (playwrightInput) or skyramp_e2e_test_generation (trace + playwrightInput) — skip trace collection. If no suitable traces exist in the repo, collect them: call skyramp_start_trace_collection (playwright: true, outputDir: absolute path from workspace.yml), have the user interact with the target endpoint in the browser, then call skyramp_stop_trace_collection (playwrightEnabled: true, same outputDir). Use the produced trace file and Playwright zip as inputs for generation.\n\nFor all other types in the 'missing' list, call the corresponding generation tool using endpointURL from the one-click invocation (or api.baseUrl + endpoint path from recommendation), apiSchema, language, framework, and outputDir from workspace.yml. Capture each generated test file path.",
|
|
51
|
+
conditionalGuidance: "Tool mapping by type — smoke: skyramp_smoke_test_generation, contract: skyramp_contract_test_generation, integration: skyramp_integration_test_generation, fuzz: skyramp_fuzz_test_generation, load: skyramp_load_test_generation, e2e: skyramp_e2e_test_generation, ui: skyramp_ui_test_generation. Only generate types in the 'missing' list. For UI/E2E: (1) search repo for existing traces for the target endpoint; (2) if found, use them directly; (3) if not found, run skyramp_start_trace_collection → user interaction → skyramp_stop_trace_collection, then use produced traces. Record each generated test file path for step 4.",
|
|
52
|
+
},
|
|
53
|
+
{
|
|
54
|
+
stepIndex: 4,
|
|
55
|
+
title: "Execute generated tests",
|
|
56
|
+
description: "Before calling skyramp_execute_test, resolve the auth token using only information available to the workflow: (1) if the user provided a token param, use it directly; (2) otherwise, check .skyramp/workspace.yml or other repo config for a documented token value; (3) if no token can be resolved, proceed with empty string '' and rely on skyramp_execute_test to surface 401/403 errors — then report the auth failure to the user and ask them to provide a Bearer token to re-run.",
|
|
57
|
+
toolCall: {
|
|
58
|
+
toolName: "skyramp_execute_test",
|
|
59
|
+
description: "Execute each generated test; resolve token before calling. For each generated test file path returned directly by the generation tools in step 3, invoke this tool with testFile set to that path.",
|
|
60
|
+
inputs: {
|
|
61
|
+
workspacePath: { source: "user", paramKey: "repositoryPath" },
|
|
62
|
+
language: { source: "literal", value: "from workspace" },
|
|
63
|
+
testType: { source: "literal", value: "from generated test / recommendation" },
|
|
64
|
+
testFile: {
|
|
65
|
+
source: "literal",
|
|
66
|
+
value: "path to a single generated test file; for each test path returned by generation tools in step 3, call skyramp_execute_test separately with testFile set to that path",
|
|
67
|
+
},
|
|
68
|
+
token: { source: "user", paramKey: "token" },
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
conditionalGuidance: "Skip if step 3 generated no tests. Iterate over each generated test file path returned directly from the tools invoked in step 3 and call skyramp_execute_test once per file. Token resolution: (1) user-provided token; (2) token from .skyramp/workspace.yml or repo config; (3) empty string '' — let skyramp_execute_test surface auth errors, then ask the user for a Bearer token to re-run.",
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
stepIndex: 5,
|
|
75
|
+
title: "Analyze test health for existing tests (only if step 1 found existing tests for the target endpoint)",
|
|
76
|
+
description: "Run only if step 1 discovered at least one existing test specifically for the target endpoint. Call skyramp_analyze_test_health with the stateFile from step 1. This assesses each existing test for drift and health, then returns instructions for the LLM to assign health scores and actions (UPDATE/REGENERATE/VERIFY). Follow the returned prompt to complete the assessment, then call skyramp_actions in step 7.",
|
|
77
|
+
toolCall: {
|
|
78
|
+
toolName: "skyramp_analyze_test_health",
|
|
79
|
+
description: "Drift and health assessment for existing tests; returns LLM assessment prompt",
|
|
80
|
+
inputs: {
|
|
81
|
+
stateFile: { source: "step", stepIndex: 1, outputKey: "stateFile" },
|
|
82
|
+
},
|
|
83
|
+
outputs: ["stateFile"],
|
|
84
|
+
},
|
|
85
|
+
conditionalGuidance: "Only run when step 1 found existing tests specifically for the target endpoint. If no tests were found for the target endpoint, skip steps 5–7 and go to step 8 (cleanup).",
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
stepIndex: 6,
|
|
89
|
+
title: "Optional: execute existing tests in batch (only if step 5 ran)",
|
|
90
|
+
description: "Run only if step 5 ran. Optionally call skyramp_execute_tests with the stateFile from step 5 to run existing tests and capture pass/fail results. Merge results back into the state file for use by skyramp_actions. Use token from user or empty string. If you skip this step, pass the stateFile from step 5 directly to step 7.",
|
|
91
|
+
toolCall: {
|
|
92
|
+
toolName: "skyramp_execute_tests",
|
|
93
|
+
description: "Optionally run existing tests in batch; updates state with results",
|
|
94
|
+
inputs: {
|
|
95
|
+
stateFile: { source: "step", stepIndex: 5, outputKey: "stateFile" },
|
|
96
|
+
authToken: { source: "user", paramKey: "token" },
|
|
97
|
+
},
|
|
98
|
+
outputs: ["stateFile"],
|
|
99
|
+
},
|
|
100
|
+
conditionalGuidance: "Only run when step 5 was executed. This step is optional — skip if batch execution is not needed.",
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
stepIndex: 7,
|
|
104
|
+
title: "Run maintenance actions (only if step 5 ran)",
|
|
105
|
+
description: "Run only if step 5 ran. Call skyramp_actions with the stateFile from step 6 if step 6 ran, or step 5's stateFile if step 6 was skipped. This applies recommended fixes (UPDATE/REGENERATE/VERIFY) to existing tests and generates tests for new endpoints. Call it immediately after the assessment without waiting for user confirmation.",
|
|
106
|
+
conditionalGuidance: "Only run when step 5 was executed. Use step 6's stateFile if step 6 ran; use step 5's stateFile if step 6 was skipped. Call skyramp_actions with the resolved stateFile.",
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
stepIndex: 8,
|
|
110
|
+
title: "Clean up state files",
|
|
111
|
+
description: "Call skyramp_state_cleanup with action 'cleanup' and maxAgeHours set to 1 to remove temporary state files created by the analysis and maintenance toolsets. These live in system temp (e.g. /tmp) — not in the user repo.",
|
|
112
|
+
toolCall: {
|
|
113
|
+
toolName: "skyramp_state_cleanup",
|
|
114
|
+
description: "Remove temporary state files from system temp",
|
|
115
|
+
inputs: {
|
|
116
|
+
action: { source: "literal", value: "cleanup" },
|
|
117
|
+
maxAgeHours: { source: "literal", value: 1 },
|
|
118
|
+
},
|
|
119
|
+
},
|
|
120
|
+
},
|
|
121
|
+
];
|
|
122
|
+
export const TEST_GIVEN_ENDPOINT_COMPREHENSIVELY_COMMAND = {
|
|
123
|
+
id: "test_given_endpoint_comprehensively",
|
|
124
|
+
name: "Test this endpoint comprehensively",
|
|
125
|
+
description: "Comprehensively test a REST API endpoint: analyze changes + discover existing tests + get recommendations in one step, then evaluate missing tests, generate only missing tests, execute them — then (if existing tests were found) run the test health analysis toolset (health assessment → optional batch execute → actions). State files stay in system temp.",
|
|
126
|
+
intent: {
|
|
127
|
+
contextIndicators: [
|
|
128
|
+
"Use when the user names or references a specific REST API endpoint (by path such as /api/products, full URL, or name such as 'products endpoint') and wants a complete multi-step test pipeline for it",
|
|
129
|
+
"Covers the full endpoint testing lifecycle: discover existing tests → evaluate missing coverage → generate missing tests → execute → health analysis → maintenance actions",
|
|
130
|
+
"Only use when the user explicitly signals a full or deep pipeline — words like 'comprehensively', 'thoroughly', 'deep test', 'full test pipeline', or 'all test types'. Do NOT trigger on bare phrases like 'test this endpoint' or 'test the products endpoint' alone — those are ambiguous and may only mean a single smoke test",
|
|
131
|
+
"Do NOT use for broad repo-level requests where no specific endpoint is named — use skyramp_analyze_changes directly instead",
|
|
132
|
+
"Do NOT use for simple single-tool requests such as 'generate a smoke test for this endpoint' — those go directly to the generation tool",
|
|
133
|
+
],
|
|
134
|
+
purpose: "Deep test a given endpoint: discover existing → evaluate missing → generate missing → execute → (if existing found) health analysis → maintenance actions → clean up",
|
|
135
|
+
workflowSummary: "Analyze Changes → Evaluate missing → Generate missing → Execute generated → [if existing] Test Health → Batch execute → Actions → Clean up",
|
|
136
|
+
examples: {
|
|
137
|
+
use: [
|
|
138
|
+
"comprehensively test the products endpoint",
|
|
139
|
+
"thoroughly test /api/v1/orders",
|
|
140
|
+
"deep test the authentication endpoint",
|
|
141
|
+
"run the full test pipeline for /api/users",
|
|
142
|
+
],
|
|
143
|
+
doNotUse: [
|
|
144
|
+
"test this endpoint → ambiguous, may mean a single smoke test only",
|
|
145
|
+
"generate a smoke test for /api/products → single tool request, call skyramp_smoke_test_generation directly",
|
|
146
|
+
"recommend tests for my PR diff → use skyramp_analyze_changes directly",
|
|
147
|
+
"scan the full repo and run top tests → use full_repo_scan_recommend_generate_and_execute_top_n_tests instead",
|
|
148
|
+
],
|
|
149
|
+
},
|
|
150
|
+
},
|
|
151
|
+
steps: comprehensivelyTestGivenEndpointSteps,
|
|
152
|
+
inputParams: {
|
|
153
|
+
required: ["repositoryPath"],
|
|
154
|
+
optional: ["endpointURL", "apiSchema", "topN", "token"],
|
|
155
|
+
},
|
|
156
|
+
};
|
package/build/index.js
CHANGED
|
@@ -7,6 +7,7 @@ import { registerTraceTool } from "./tools/trace/startTraceCollectionTool.js";
|
|
|
7
7
|
import { registerTraceStopTool } from "./tools/trace/stopTraceCollectionTool.js";
|
|
8
8
|
import { registerExecuteSkyrampTestTool } from "./tools/executeSkyrampTestTool.js";
|
|
9
9
|
import { registerTestGenerationPrompt } from "./prompts/testGenerationPrompt.js";
|
|
10
|
+
import { AUTH_PLACEHOLDER_TOKEN } from "./types/TestTypes.js";
|
|
10
11
|
import { logger } from "./utils/logger.js";
|
|
11
12
|
import { registerUITestTool } from "./tools/generate-tests/generateUIRestTool.js";
|
|
12
13
|
import { registerSmokeTestTool } from "./tools/generate-tests/generateSmokeRestTool.js";
|
|
@@ -26,6 +27,7 @@ import { registerAnalyzeChangesTool, registerAnalyzeTestHealthTool, registerExec
|
|
|
26
27
|
import { registerTestbotPrompt, registerTestbotResource, } from "./prompts/testbot/testbot-prompts.js";
|
|
27
28
|
import { registerSubmitReportTool } from "./tools/submitReportTool.js";
|
|
28
29
|
import { registerInitializeWorkspaceTool } from "./tools/workspace/initializeWorkspaceTool.js";
|
|
30
|
+
import { registerOneClickTool } from "./tools/one-click/oneClickTool.js";
|
|
29
31
|
import { registerAnalysisResources } from "./resources/analysisResources.js";
|
|
30
32
|
import { AnalyticsService } from "./services/AnalyticsService.js";
|
|
31
33
|
import { initCheck } from "./utils/initAgent.js";
|
|
@@ -49,6 +51,8 @@ const server = new McpServer({
|
|
|
49
51
|
## Rules
|
|
50
52
|
- NEVER show CLI commands. ALWAYS use the MCP tools provided.
|
|
51
53
|
- For UI and E2E tests, use the trace collection start/stop tools.
|
|
54
|
+
- When the user asks to comprehensively, thoroughly, or deeply test a specific endpoint: MUST call \`skyramp_one_click_tool\` with workflow \`test_given_endpoint_comprehensively\` first. Do NOT self-orchestrate the steps manually.
|
|
55
|
+
- When the user asks to scan the full repo, recommend, generate, and execute top N tests: MUST call \`skyramp_one_click_tool\` with workflow \`full_repo_scan_recommend_generate_and_execute_top_n_tests\`.
|
|
52
56
|
|
|
53
57
|
## Test Management Flow
|
|
54
58
|
Use \`skyramp_analyze_changes\` as the single entry point for both test recommendations and test health analysis.
|
|
@@ -92,14 +96,16 @@ Follow this flow EVERY time before calling any Skyramp tool:
|
|
|
92
96
|
Before calling ANY test generation tool, you MUST follow this flow:
|
|
93
97
|
|
|
94
98
|
1. **Read** the .skyramp/workspace.yml file to get the configured defaults.
|
|
95
|
-
2. **Extract** the \`language\`, \`framework\`, \`outputDir\`, and \`api.
|
|
99
|
+
2. **Extract** the \`language\`, \`framework\`, \`outputDir\`, \`api.baseUrl\`, \`api.authHeader\`, and \`api.authType\` from the services section.
|
|
96
100
|
3. **Use those values** as defaults for the test generation tool call. Do NOT ask the user for these values if they are already configured in the workspace file.
|
|
97
101
|
4. **CRITICAL — endpointURL**: The \`endpointURL\` parameter MUST be the full URL to the specific endpoint being tested, NOT just the base URL. Construct it by combining \`api.baseUrl\` with the endpoint path. Example: if \`api.baseUrl\` is \`http://localhost:8000\` and the endpoint is \`/api/v1/products\`, pass \`endpointURL: "http://localhost:8000/api/v1/products"\`. NEVER pass just the base URL (e.g. \`http://localhost:8000\`) as \`endpointURL\`.
|
|
98
102
|
5. **CRITICAL — scenario generation**: When calling \`skyramp_scenario_test_generation\`, ALWAYS pass:
|
|
99
103
|
- \`baseURL\`: The full base URL from \`api.baseUrl\` (e.g., \`http://localhost:3000\`). This determines the scheme, host, and port in the generated trace. Without it, the trace defaults to https:443 which is almost always wrong for local development.
|
|
100
|
-
- \`authHeader\`:
|
|
104
|
+
- \`authHeader\`: Which HTTP header carries the auth credential. Get it from \`api.authHeader\` in workspace config. Examples: \`Authorization\` (Bearer/Token auth), \`X-Api-Key\` (API key auth), \`Cookie\` (session/cookie auth like NextAuth). Pass \`""\` to skip auth entirely (unauthenticated endpoints or \`api.authType: "none"\`).
|
|
105
|
+
- \`authScheme\`: Only when \`authHeader\` is \`Authorization\`. The prefix before the token (e.g., \`"Bearer"\` → \`Authorization: Bearer <token>\`). Common values: \`"Bearer"\`, \`"Basic"\`, \`"Token"\`. Leave empty if the API uses raw tokens with no prefix. Ignored for non-Authorization headers. **Derive from**: (1) OpenAPI spec \`securitySchemes\`/\`securityDefinitions\`, (2) source code auth middleware (\`passport.use\`, \`jwt.verify\`, header parsing logic), (3) workspace \`api.authType\`. **Do NOT guess.**
|
|
106
|
+
- \`authToken\`: The full header value, used verbatim. When omitted, \`SKYRAMP_PLACEHOLDER_TOKEN\` is auto-generated (combined with \`authScheme\` for Authorization headers). Only provide when the header needs a specific format — e.g., \`"session=${AUTH_PLACEHOLDER_TOKEN}"\` for Cookie. For Authorization headers, use \`authScheme\` instead. **Do NOT fabricate token values.**
|
|
101
107
|
- \`apiSchema\` is OPTIONAL — omit it for code-first apps without OpenAPI specs.
|
|
102
|
-
6. **CRITICAL — integration test from scenario**: When calling \`skyramp_integration_test_generation\` with a \`scenarioFile\`, ALSO pass \`authHeader\`
|
|
108
|
+
6. **CRITICAL — integration test from scenario**: When calling \`skyramp_integration_test_generation\` with a \`scenarioFile\`, ALSO pass \`authHeader\` and the same auth parameters used in scenario generation (\`authScheme\` for Authorization headers, or \`authToken\` for non-Authorization headers).
|
|
103
109
|
7. **If the workspace file does not exist**, or the needed values (language, framework, outputDir) are missing from the workspace config, ASK the user which language and framework they want before calling the tool.
|
|
104
110
|
8. The user can always override workspace defaults by explicitly specifying values in their request.
|
|
105
111
|
`,
|
|
@@ -190,6 +196,8 @@ registerActionsTool(server);
|
|
|
190
196
|
registerStateCleanupTool(server);
|
|
191
197
|
// Register workspace management tools
|
|
192
198
|
registerInitializeWorkspaceTool(server);
|
|
199
|
+
// Register one-click orchestrated workflows
|
|
200
|
+
registerOneClickTool(server);
|
|
193
201
|
// Register other Skyramp tools
|
|
194
202
|
const infrastructureTools = [
|
|
195
203
|
registerLoginTool,
|
|
@@ -34,6 +34,13 @@ Mounting context.`
|
|
|
34
34
|
UI-only PR — read changed components to find API calls (fetch, axios, hooks).`
|
|
35
35
|
: `### Step 2: Identify affected endpoints
|
|
36
36
|
No API route changes detected — read changed files to identify affected endpoints.`;
|
|
37
|
+
const criticalPatternStep = `### Step 2.5: Identify critical patterns for test categorization
|
|
38
|
+
Look for these patterns in model/schema/handler files to inform test recommendations:
|
|
39
|
+
- **Unique constraints**: \`@unique\`, \`unique: true\`, unique indexes, \`.refine()\` uniqueness checks, \`UNIQUE\` in SQL migrations
|
|
40
|
+
- **Cascade deletes**: \`ON DELETE CASCADE\`, \`.onDelete("cascade")\`, manual cascade logic in delete handlers
|
|
41
|
+
- **Permission checks**: auth middleware, ownership guards (\`req.user.id === resource.ownerId\`), role-based access control, \`isOwner\` assertions
|
|
42
|
+
- **Breaking changes in diff**: route renames, auth header changes, removed required fields, changed status codes
|
|
43
|
+
Tag each finding with its category (security_boundary, business_rule, data_integrity, breaking_change) for the recommendation step.`;
|
|
37
44
|
const step3Content = useHealthFlow
|
|
38
45
|
? `### Step 3: Identify tests at risk of drift
|
|
39
46
|
Assess which existing tests may be broken by the changes in this diff.
|
|
@@ -58,6 +65,8 @@ ${changedFiles}
|
|
|
58
65
|
|
|
59
66
|
${step2}
|
|
60
67
|
|
|
68
|
+
${criticalPatternStep}
|
|
69
|
+
|
|
61
70
|
${step3Content}`;
|
|
62
71
|
}
|
|
63
72
|
export function buildAnalysisOutputText(p) {
|
|
@@ -79,7 +88,7 @@ ${p.scannedEndpoints.map((ep) => ` ${ep.methods.join("|")} ${ep.path} (${ep.sou
|
|
|
79
88
|
`
|
|
80
89
|
: "";
|
|
81
90
|
const wsLine = p.wsBaseUrl
|
|
82
|
-
? `**Base URL**: \`${p.wsBaseUrl}
|
|
91
|
+
? `**Base URL**: \`${p.wsBaseUrl}\`${p.wsAuthHeader ? ` | **Auth header**: \`${p.wsAuthHeader}\`` : ""}${p.wsAuthType ? ` | **Auth type**: \`${p.wsAuthType}\`` : ""}`
|
|
83
92
|
: "";
|
|
84
93
|
const specSection = p.wsSchemaPath
|
|
85
94
|
? `
|