@skyramp/mcp 0.0.38 → 0.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +14 -0
- package/build/prompts/test-recommendation/repository-analysis-prompt.js +287 -0
- package/build/prompts/test-recommendation/test-mapping-prompt.js +266 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +125 -0
- package/build/prompts/testGenerationPrompt.js +7 -0
- package/build/services/TestGenerationService.js +0 -6
- package/build/tools/executeSkyrampTestTool.js +6 -0
- package/build/tools/generate-tests/generateIntegrationRestTool.js +4 -3
- package/build/tools/test-recommendation/analyzeRepositoryTool.js +96 -0
- package/build/tools/test-recommendation/mapTestsTool.js +195 -0
- package/build/tools/test-recommendation/recommendTestsTool.js +131 -0
- package/build/types/RepositoryAnalysis.js +106 -0
- package/build/types/TestMapping.js +173 -0
- package/build/types/TestRecommendation.js +52 -0
- package/build/types/TestTypes.js +1 -1
- package/build/utils/scoring-engine.js +262 -0
- package/package.json +1 -1
package/build/index.js
CHANGED
|
@@ -17,6 +17,9 @@ import { registerE2ETestTool } from "./tools/generate-tests/generateE2ERestTool.
|
|
|
17
17
|
import { registerLoginTool } from "./tools/auth/loginTool.js";
|
|
18
18
|
import { registerLogoutTool } from "./tools/auth/logoutTool.js";
|
|
19
19
|
import { registerFixErrorTool } from "./tools/fixErrorTool.js";
|
|
20
|
+
import { registerAnalyzeRepositoryTool } from "./tools/test-recommendation/analyzeRepositoryTool.js";
|
|
21
|
+
import { registerMapTestsTool } from "./tools/test-recommendation/mapTestsTool.js";
|
|
22
|
+
import { registerRecommendTestsTool } from "./tools/test-recommendation/recommendTestsTool.js";
|
|
20
23
|
import { registerModularizationTool } from "./tools/code-refactor/modularizationTool.js";
|
|
21
24
|
import { registerCodeReuseTool } from "./tools/code-refactor/codeReuseTool.js";
|
|
22
25
|
import { registerScenarioTestTool } from "./tools/generate-tests/generateScenarioRestTool.js";
|
|
@@ -59,6 +62,17 @@ const codeQualityTools = [
|
|
|
59
62
|
registerCodeReuseTool,
|
|
60
63
|
];
|
|
61
64
|
codeQualityTools.forEach((registerTool) => registerTool(server));
|
|
65
|
+
// Register internal/test recommendation tools (controlled by environment variable)
|
|
66
|
+
const enableInternalTools = process.env.SKYRAMP_ENABLE_INTERNAL_TOOLS === "true";
|
|
67
|
+
if (enableInternalTools) {
|
|
68
|
+
logger.info("Internal tools enabled - registering test recommendation tools");
|
|
69
|
+
registerAnalyzeRepositoryTool(server);
|
|
70
|
+
registerMapTestsTool(server);
|
|
71
|
+
registerRecommendTestsTool(server);
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
logger.info("Internal tools disabled. Set SKYRAMP_ENABLE_INTERNAL_TOOLS=true to enable.");
|
|
75
|
+
}
|
|
62
76
|
// Register other Skyramp tools
|
|
63
77
|
const infrastructureTools = [
|
|
64
78
|
registerLoginTool,
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Repository Analysis Prompt
|
|
3
|
+
* Comprehensive prompt for analyzing code repositories
|
|
4
|
+
*/
|
|
5
|
+
export function getRepositoryAnalysisPrompt(repositoryPath) {
|
|
6
|
+
return `
|
|
7
|
+
Analyze this repository systematically using the guide below.
|
|
8
|
+
|
|
9
|
+
REPOSITORY PATH: ${repositoryPath}
|
|
10
|
+
|
|
11
|
+
IMPORTANT OUTPUT FORMAT:
|
|
12
|
+
- Return analysis as valid JSON matching the RepositoryAnalysis interface
|
|
13
|
+
- Use empty arrays [] for unavailable data rather than omitting fields
|
|
14
|
+
- Use "unknown" for unknown string values
|
|
15
|
+
- Provide evidence (file paths, line numbers) where possible
|
|
16
|
+
|
|
17
|
+
## CRITICAL RULES
|
|
18
|
+
- If user flows are defined in documentation explicitly, follow them strictly and do not generate new ones.
|
|
19
|
+
|
|
20
|
+
# Repository Analysis Guide
|
|
21
|
+
|
|
22
|
+
## 1. Initial Repository Analysis
|
|
23
|
+
|
|
24
|
+
### Documentation
|
|
25
|
+
Review documentation files for any project-specific details and instructions
|
|
26
|
+
|
|
27
|
+
### Project Type
|
|
28
|
+
Identify what kind of application this is:
|
|
29
|
+
- REST API service
|
|
30
|
+
- Frontend application (React/Vue/Angular)
|
|
31
|
+
- Full-stack application
|
|
32
|
+
- Library/SDK
|
|
33
|
+
- CLI tool
|
|
34
|
+
- Microservices
|
|
35
|
+
- Other (describe)
|
|
36
|
+
|
|
37
|
+
### Primary Technology Stack
|
|
38
|
+
- **Language**: Identify the primary programming language(s)
|
|
39
|
+
- **Framework**: Identify the main framework(s) used
|
|
40
|
+
- **Key Dependencies**: List critical dependencies from package files
|
|
41
|
+
|
|
42
|
+
### Main Purpose
|
|
43
|
+
Describe what this application does in 1-2 sentences.
|
|
44
|
+
|
|
45
|
+
### Business Logic
|
|
46
|
+
- **Common User Flows**: How do users typically interact with this system?
|
|
47
|
+
- Infer from documentation, endpoint analysis, and code structure
|
|
48
|
+
- **Common Data Flows**: How does data move through the system?
|
|
49
|
+
- **Common Integration Patterns**: How does this system integrate with other services?
|
|
50
|
+
|
|
51
|
+
## 2. Artifact Discovery
|
|
52
|
+
|
|
53
|
+
### OpenAPI/Swagger Specifications
|
|
54
|
+
**Search Patterns:**
|
|
55
|
+
- \`**/*{openapi,swagger}*.{yaml,yml,json}\`
|
|
56
|
+
- \`**/api-spec.{yaml,yml,json}\`
|
|
57
|
+
- \`**/docs/**/*.{yaml,yml,json}\`
|
|
58
|
+
|
|
59
|
+
**Extract:**
|
|
60
|
+
- OpenAPI/Swagger version
|
|
61
|
+
- Total endpoint count
|
|
62
|
+
- Base URL(s)
|
|
63
|
+
- Authentication schemes
|
|
64
|
+
- Paths and operations
|
|
65
|
+
|
|
66
|
+
### Playwright Recordings
|
|
67
|
+
**Search Patterns:**
|
|
68
|
+
- \`**/*.zip\` in \`/recordings/\`, \`/tests/\`, \`/e2e/\` directories
|
|
69
|
+
|
|
70
|
+
### API Trace Files
|
|
71
|
+
**Search Patterns:**
|
|
72
|
+
- \`**/*trace*.json\` - JSON trace files
|
|
73
|
+
- \`**/*.har\` - HAR (HTTP Archive) files
|
|
74
|
+
- \`**/traces/**/*.json\` - Trace directories
|
|
75
|
+
|
|
76
|
+
## 3. API Endpoint Discovery
|
|
77
|
+
|
|
78
|
+
### If OpenAPI Spec Available
|
|
79
|
+
- Parse all paths and operations
|
|
80
|
+
- Extract: path, HTTP methods, authentication requirements, path parameters
|
|
81
|
+
- Count total endpoints
|
|
82
|
+
- Identify base URL from servers section
|
|
83
|
+
- Detect authentication type (bearer, api_key, oauth2, basic)
|
|
84
|
+
|
|
85
|
+
### If No OpenAPI Spec
|
|
86
|
+
Scan code for route definitions based on framework:
|
|
87
|
+
|
|
88
|
+
**Express (Node.js):**
|
|
89
|
+
- Patterns: \`app.get(\`, \`app.post(\`, \`router.get(\`, \`router.post(\`
|
|
90
|
+
|
|
91
|
+
**FastAPI (Python):**
|
|
92
|
+
- Patterns: \`@app.get(\`, \`@router.post(\`, \`@app.put(\`
|
|
93
|
+
|
|
94
|
+
**Spring (Java):**
|
|
95
|
+
- Patterns: \`@GetMapping\`, \`@PostMapping\`, \`@RestController\`, \`@RequestMapping\`
|
|
96
|
+
|
|
97
|
+
**Django (Python):**
|
|
98
|
+
- Patterns: \`path(\`, \`url(\` in \`urls.py\` files
|
|
99
|
+
|
|
100
|
+
**Flask (Python):**
|
|
101
|
+
- Patterns: \`@app.route(\`, \`@blueprint.route(\`
|
|
102
|
+
|
|
103
|
+
## 4. Authentication Analysis
|
|
104
|
+
|
|
105
|
+
Investigate how the application handles authentication and authorization.
|
|
106
|
+
|
|
107
|
+
### Look For:
|
|
108
|
+
- Environment variables: \`API_KEY\`, \`TOKEN\`, \`SECRET\`, \`AUTH_\`, \`JWT_\`
|
|
109
|
+
- Authentication middleware or decorators
|
|
110
|
+
- JWT handling libraries
|
|
111
|
+
- OAuth configuration files
|
|
112
|
+
- API key validation logic
|
|
113
|
+
- Session management
|
|
114
|
+
|
|
115
|
+
### Identify:
|
|
116
|
+
1. **Authentication Method**: Bearer token, API key, OAuth 2.0, Basic Auth, JWT, etc.
|
|
117
|
+
2. **Configuration Location**: Where auth is configured (env files, config files, middleware)
|
|
118
|
+
3. **Credential Setup**: How to set authentication credentials
|
|
119
|
+
|
|
120
|
+
## 5. Infrastructure Analysis
|
|
121
|
+
|
|
122
|
+
Analyze deployment and infrastructure configuration.
|
|
123
|
+
|
|
124
|
+
### Containerization
|
|
125
|
+
**Check for:**
|
|
126
|
+
- \`Dockerfile\`
|
|
127
|
+
- \`docker-compose.yml\`
|
|
128
|
+
- \`.dockerignore\`
|
|
129
|
+
|
|
130
|
+
### Orchestration
|
|
131
|
+
**Kubernetes:**
|
|
132
|
+
- Check for manifests in \`k8s/\`, \`kubernetes/\`, or \`.yaml\` files with \`kind: Deployment\`
|
|
133
|
+
|
|
134
|
+
**Docker Compose:**
|
|
135
|
+
- Check for multi-service setup
|
|
136
|
+
|
|
137
|
+
### CI/CD
|
|
138
|
+
**Identify platforms from config files:**
|
|
139
|
+
- GitHub Actions: \`.github/workflows/\`
|
|
140
|
+
- GitLab CI: \`.gitlab-ci.yml\`
|
|
141
|
+
- Jenkins: \`Jenkinsfile\`
|
|
142
|
+
- CircleCI: \`.circleci/config.yml\`
|
|
143
|
+
- Travis CI: \`.travis.yml\`
|
|
144
|
+
|
|
145
|
+
### Deployment Pattern Inference
|
|
146
|
+
- **Microservices**: Kubernetes AND multiple services detected
|
|
147
|
+
- **Full-stack**: Docker Compose AND both API and Frontend components detected
|
|
148
|
+
- **Containerized Monolith**: Single Dockerfile only (no compose, no K8s)
|
|
149
|
+
- **Traditional**: No container configurations found
|
|
150
|
+
|
|
151
|
+
## 6. Existing Test Analysis
|
|
152
|
+
|
|
153
|
+
Analyze the testing infrastructure and coverage.
|
|
154
|
+
|
|
155
|
+
### Test Directory Detection
|
|
156
|
+
**Search Patterns:**
|
|
157
|
+
- \`**/test/**\`
|
|
158
|
+
- \`**/tests/**\`
|
|
159
|
+
- \`**/__tests__/**\`
|
|
160
|
+
- \`**/spec/**\`
|
|
161
|
+
|
|
162
|
+
**Count test files:**
|
|
163
|
+
- \`**/*{.test,.spec}.{ts,js,tsx,jsx,py,java,go}\`
|
|
164
|
+
|
|
165
|
+
### Test Framework Detection
|
|
166
|
+
**Configuration Files:**
|
|
167
|
+
- \`pytest.ini\`, \`setup.cfg\` (pytest)
|
|
168
|
+
- \`jest.config.js\`, \`jest.config.ts\` (Jest)
|
|
169
|
+
- \`karma.conf.js\` (Karma)
|
|
170
|
+
- \`playwright.config.ts\` (Playwright)
|
|
171
|
+
- \`cypress.config.js\` (Cypress)
|
|
172
|
+
|
|
173
|
+
### Test Type Inference
|
|
174
|
+
**File Name Patterns:**
|
|
175
|
+
- **Unit Tests**: \`*unit*.{test,spec}\`, \`*.unit.*\`
|
|
176
|
+
- **Integration Tests**: \`*integration*.{test,spec}\`, \`*.integration.*\`
|
|
177
|
+
- **E2E Tests**: \`*e2e*.{test,spec}\`, \`*end-to-end*\`, \`*.e2e.*\`
|
|
178
|
+
- **Smoke Tests**: \`*smoke*.{test,spec}\`
|
|
179
|
+
- **Load Tests**: \`*load*.{test,spec}\`, \`*performance*\`
|
|
180
|
+
|
|
181
|
+
## OUTPUT STRUCTURE
|
|
182
|
+
|
|
183
|
+
Return a JSON object with this exact structure:
|
|
184
|
+
|
|
185
|
+
\`\`\`json
|
|
186
|
+
{
|
|
187
|
+
"metadata": {
|
|
188
|
+
"repositoryName": "name-from-path",
|
|
189
|
+
"analysisDate": "2025-10-15T14:30:00Z",
|
|
190
|
+
"scanDepth": "full"
|
|
191
|
+
},
|
|
192
|
+
"projectClassification": {
|
|
193
|
+
"projectType": "rest-api | frontend | full-stack | microservices | library | cli | other",
|
|
194
|
+
"primaryLanguage": "language",
|
|
195
|
+
"primaryFramework": "framework",
|
|
196
|
+
"deploymentPattern": "microservices | full-stack | containerized-monolith | traditional | unknown"
|
|
197
|
+
},
|
|
198
|
+
"technologyStack": {
|
|
199
|
+
"languages": ["language1", "language2"],
|
|
200
|
+
"frameworks": ["framework1", "framework2"],
|
|
201
|
+
"runtime": "runtime version",
|
|
202
|
+
"keyDependencies": [
|
|
203
|
+
{ "name": "dep1", "version": "1.0.0", "purpose": "description" }
|
|
204
|
+
]
|
|
205
|
+
},
|
|
206
|
+
"businessContext": {
|
|
207
|
+
"mainPurpose": "1-2 sentence description",
|
|
208
|
+
"userFlows": ["flow1", "flow2"],
|
|
209
|
+
"dataFlows": ["flow1", "flow2"],
|
|
210
|
+
"integrationPatterns": ["pattern1", "pattern2"]
|
|
211
|
+
},
|
|
212
|
+
"artifacts": {
|
|
213
|
+
"openApiSpecs": [
|
|
214
|
+
{
|
|
215
|
+
"path": "./docs/openapi.yaml",
|
|
216
|
+
"version": "3.0.0",
|
|
217
|
+
"endpointCount": 18,
|
|
218
|
+
"baseUrl": "http://localhost:3000/api/v1",
|
|
219
|
+
"authType": "bearer"
|
|
220
|
+
}
|
|
221
|
+
],
|
|
222
|
+
"playwrightRecordings": [],
|
|
223
|
+
"traceFiles": [
|
|
224
|
+
{
|
|
225
|
+
"path": "./traces/user-session.json",
|
|
226
|
+
"format": "json"
|
|
227
|
+
}
|
|
228
|
+
],
|
|
229
|
+
"notFound": ["Playwright recordings"]
|
|
230
|
+
},
|
|
231
|
+
"apiEndpoints": {
|
|
232
|
+
"totalCount": 18,
|
|
233
|
+
"baseUrl": "http://localhost:3000/api",
|
|
234
|
+
"endpoints": [
|
|
235
|
+
{
|
|
236
|
+
"path": "/users",
|
|
237
|
+
"method": "GET",
|
|
238
|
+
"resourceGroup": "Users",
|
|
239
|
+
"authRequired": true,
|
|
240
|
+
"sourceFile": "src/routes/user.js:10"
|
|
241
|
+
}
|
|
242
|
+
]
|
|
243
|
+
},
|
|
244
|
+
"authentication": {
|
|
245
|
+
"method": "bearer | api-key | oauth2 | basic | jwt | none",
|
|
246
|
+
"configLocation": "path/to/config",
|
|
247
|
+
"envVarsRequired": ["VAR1", "VAR2"],
|
|
248
|
+
"setupExample": "export API_KEY=value"
|
|
249
|
+
},
|
|
250
|
+
"infrastructure": {
|
|
251
|
+
"isContainerized": true,
|
|
252
|
+
"hasDockerCompose": true,
|
|
253
|
+
"hasKubernetes": false,
|
|
254
|
+
"hasCiCd": true,
|
|
255
|
+
"ciCdPlatform": "github-actions"
|
|
256
|
+
},
|
|
257
|
+
"existingTests": {
|
|
258
|
+
"frameworks": ["jest", "supertest"],
|
|
259
|
+
"coverage": {
|
|
260
|
+
"unit": 45,
|
|
261
|
+
"integration": 0,
|
|
262
|
+
"e2e": 0,
|
|
263
|
+
"ui": 0,
|
|
264
|
+
"load": 0,
|
|
265
|
+
"contract": 0,
|
|
266
|
+
"smoke": 0
|
|
267
|
+
},
|
|
268
|
+
"testLocations": {
|
|
269
|
+
"unit": "src/tests/unit/"
|
|
270
|
+
},
|
|
271
|
+
"hasCoverageReports": true,
|
|
272
|
+
"estimatedCoverage": 78
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
\`\`\`
|
|
276
|
+
|
|
277
|
+
VALIDATION CHECKLIST:
|
|
278
|
+
- [ ] Project type identified with evidence
|
|
279
|
+
- [ ] All artifacts searched with file paths
|
|
280
|
+
- [ ] API endpoints counted and categorized
|
|
281
|
+
- [ ] Authentication method determined
|
|
282
|
+
- [ ] Infrastructure flags verified
|
|
283
|
+
- [ ] Existing tests catalogued
|
|
284
|
+
|
|
285
|
+
Begin analysis now. Return ONLY the JSON object, no other text.
|
|
286
|
+
`;
|
|
287
|
+
}
|
|
@@ -0,0 +1,266 @@
|
|
|
1
|
+
import { BASE_SCORES, CONTEXT_RULES } from "../../types/TestMapping.js";
|
|
2
|
+
/**
|
|
3
|
+
* Test Mapping Prompt
|
|
4
|
+
* Prompt for calculating test priority scores
|
|
5
|
+
*/
|
|
6
|
+
export function getTestMappingPrompt(analysis) {
|
|
7
|
+
return `
|
|
8
|
+
Calculate priority scores for Skyramp test types based on repository analysis results and the test definitions below.
|
|
9
|
+
|
|
10
|
+
## CRITICAL RULES
|
|
11
|
+
- Do not suggest test scenarios at this stage.
|
|
12
|
+
|
|
13
|
+
# Skyramp Test Types & Requirements
|
|
14
|
+
|
|
15
|
+
Skyramp offers comprehensive testing capabilities across multiple test types.
|
|
16
|
+
Each test type serves specific purposes and requires different inputs for generation.
|
|
17
|
+
|
|
18
|
+
## 1. Smoke Tests (Base Score: ${BASE_SCORES.smoke})
|
|
19
|
+
|
|
20
|
+
**Purpose**: Quickly verify that an endpoint is accessible and returns a valid response. Ideal for identifying critical defects after significant changes.
|
|
21
|
+
|
|
22
|
+
**Required Inputs**:
|
|
23
|
+
- **OpenAPI Schema** (JSON/YAML) - API specification file
|
|
24
|
+
- **Sample Request Data** (Optional) - JSON blob or file for request body
|
|
25
|
+
- **Endpoint URL** - Target endpoint to test
|
|
26
|
+
- **HTTP Method** (Optional) - Specific method to test
|
|
27
|
+
|
|
28
|
+
**Use Cases**:
|
|
29
|
+
- Post-deployment verification
|
|
30
|
+
- Quick health checks
|
|
31
|
+
- Critical path validation
|
|
32
|
+
|
|
33
|
+
## 2. Contract Tests (Base Score: ${BASE_SCORES.contract})
|
|
34
|
+
|
|
35
|
+
**Purpose**: Ensure services adhere to agreed-upon API contracts, preventing integration issues between services.
|
|
36
|
+
|
|
37
|
+
**Required Inputs**:
|
|
38
|
+
- **OpenAPI Schema** (JSON/YAML) - API specification file
|
|
39
|
+
- **Endpoint URL** - Target endpoint to test
|
|
40
|
+
- **Response Validation** (Optional) - Expected response structure
|
|
41
|
+
|
|
42
|
+
**Use Cases**:
|
|
43
|
+
- API compatibility validation
|
|
44
|
+
- Service contract enforcement
|
|
45
|
+
- Breaking change detection
|
|
46
|
+
|
|
47
|
+
## 3. Fuzz Tests (Base Score: ${BASE_SCORES.fuzz})
|
|
48
|
+
|
|
49
|
+
**Purpose**: Identify vulnerabilities and unexpected behaviors by sending random or invalid data to the application.
|
|
50
|
+
|
|
51
|
+
**Required Inputs**:
|
|
52
|
+
- **OpenAPI Schema** (JSON/YAML) - API specification file
|
|
53
|
+
- **Endpoint URL** - Target endpoint to test
|
|
54
|
+
- **Fuzz Parameters** (Optional) - Custom fuzzing configuration
|
|
55
|
+
|
|
56
|
+
**Use Cases**:
|
|
57
|
+
- Security vulnerability testing
|
|
58
|
+
- Input validation verification
|
|
59
|
+
- Edge case discovery
|
|
60
|
+
|
|
61
|
+
## 4. Integration Tests (Base Score: ${BASE_SCORES.integration})
|
|
62
|
+
|
|
63
|
+
**Purpose**: Verify that different components of a system work together as expected, ensuring reliable data flow and system behavior.
|
|
64
|
+
|
|
65
|
+
**Required Inputs**:
|
|
66
|
+
- **OpenAPI Schema** (JSON/YAML) - API specification file
|
|
67
|
+
- **Endpoint URL** - Target endpoint to test
|
|
68
|
+
- **Integration Scenarios** (Optional) - Specific integration flows
|
|
69
|
+
|
|
70
|
+
**Use Cases**:
|
|
71
|
+
- Microservices integration
|
|
72
|
+
- Database connectivity testing
|
|
73
|
+
- External API integration
|
|
74
|
+
|
|
75
|
+
## 5. Load Tests (Base Score: ${BASE_SCORES.load})
|
|
76
|
+
|
|
77
|
+
**Purpose**: Assess performance and scalability under various load conditions.
|
|
78
|
+
|
|
79
|
+
**Required Inputs**:
|
|
80
|
+
- **OpenAPI Schema** (JSON/YAML) - API specification file
|
|
81
|
+
- **Endpoint URL** - Target endpoint to test
|
|
82
|
+
- **Load Parameters**:
|
|
83
|
+
- \`--load-duration\` (default: 5 seconds)
|
|
84
|
+
- \`--load-num-threads\` (default: 1)
|
|
85
|
+
- \`--load-target-rps\` (requests per second)
|
|
86
|
+
- \`--load-rampup-duration\` (gradual load increase)
|
|
87
|
+
|
|
88
|
+
**Use Cases**:
|
|
89
|
+
- Performance benchmarking
|
|
90
|
+
- Scalability assessment
|
|
91
|
+
- Stress testing
|
|
92
|
+
|
|
93
|
+
## 6. UI Tests (Base Score: ${BASE_SCORES.ui})
|
|
94
|
+
|
|
95
|
+
**Purpose**: Validate user interface functionality and user experience.
|
|
96
|
+
|
|
97
|
+
**Required Inputs**:
|
|
98
|
+
- **Playwright Recording** - ZIP file containing UI interaction traces
|
|
99
|
+
- **Browser Context** (Optional) - Specific browser/device configurations
|
|
100
|
+
|
|
101
|
+
**Use Cases**:
|
|
102
|
+
- User workflow validation
|
|
103
|
+
- Cross-browser compatibility
|
|
104
|
+
- UI regression testing
|
|
105
|
+
|
|
106
|
+
## 7. End-to-End (E2E) Tests (Base Score: ${BASE_SCORES.e2e})
|
|
107
|
+
|
|
108
|
+
**Purpose**: Test complete application flow from start to finish, ensuring all integrated components function as expected.
|
|
109
|
+
|
|
110
|
+
**Required Inputs**:
|
|
111
|
+
- **Trace File** - JSON file containing API interactions
|
|
112
|
+
- **Playwright Recording** - ZIP file containing UI interactions
|
|
113
|
+
- **Integration Scenarios** - Complete user journey definitions
|
|
114
|
+
|
|
115
|
+
**Use Cases**:
|
|
116
|
+
- Complete user journey testing
|
|
117
|
+
- Full-stack integration validation
|
|
118
|
+
- Business process verification
|
|
119
|
+
|
|
120
|
+
## Test Type Priority Hierarchy (Base Impact Score)
|
|
121
|
+
|
|
122
|
+
Based on impact and value:
|
|
123
|
+
|
|
124
|
+
1. **E2E Tests: ${BASE_SCORES.e2e}**
|
|
125
|
+
- Highest value: validates complete user journeys
|
|
126
|
+
- Catches integration issues across full stack
|
|
127
|
+
- Most realistic representation of user experience
|
|
128
|
+
|
|
129
|
+
2. **UI Tests: ${BASE_SCORES.ui}**
|
|
130
|
+
- Critical for user-facing functionality
|
|
131
|
+
- Prevents UI regressions
|
|
132
|
+
- Validates actual user interactions
|
|
133
|
+
|
|
134
|
+
3. **Integration Tests: ${BASE_SCORES.integration}**
|
|
135
|
+
- Validates component interactions
|
|
136
|
+
- Catches interface mismatches
|
|
137
|
+
- Tests realistic workflows
|
|
138
|
+
|
|
139
|
+
4. **Load Tests: ${BASE_SCORES.load}**
|
|
140
|
+
- Ensures performance under pressure
|
|
141
|
+
- Prevents production outages from traffic spikes
|
|
142
|
+
- Validates scalability
|
|
143
|
+
|
|
144
|
+
5. **Fuzz Tests: ${BASE_SCORES.fuzz}**
|
|
145
|
+
- Uncovers security vulnerabilities
|
|
146
|
+
- Finds edge cases
|
|
147
|
+
- Validates input handling
|
|
148
|
+
|
|
149
|
+
6. **Contract Tests: ${BASE_SCORES.contract}**
|
|
150
|
+
- Prevents API breaking changes
|
|
151
|
+
- Ensures service compatibility
|
|
152
|
+
- Important for microservices
|
|
153
|
+
|
|
154
|
+
7. **Smoke Tests: ${BASE_SCORES.smoke}**
|
|
155
|
+
- Basic verification only
|
|
156
|
+
- Quick feedback but shallow coverage
|
|
157
|
+
- Catches only obvious failures
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## Contextual Score Adjustment by Test Type
|
|
162
|
+
|
|
163
|
+
### Scoring Formula
|
|
164
|
+
\`\`\`
|
|
165
|
+
_finalScore = _baseScore × contextMultiplier
|
|
166
|
+
\`\`\`
|
|
167
|
+
|
|
168
|
+
### E2E Tests Context Multipliers
|
|
169
|
+
- **Full-stack application**: ×1.2 (full value)
|
|
170
|
+
- **Backend-only API**: ×0.7 (reduced, no UI to test end-to-end)
|
|
171
|
+
- **Frontend SPA without backend**: ×0.9 (slightly reduced)
|
|
172
|
+
- **Library/SDK**: ×0 (not applicable)
|
|
173
|
+
|
|
174
|
+
### UI Tests Context Multipliers
|
|
175
|
+
- **Frontend SPA or full-stack**: ×1.2 (full value)
|
|
176
|
+
- **Backend-only**: ×0 (not applicable)
|
|
177
|
+
|
|
178
|
+
### Integration Tests Context Multipliers
|
|
179
|
+
- **Full-stack application**: ×1.2 (full value)
|
|
180
|
+
- **Microservices**: ×1.1 (service communication important)
|
|
181
|
+
- **Has unit tests, missing integration**: ×1.2 (fill coverage gap)
|
|
182
|
+
|
|
183
|
+
### Load Tests Context Multipliers
|
|
184
|
+
- **Has Kubernetes or Docker Compose**: ×1.2 (scaled infrastructure suggests high traffic)
|
|
185
|
+
- **Daily deployments**: ×1.15 (frequent deploys need performance validation)
|
|
186
|
+
- **CLI tool or library**: ×0.4 (low traffic expected)
|
|
187
|
+
- **Internal tool (<10 users)**: ×0.4 (minimal load)
|
|
188
|
+
|
|
189
|
+
### Fuzz Tests Context Multipliers
|
|
190
|
+
- **Handles payments or PII**: ×1.2 (security critical)
|
|
191
|
+
- **OAuth2 authentication**: ×1.15 (public API indicator)
|
|
192
|
+
- **Public-facing API**: ×1.2 (higher security needs)
|
|
193
|
+
- **Internal service**: ×0.9 (lower security priority)
|
|
194
|
+
|
|
195
|
+
### Contract Tests Context Multipliers
|
|
196
|
+
- **Microservices architecture**: ×1.2 (critical for service contracts)
|
|
197
|
+
- **Multiple services detected**: ×1.15 (service interactions important)
|
|
198
|
+
- **Monolithic application**: ×0.9 (less critical)
|
|
199
|
+
|
|
200
|
+
### Smoke Tests Context Multipliers
|
|
201
|
+
- **No existing tests**: ×1.2 (quick wins needed)
|
|
202
|
+
- **Production system**: ×1.1 (deployment validation)
|
|
203
|
+
- **Already has comprehensive tests**: ×0.8 (less valuable)
|
|
204
|
+
|
|
205
|
+
---
|
|
206
|
+
|
|
207
|
+
## Context Multiplier Rules (Programmatic)
|
|
208
|
+
|
|
209
|
+
${generateContextRulesTable()}
|
|
210
|
+
|
|
211
|
+
## Repository Analysis
|
|
212
|
+
|
|
213
|
+
\`\`\`json
|
|
214
|
+
${JSON.stringify(analysis, null, 2)}
|
|
215
|
+
\`\`\`
|
|
216
|
+
|
|
217
|
+
## Your Task
|
|
218
|
+
|
|
219
|
+
Calculate priority scores for ALL test types and return a JSON object with this structure:
|
|
220
|
+
|
|
221
|
+
\`\`\`json
|
|
222
|
+
{
|
|
223
|
+
"priorityScores": [
|
|
224
|
+
{
|
|
225
|
+
"testType": "integration",
|
|
226
|
+
"_baseScore": 85,
|
|
227
|
+
"contextMultiplier": 1.2,
|
|
228
|
+
"_finalScore": 102,
|
|
229
|
+
"feasibility": "high",
|
|
230
|
+
"requiredArtifacts": {
|
|
231
|
+
"available": ["openApiSpec"],
|
|
232
|
+
"missing": []
|
|
233
|
+
},
|
|
234
|
+
"reasoning": "Detailed explanation of why this score was calculated"
|
|
235
|
+
}
|
|
236
|
+
],
|
|
237
|
+
"contextFactors": {
|
|
238
|
+
"applied": [
|
|
239
|
+
{
|
|
240
|
+
"factor": "hasDockerCompose",
|
|
241
|
+
"impact": "Increases integration test importance",
|
|
242
|
+
"multiplier": 1.1
|
|
243
|
+
}
|
|
244
|
+
]
|
|
245
|
+
},
|
|
246
|
+
"summary": {
|
|
247
|
+
"highPriority": ["integration", "fuzz"],
|
|
248
|
+
"mediumPriority": ["contract", "load"],
|
|
249
|
+
"lowPriority": ["smoke"]
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
\`\`\`
|
|
253
|
+
|
|
254
|
+
Calculate scores now. Return ONLY the JSON object, no other text.
|
|
255
|
+
`;
|
|
256
|
+
}
|
|
257
|
+
function generateContextRulesTable() {
|
|
258
|
+
let table = "";
|
|
259
|
+
for (const [testType, rules] of Object.entries(CONTEXT_RULES)) {
|
|
260
|
+
table += `\n**${testType.toUpperCase()} Context Multipliers:**\n`;
|
|
261
|
+
for (const rule of rules) {
|
|
262
|
+
table += `- ${rule.condition}: ×${rule.multiplier} (${rule.description})\n`;
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
return table;
|
|
266
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test Recommendation Prompt
|
|
3
|
+
* Prompt for generating actionable test recommendations
|
|
4
|
+
*/
|
|
5
|
+
export function getTestRecommendationPrompt(mapping, analysis, topN = 7) {
|
|
6
|
+
return `
|
|
7
|
+
Generate actionable test recommendations based on priority and repository analysis.
|
|
8
|
+
|
|
9
|
+
**MANDATORY RULES**:
|
|
10
|
+
1. **Priority Scores Must Remain Unchanged**: When a test type is missing required inputs (e.g., Playwright recordings, traces), do NOT:
|
|
11
|
+
- Mark the test as "blocked" in the output
|
|
12
|
+
- Adjust or reduce the priority score
|
|
13
|
+
- Exclude it from recommendations if it ranks in the top ${topN}
|
|
14
|
+
|
|
15
|
+
# Priority
|
|
16
|
+
|
|
17
|
+
\`\`\`json
|
|
18
|
+
${JSON.stringify(mapping, null, 2)}
|
|
19
|
+
\`\`\`
|
|
20
|
+
|
|
21
|
+
# Repository Context
|
|
22
|
+
|
|
23
|
+
\`\`\`json
|
|
24
|
+
${JSON.stringify(analysis, null, 2)}
|
|
25
|
+
\`\`\`
|
|
26
|
+
|
|
27
|
+
# Your Task
|
|
28
|
+
|
|
29
|
+
Generate specific, actionable test recommendations for the top ${topN} highest-scoring test types.
|
|
30
|
+
|
|
31
|
+
## CRITICAL RULES
|
|
32
|
+
- **SCORING**: Use the numeric scores ONLY internally to:
|
|
33
|
+
1. Select the top ${topN} test types by score (highest to lowest)
|
|
34
|
+
2. Assign priority levels based on score ranges:
|
|
35
|
+
* Score >= 100: priority = "high"
|
|
36
|
+
* Score 70-99: priority = "medium"
|
|
37
|
+
* Score < 70: priority = "low"
|
|
38
|
+
- **DO NOT include the "score" field** in the JSON output (it has been removed from the schema).
|
|
39
|
+
- DO NOT mention or display numerical scores anywhere in user-facing text.
|
|
40
|
+
- Order test recommendations by their internal priority scores (highest to lowest).
|
|
41
|
+
|
|
42
|
+
## Requirements
|
|
43
|
+
|
|
44
|
+
For each recommended test type:
|
|
45
|
+
1. **Provide rationale for prioritization based on repository attributes.
|
|
46
|
+
2. **Suggest 3-4 ACTUAL tests** that should be created based on individual user flows and API endpoints.
|
|
47
|
+
3. **Include available artifacts** with specific file paths
|
|
48
|
+
4. **Provide guidance** for creating missing artifacts
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
## Output Structure
|
|
52
|
+
|
|
53
|
+
Return a JSON object with this structure:
|
|
54
|
+
|
|
55
|
+
\`\`\`json
|
|
56
|
+
{
|
|
57
|
+
"summary": {
|
|
58
|
+
"totalRecommended": 3,
|
|
59
|
+
"highPriorityCount": 2,
|
|
60
|
+
"estimatedEffort": "4-6 hours for top 3 tests",
|
|
61
|
+
"quickWins": [
|
|
62
|
+
"Smoke tests (OpenAPI available, 30 min)",
|
|
63
|
+
"Contract tests (OpenAPI available, 1 hour)"
|
|
64
|
+
]
|
|
65
|
+
},
|
|
66
|
+
"recommendations": [
|
|
67
|
+
{
|
|
68
|
+
"priority": "high",
|
|
69
|
+
"testType": "integration",
|
|
70
|
+
"rationale": "Detailed explanation of why this is high priority based on repository characteristics (CRITICAL: DO NOT MENTION THE NUMERICAL SCORE HERE)",
|
|
71
|
+
"specificTests": [
|
|
72
|
+
{
|
|
73
|
+
"testName": "User Registration and Authentication Flow",
|
|
74
|
+
"description": "Test complete user lifecycle: register → verify → login → access protected resource",
|
|
75
|
+
"targetEndpoint": "/api/v1/auth/register, /api/v1/auth/login",
|
|
76
|
+
"targetFlow": "New user onboarding",
|
|
77
|
+
"requiredInputs": {
|
|
78
|
+
"available": [
|
|
79
|
+
{
|
|
80
|
+
"name": "openApiSpec",
|
|
81
|
+
"path": "./docs/openapi.yaml"
|
|
82
|
+
}
|
|
83
|
+
],
|
|
84
|
+
"missing": [
|
|
85
|
+
{
|
|
86
|
+
"name": "traceFile",
|
|
87
|
+
"guidance": "Use Skyramp trace collection: Run 'skyramp_start_trace_collection'..."
|
|
88
|
+
}
|
|
89
|
+
]
|
|
90
|
+
},
|
|
91
|
+
"estimatedValue": "High - catches integration bugs between auth and user services"
|
|
92
|
+
}
|
|
93
|
+
],
|
|
94
|
+
"gettingStarted": {
|
|
95
|
+
"prerequisites": [
|
|
96
|
+
"Docker and Docker Compose installed",
|
|
97
|
+
"MongoDB running (via docker-compose up)",
|
|
98
|
+
"JWT_SECRET environment variable set"
|
|
99
|
+
],
|
|
100
|
+
"quickStartCommand": "npm install && docker-compose up -d && npm run test:integration",
|
|
101
|
+
"documentationUrl": "https://www.skyramp.dev/docs/integration-tests"
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
],
|
|
105
|
+
"nextSteps": [
|
|
106
|
+
"1. Start with Integration Tests (highest priority, fills critical gap)",
|
|
107
|
+
"2. Copy the 'User Registration and Authentication Flow' prompt above",
|
|
108
|
+
"3. Run: skyramp_integration_test_generation with the prompt",
|
|
109
|
+
"4. Execute generated tests: npm run test:integration",
|
|
110
|
+
"5. Once integration tests pass, proceed with Fuzz Tests for security"
|
|
111
|
+
]
|
|
112
|
+
}
|
|
113
|
+
\`\`\`
|
|
114
|
+
|
|
115
|
+
## Important Guidelines
|
|
116
|
+
|
|
117
|
+
1. **Be Specific**: Use actual endpoint paths, file paths, and flow names from the repository
|
|
118
|
+
2. **Be Actionable**: Prompts should work without modification
|
|
119
|
+
3. **Highlight Quick Wins**: Highlight tests with all artifacts available
|
|
120
|
+
4. **Provide Guidance**: Clear instructions for creating missing artifacts
|
|
121
|
+
5. **Include Evidence**: Reference specific repository characteristics in rationale
|
|
122
|
+
|
|
123
|
+
Generate recommendations now. Return ONLY the JSON object, no other text.
|
|
124
|
+
`;
|
|
125
|
+
}
|