@kirrosh/zond 0.14.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -112
- package/README.md +3 -10
- package/package.json +4 -4
- package/src/cli/commands/ci-init.ts +12 -1
- package/src/cli/commands/coverage.ts +21 -1
- package/src/cli/commands/db.ts +121 -0
- package/src/cli/commands/describe.ts +60 -0
- package/src/cli/commands/export.ts +144 -0
- package/src/cli/commands/generate.ts +158 -0
- package/src/cli/commands/guide.ts +127 -0
- package/src/cli/commands/init.ts +57 -0
- package/src/cli/commands/request.ts +57 -0
- package/src/cli/commands/run.ts +74 -14
- package/src/cli/commands/serve.ts +62 -3
- package/src/cli/commands/sync.ts +240 -0
- package/src/cli/commands/validate.ts +18 -2
- package/src/cli/index.ts +258 -17
- package/src/cli/json-envelope.ts +19 -0
- package/src/core/diagnostics/db-analysis.ts +423 -0
- package/src/core/diagnostics/failure-hints.ts +40 -0
- package/src/core/exporter/postman.ts +963 -0
- package/src/core/generator/data-factory.ts +55 -9
- package/src/core/generator/describe.ts +250 -0
- package/src/core/generator/guide-builder.ts +20 -0
- package/src/core/generator/index.ts +1 -1
- package/src/core/generator/openapi-reader.ts +6 -0
- package/src/core/generator/serializer.ts +17 -2
- package/src/core/generator/suite-generator.ts +291 -29
- package/src/core/generator/types.ts +1 -0
- package/src/core/meta/meta-store.ts +78 -0
- package/src/core/meta/types.ts +21 -0
- package/src/core/parser/schema.ts +12 -2
- package/src/core/parser/types.ts +12 -1
- package/src/core/parser/variables.ts +3 -0
- package/src/core/parser/yaml-parser.ts +2 -1
- package/src/core/runner/assertions.ts +44 -20
- package/src/core/runner/execute-run.ts +31 -8
- package/src/core/runner/executor.ts +35 -8
- package/src/core/runner/http-client.ts +1 -1
- package/src/core/runner/send-request.ts +94 -0
- package/src/core/runner/types.ts +2 -0
- package/src/core/sync/spec-differ.ts +38 -0
- package/src/db/queries.ts +4 -2
- package/src/db/schema.ts +11 -3
- package/src/web/views/suites-tab.ts +1 -1
- package/src/cli/commands/mcp.ts +0 -16
- package/src/mcp/descriptions.ts +0 -71
- package/src/mcp/server.ts +0 -45
- package/src/mcp/tools/ci-init.ts +0 -54
- package/src/mcp/tools/coverage-analysis.ts +0 -141
- package/src/mcp/tools/describe-endpoint.ts +0 -242
- package/src/mcp/tools/generate-and-save.ts +0 -202
- package/src/mcp/tools/manage-server.ts +0 -86
- package/src/mcp/tools/query-db.ts +0 -300
- package/src/mcp/tools/run-tests.ts +0 -115
- package/src/mcp/tools/save-test-suite.ts +0 -218
- package/src/mcp/tools/send-request.ts +0 -97
- package/src/mcp/tools/set-work-dir.ts +0 -35
- package/src/mcp/tools/setup-api.ts +0 -88
|
@@ -244,15 +244,24 @@ export function checkAssertions(expect: TestStepExpect, response: HttpResponse):
|
|
|
244
244
|
}
|
|
245
245
|
|
|
246
246
|
if (expect.headers) {
|
|
247
|
-
for (const [key,
|
|
247
|
+
for (const [key, rule] of Object.entries(expect.headers)) {
|
|
248
248
|
const actual = response.headers[key.toLowerCase()];
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
249
|
+
if (typeof rule === "string") {
|
|
250
|
+
results.push({
|
|
251
|
+
field: `headers.${key}`,
|
|
252
|
+
rule: `equals "${rule}"`,
|
|
253
|
+
passed: actual === rule,
|
|
254
|
+
actual,
|
|
255
|
+
expected: rule,
|
|
256
|
+
});
|
|
257
|
+
} else {
|
|
258
|
+
// AssertionRule in header — supports capture and other checks
|
|
259
|
+
const ruleResults = checkRule(key, rule, actual).map(r => ({
|
|
260
|
+
...r,
|
|
261
|
+
field: r.field.replace(/^body\./, "headers."),
|
|
262
|
+
}));
|
|
263
|
+
results.push(...ruleResults);
|
|
264
|
+
}
|
|
256
265
|
}
|
|
257
266
|
}
|
|
258
267
|
|
|
@@ -276,24 +285,39 @@ export function checkAssertions(expect: TestStepExpect, response: HttpResponse):
|
|
|
276
285
|
export function extractCaptures(
|
|
277
286
|
bodyRules: Record<string, AssertionRule> | undefined,
|
|
278
287
|
responseBody: unknown,
|
|
288
|
+
headerRules?: Record<string, string | AssertionRule>,
|
|
289
|
+
responseHeaders?: Record<string, string>,
|
|
279
290
|
): Record<string, unknown> {
|
|
280
291
|
const captures: Record<string, unknown> = {};
|
|
281
|
-
if (!bodyRules || responseBody === undefined) return captures;
|
|
282
292
|
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
293
|
+
if (bodyRules && responseBody !== undefined) {
|
|
294
|
+
for (const [path, rule] of Object.entries(bodyRules)) {
|
|
295
|
+
if (rule.capture) {
|
|
296
|
+
let value: unknown;
|
|
297
|
+
if (path === "_body") {
|
|
298
|
+
value = responseBody;
|
|
299
|
+
} else if (path.startsWith("_body.")) {
|
|
300
|
+
value = getByPath(responseBody, path.slice(6));
|
|
301
|
+
} else {
|
|
302
|
+
value = getByPath(responseBody, path);
|
|
303
|
+
}
|
|
304
|
+
if (value !== undefined) {
|
|
305
|
+
captures[rule.capture] = value;
|
|
306
|
+
}
|
|
292
307
|
}
|
|
293
|
-
|
|
294
|
-
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
if (headerRules && responseHeaders) {
|
|
312
|
+
for (const [key, rule] of Object.entries(headerRules)) {
|
|
313
|
+
if (typeof rule !== "string" && rule.capture) {
|
|
314
|
+
const value = responseHeaders[key.toLowerCase()];
|
|
315
|
+
if (value !== undefined) {
|
|
316
|
+
captures[rule.capture] = value;
|
|
317
|
+
}
|
|
295
318
|
}
|
|
296
319
|
}
|
|
297
320
|
}
|
|
321
|
+
|
|
298
322
|
return captures;
|
|
299
323
|
}
|
|
@@ -8,6 +8,8 @@ import { dirname, resolve } from "path";
|
|
|
8
8
|
import { stat } from "node:fs/promises";
|
|
9
9
|
import type { TestRunResult } from "./types.ts";
|
|
10
10
|
|
|
11
|
+
export const AUTH_PATH_RE = /\/(auth|login|signin|token|oauth)\b/i;
|
|
12
|
+
|
|
11
13
|
export interface ExecuteRunOptions {
|
|
12
14
|
testPath: string;
|
|
13
15
|
envName?: string;
|
|
@@ -52,14 +54,14 @@ export async function executeRun(options: ExecuteRunOptions): Promise<ExecuteRun
|
|
|
52
54
|
}
|
|
53
55
|
}
|
|
54
56
|
|
|
55
|
-
// Safe mode: filter to GET
|
|
57
|
+
// Safe mode: filter to GET + auth endpoints (same logic as run.ts)
|
|
56
58
|
if (safe) {
|
|
57
59
|
for (const suite of suites) {
|
|
58
|
-
suite.tests = suite.tests.filter(t => t.method === "GET");
|
|
60
|
+
suite.tests = suite.tests.filter(t => t.method === "GET" || !t.method || AUTH_PATH_RE.test(t.path));
|
|
59
61
|
}
|
|
60
62
|
suites = suites.filter(s => s.tests.length > 0);
|
|
61
63
|
if (suites.length === 0) {
|
|
62
|
-
throw new Error("No
|
|
64
|
+
throw new Error("No safe tests found. Nothing to run in safe mode.");
|
|
63
65
|
}
|
|
64
66
|
}
|
|
65
67
|
|
|
@@ -83,19 +85,40 @@ export async function executeRun(options: ExecuteRunOptions): Promise<ExecuteRun
|
|
|
83
85
|
return env;
|
|
84
86
|
}
|
|
85
87
|
|
|
86
|
-
|
|
88
|
+
// Phase 1: run setup suites first (sequentially), collect their captures
|
|
89
|
+
const setupSuites = suites.filter(s => s.setup);
|
|
90
|
+
const regularSuites = suites.filter(s => !s.setup);
|
|
91
|
+
const setupResults: Awaited<ReturnType<typeof runSuite>>[] = [];
|
|
92
|
+
const setupCaptures: Record<string, string> = {};
|
|
93
|
+
|
|
94
|
+
for (const suite of setupSuites) {
|
|
95
|
+
const suiteDir = suite.filePath ? dirname(suite.filePath) : envDir;
|
|
96
|
+
const env = await loadEnvWithOverrides(suiteDir);
|
|
97
|
+
const result = await runSuite(suite, env, options.dryRun);
|
|
98
|
+
setupResults.push(result);
|
|
99
|
+
for (const step of result.steps) {
|
|
100
|
+
for (const [k, v] of Object.entries(step.captures)) {
|
|
101
|
+
setupCaptures[k] = String(v);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Phase 2: run regular suites with env enriched by setup captures
|
|
107
|
+
let regularResults: Awaited<ReturnType<typeof runSuite>>[];
|
|
87
108
|
if (isDirectory) {
|
|
88
|
-
|
|
89
|
-
results = await Promise.all(suites.map(async (s) => {
|
|
109
|
+
regularResults = await Promise.all(regularSuites.map(async (s) => {
|
|
90
110
|
const suiteDir = s.filePath ? dirname(s.filePath) : envDir;
|
|
91
111
|
const env = await loadEnvWithOverrides(suiteDir);
|
|
92
|
-
return runSuite(s, env, options.dryRun);
|
|
112
|
+
return runSuite(s, { ...env, ...setupCaptures }, options.dryRun);
|
|
93
113
|
}));
|
|
94
114
|
} else {
|
|
95
115
|
const env = await loadEnvWithOverrides(envDir);
|
|
96
|
-
|
|
116
|
+
const enrichedEnv = { ...env, ...setupCaptures };
|
|
117
|
+
regularResults = await Promise.all(regularSuites.map(s => runSuite(s, enrichedEnv, options.dryRun)));
|
|
97
118
|
}
|
|
98
119
|
|
|
120
|
+
const results = [...setupResults, ...regularResults];
|
|
121
|
+
|
|
99
122
|
const runId = createRun({
|
|
100
123
|
started_at: results[0]?.started_at ?? new Date().toISOString(),
|
|
101
124
|
environment: effectiveEnvName,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { resolve, dirname, basename } from "node:path";
|
|
1
2
|
import type { TestSuite, TestStep, Environment } from "../parser/types.ts";
|
|
2
3
|
import { substituteString, substituteStep, substituteDeep, extractVariableReferences } from "../parser/variables.ts";
|
|
3
4
|
import type { TestRunResult, StepResult, HttpRequest } from "./types.ts";
|
|
@@ -66,16 +67,16 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
66
67
|
const expandedStep = rawSteps[stepIndex + i]!;
|
|
67
68
|
// Temporarily inject into variables when we reach this step
|
|
68
69
|
// We need a way to pass the variable — use a hidden _for_each_vars
|
|
69
|
-
(expandedStep as Record<string, unknown>).__for_each_var = { key: step.for_each.var, value: items[i] };
|
|
70
|
+
(expandedStep as unknown as Record<string, unknown>).__for_each_var = { key: step.for_each.var, value: items[i] };
|
|
70
71
|
}
|
|
71
72
|
continue;
|
|
72
73
|
}
|
|
73
74
|
|
|
74
75
|
// Inject for_each variable if present
|
|
75
|
-
const forEachData = (step as Record<string, unknown>).__for_each_var as { key: string; value: unknown } | undefined;
|
|
76
|
+
const forEachData = (step as unknown as Record<string, unknown>).__for_each_var as { key: string; value: unknown } | undefined;
|
|
76
77
|
if (forEachData) {
|
|
77
78
|
variables[forEachData.key] = forEachData.value;
|
|
78
|
-
delete (step as Record<string, unknown>).__for_each_var;
|
|
79
|
+
delete (step as unknown as Record<string, unknown>).__for_each_var;
|
|
79
80
|
}
|
|
80
81
|
|
|
81
82
|
// Handle set-only steps (no HTTP request)
|
|
@@ -112,6 +113,14 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
112
113
|
}
|
|
113
114
|
}
|
|
114
115
|
|
|
116
|
+
// Process set: on HTTP steps — evaluate generators once before building request
|
|
117
|
+
if (step.set) {
|
|
118
|
+
for (const [key, rawDirective] of Object.entries(step.set)) {
|
|
119
|
+
const substituted = substituteDeep(rawDirective, variables);
|
|
120
|
+
variables[key] = applyTransform(substituted);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
115
124
|
// Substitute variables
|
|
116
125
|
const resolved = substituteStep(step, variables);
|
|
117
126
|
|
|
@@ -121,6 +130,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
121
130
|
const url = buildUrl(resolvedBaseUrl, resolved.path, resolved.query);
|
|
122
131
|
const headers: Record<string, string> = { ...resolvedSuiteHeaders, ...resolved.headers };
|
|
123
132
|
let body: string | undefined;
|
|
133
|
+
let formData: FormData | undefined;
|
|
124
134
|
|
|
125
135
|
if (resolved.json !== undefined) {
|
|
126
136
|
body = JSON.stringify(resolved.json);
|
|
@@ -132,9 +142,23 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
132
142
|
if (!headers["Content-Type"] && !headers["content-type"]) {
|
|
133
143
|
headers["Content-Type"] = "application/x-www-form-urlencoded";
|
|
134
144
|
}
|
|
145
|
+
} else if (resolved.multipart) {
|
|
146
|
+
const basedir = suite.filePath ? dirname(suite.filePath) : process.cwd();
|
|
147
|
+
formData = new FormData();
|
|
148
|
+
for (const [key, field] of Object.entries(resolved.multipart)) {
|
|
149
|
+
if (typeof field === "string") {
|
|
150
|
+
formData.append(key, field);
|
|
151
|
+
} else {
|
|
152
|
+
const absPath = resolve(basedir, field.file);
|
|
153
|
+
const buf = await Bun.file(absPath).arrayBuffer();
|
|
154
|
+
const mime = field.content_type ?? "application/octet-stream";
|
|
155
|
+
const filename = field.filename ?? basename(absPath);
|
|
156
|
+
formData.append(key, new Blob([buf], { type: mime }), filename);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
135
159
|
}
|
|
136
160
|
|
|
137
|
-
const request: HttpRequest = { method: resolved.method, url, headers, body };
|
|
161
|
+
const request: HttpRequest = { method: resolved.method, url, headers, body, formData };
|
|
138
162
|
|
|
139
163
|
// Validate absolute URL before attempting fetch
|
|
140
164
|
if (!url.startsWith("http://") && !url.startsWith("https://")) {
|
|
@@ -156,7 +180,9 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
156
180
|
}
|
|
157
181
|
|
|
158
182
|
if (dryRun) {
|
|
159
|
-
const bodyPreview =
|
|
183
|
+
const bodyPreview = formData
|
|
184
|
+
? ` [multipart: ${[...formData.keys()].length} field(s)]`
|
|
185
|
+
: body ? ` ${body.slice(0, 200)}` : "";
|
|
160
186
|
steps.push({
|
|
161
187
|
name: step.name,
|
|
162
188
|
status: "pass",
|
|
@@ -176,7 +202,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
176
202
|
for (let attempt = 0; attempt < rt.max_attempts; attempt++) {
|
|
177
203
|
try {
|
|
178
204
|
const response = await executeRequest(request, fetchOptions);
|
|
179
|
-
const captures = extractCaptures(resolved.expect.body, response.body_parsed);
|
|
205
|
+
const captures = extractCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
|
|
180
206
|
const assertions = checkAssertions(resolved.expect, response);
|
|
181
207
|
const allPassed = assertions.every((a) => a.passed);
|
|
182
208
|
|
|
@@ -225,8 +251,8 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
225
251
|
try {
|
|
226
252
|
const response = await executeRequest(request, fetchOptions);
|
|
227
253
|
|
|
228
|
-
// Extract captures
|
|
229
|
-
const captures = extractCaptures(resolved.expect.body, response.body_parsed);
|
|
254
|
+
// Extract captures (body + header)
|
|
255
|
+
const captures = extractCaptures(resolved.expect.body, response.body_parsed, resolved.expect.headers, response.headers);
|
|
230
256
|
Object.assign(variables, captures);
|
|
231
257
|
|
|
232
258
|
// Track expected captures that weren't obtained
|
|
@@ -286,6 +312,7 @@ export async function runSuite(suite: TestSuite, env: Environment = {}, dryRun =
|
|
|
286
312
|
suite_name: suite.name,
|
|
287
313
|
suite_tags: suite.tags,
|
|
288
314
|
suite_description: suite.description,
|
|
315
|
+
suite_file: suite.filePath,
|
|
289
316
|
started_at: startedAt,
|
|
290
317
|
finished_at: finishedAt,
|
|
291
318
|
total: steps.length,
|
|
@@ -34,7 +34,7 @@ export async function executeRequest(
|
|
|
34
34
|
const response = await fetch(request.url, {
|
|
35
35
|
method: request.method,
|
|
36
36
|
headers: request.headers,
|
|
37
|
-
body: request.body ?? undefined,
|
|
37
|
+
body: request.formData ?? request.body ?? undefined,
|
|
38
38
|
signal: controller.signal,
|
|
39
39
|
redirect: opts.follow_redirects ? "follow" : "manual",
|
|
40
40
|
tls: { rejectUnauthorized: false },
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { executeRequest } from "./http-client.ts";
|
|
2
|
+
import { loadEnvironment, substituteString, substituteDeep } from "../parser/variables.ts";
|
|
3
|
+
import { getDb } from "../../db/schema.ts";
|
|
4
|
+
import { findCollectionByNameOrId } from "../../db/queries.ts";
|
|
5
|
+
|
|
6
|
+
function extractByPath(obj: unknown, path: string): unknown {
|
|
7
|
+
const segments = path.replace(/\[(\d+)\]/g, '.$1').split('.').filter(Boolean);
|
|
8
|
+
let current: unknown = obj;
|
|
9
|
+
for (const seg of segments) {
|
|
10
|
+
if (current === null || current === undefined) return undefined;
|
|
11
|
+
if (Array.isArray(current)) {
|
|
12
|
+
const idx = parseInt(seg, 10);
|
|
13
|
+
if (isNaN(idx)) return undefined;
|
|
14
|
+
current = current[idx];
|
|
15
|
+
} else if (typeof current === 'object') {
|
|
16
|
+
current = (current as Record<string, unknown>)[seg];
|
|
17
|
+
} else {
|
|
18
|
+
return undefined;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return current;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface SendAdHocRequestOptions {
|
|
25
|
+
method: string;
|
|
26
|
+
url: string;
|
|
27
|
+
headers?: Record<string, string>;
|
|
28
|
+
body?: string;
|
|
29
|
+
timeout?: number;
|
|
30
|
+
envName?: string;
|
|
31
|
+
collectionName?: string;
|
|
32
|
+
jsonPath?: string;
|
|
33
|
+
maxResponseChars?: number;
|
|
34
|
+
dbPath?: string;
|
|
35
|
+
searchDir?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface SendAdHocRequestResult {
|
|
39
|
+
status: number;
|
|
40
|
+
headers: Record<string, string>;
|
|
41
|
+
body: unknown;
|
|
42
|
+
duration_ms: number;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function sendAdHocRequest(options: SendAdHocRequestOptions): Promise<SendAdHocRequestResult> {
|
|
46
|
+
let searchDir = options.searchDir ?? process.cwd();
|
|
47
|
+
if (options.collectionName) {
|
|
48
|
+
getDb(options.dbPath);
|
|
49
|
+
const col = findCollectionByNameOrId(options.collectionName);
|
|
50
|
+
if (col?.base_dir) searchDir = col.base_dir;
|
|
51
|
+
}
|
|
52
|
+
const vars = await loadEnvironment(options.envName, searchDir);
|
|
53
|
+
|
|
54
|
+
const resolvedUrl = substituteString(options.url, vars) as string;
|
|
55
|
+
const parsedHeaders = options.headers ?? {};
|
|
56
|
+
const resolvedHeaders = Object.keys(parsedHeaders).length > 0 ? substituteDeep(parsedHeaders, vars) : {};
|
|
57
|
+
const resolvedBody = options.body ? substituteString(options.body, vars) as string : undefined;
|
|
58
|
+
|
|
59
|
+
// Auto-detect Content-Type for body if not explicitly set
|
|
60
|
+
const finalHeaders: Record<string, string> = { ...resolvedHeaders };
|
|
61
|
+
if (resolvedBody && !finalHeaders["Content-Type"] && !finalHeaders["content-type"]) {
|
|
62
|
+
try {
|
|
63
|
+
JSON.parse(resolvedBody);
|
|
64
|
+
finalHeaders["Content-Type"] = "application/json";
|
|
65
|
+
} catch {
|
|
66
|
+
// Not JSON — don't set content-type, let server decide
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const response = await executeRequest(
|
|
71
|
+
{
|
|
72
|
+
method: options.method,
|
|
73
|
+
url: resolvedUrl,
|
|
74
|
+
headers: finalHeaders,
|
|
75
|
+
body: resolvedBody,
|
|
76
|
+
},
|
|
77
|
+
options.timeout ? { timeout: options.timeout } : undefined,
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
let responseBody: unknown = response.body_parsed ?? response.body;
|
|
81
|
+
|
|
82
|
+
if (options.jsonPath && responseBody !== undefined) {
|
|
83
|
+
responseBody = extractByPath(responseBody, options.jsonPath);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const result: SendAdHocRequestResult = {
|
|
87
|
+
status: response.status,
|
|
88
|
+
headers: response.headers,
|
|
89
|
+
body: responseBody,
|
|
90
|
+
duration_ms: response.duration_ms,
|
|
91
|
+
};
|
|
92
|
+
|
|
93
|
+
return result;
|
|
94
|
+
}
|
package/src/core/runner/types.ts
CHANGED
|
@@ -5,6 +5,7 @@ export interface HttpRequest {
|
|
|
5
5
|
url: string;
|
|
6
6
|
headers: Record<string, string>;
|
|
7
7
|
body?: string;
|
|
8
|
+
formData?: FormData;
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
export interface HttpResponse {
|
|
@@ -38,6 +39,7 @@ export interface TestRunResult {
|
|
|
38
39
|
suite_name: string;
|
|
39
40
|
suite_tags?: string[];
|
|
40
41
|
suite_description?: string;
|
|
42
|
+
suite_file?: string;
|
|
41
43
|
started_at: string;
|
|
42
44
|
finished_at: string;
|
|
43
45
|
total: number;
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { EndpointInfo } from "../generator/types.ts";
|
|
2
|
+
import { normalizePath } from "../generator/coverage-scanner.ts";
|
|
3
|
+
|
|
4
|
+
export interface SpecDiff {
|
|
5
|
+
/** Endpoints in current spec not present in previous snapshot */
|
|
6
|
+
newEndpoints: EndpointInfo[];
|
|
7
|
+
/** Endpoint keys from previous snapshot not present in current spec */
|
|
8
|
+
removedKeys: string[];
|
|
9
|
+
/** True if spec content hash changed (could be just description changes) */
|
|
10
|
+
specChanged: boolean;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** Produce a normalized key for an endpoint: "GET /users/{*}" */
|
|
14
|
+
export function endpointKey(method: string, path: string): string {
|
|
15
|
+
return `${method.toUpperCase()} ${normalizePath(path)}`;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Compare current endpoints against previously-known endpoint keys
|
|
20
|
+
* (stored as strings in .zond-meta.json).
|
|
21
|
+
*/
|
|
22
|
+
export function diffEndpoints(
|
|
23
|
+
prevKeys: string[],
|
|
24
|
+
currentEndpoints: EndpointInfo[],
|
|
25
|
+
): Omit<SpecDiff, "specChanged"> {
|
|
26
|
+
const prevSet = new Set(prevKeys);
|
|
27
|
+
const currentSet = new Set(
|
|
28
|
+
currentEndpoints.map((ep) => endpointKey(ep.method, ep.path)),
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
const newEndpoints = currentEndpoints.filter(
|
|
32
|
+
(ep) => !prevSet.has(endpointKey(ep.method, ep.path)),
|
|
33
|
+
);
|
|
34
|
+
|
|
35
|
+
const removedKeys = [...prevSet].filter((key) => !currentSet.has(key));
|
|
36
|
+
|
|
37
|
+
return { newEndpoints, removedKeys };
|
|
38
|
+
}
|
package/src/db/queries.ts
CHANGED
|
@@ -103,6 +103,7 @@ export interface StoredStepResult {
|
|
|
103
103
|
error_message: string | null;
|
|
104
104
|
assertions: import("../core/runner/types.ts").AssertionResult[];
|
|
105
105
|
captures: Record<string, unknown>;
|
|
106
|
+
suite_file: string | null;
|
|
106
107
|
}
|
|
107
108
|
|
|
108
109
|
// ──────────────────────────────────────────────
|
|
@@ -244,11 +245,11 @@ export function saveResults(runId: number, suiteResults: TestRunResult[]): void
|
|
|
244
245
|
INSERT INTO results
|
|
245
246
|
(run_id, suite_name, test_name, status, duration_ms,
|
|
246
247
|
request_method, request_url, request_body,
|
|
247
|
-
response_status, response_body, response_headers, error_message, assertions, captures)
|
|
248
|
+
response_status, response_body, response_headers, error_message, assertions, captures, suite_file)
|
|
248
249
|
VALUES
|
|
249
250
|
($run_id, $suite_name, $test_name, $status, $duration_ms,
|
|
250
251
|
$request_method, $request_url, $request_body,
|
|
251
|
-
$response_status, $response_body, $response_headers, $error_message, $assertions, $captures)
|
|
252
|
+
$response_status, $response_body, $response_headers, $error_message, $assertions, $captures, $suite_file)
|
|
252
253
|
`);
|
|
253
254
|
|
|
254
255
|
db.transaction(() => {
|
|
@@ -272,6 +273,7 @@ export function saveResults(runId: number, suiteResults: TestRunResult[]): void
|
|
|
272
273
|
$error_message: step.error ?? null,
|
|
273
274
|
$assertions: step.assertions.length > 0 ? JSON.stringify(step.assertions) : null,
|
|
274
275
|
$captures: Object.keys(step.captures).length > 0 ? JSON.stringify(step.captures) : null,
|
|
276
|
+
$suite_file: suite.suite_file ?? null,
|
|
275
277
|
});
|
|
276
278
|
}
|
|
277
279
|
}
|
package/src/db/schema.ts
CHANGED
|
@@ -48,7 +48,7 @@ export function resetDb(): void {
|
|
|
48
48
|
// Schema
|
|
49
49
|
// ──────────────────────────────────────────────
|
|
50
50
|
|
|
51
|
-
const SCHEMA_VERSION =
|
|
51
|
+
const SCHEMA_VERSION = 2;
|
|
52
52
|
|
|
53
53
|
const SCHEMA = `
|
|
54
54
|
CREATE TABLE IF NOT EXISTS runs (
|
|
@@ -82,7 +82,8 @@ const SCHEMA = `
|
|
|
82
82
|
error_message TEXT,
|
|
83
83
|
assertions TEXT,
|
|
84
84
|
captures TEXT,
|
|
85
|
-
response_headers TEXT
|
|
85
|
+
response_headers TEXT,
|
|
86
|
+
suite_file TEXT
|
|
86
87
|
);
|
|
87
88
|
|
|
88
89
|
CREATE TABLE IF NOT EXISTS collections (
|
|
@@ -153,7 +154,14 @@ function runMigrations(db: Database): void {
|
|
|
153
154
|
if (ver >= SCHEMA_VERSION) return;
|
|
154
155
|
|
|
155
156
|
db.transaction(() => {
|
|
156
|
-
|
|
157
|
+
if (ver === 0) {
|
|
158
|
+
// Fresh database — create all tables
|
|
159
|
+
db.exec(SCHEMA);
|
|
160
|
+
}
|
|
161
|
+
if (ver >= 1 && ver < 2) {
|
|
162
|
+
// Migration v1→v2: add suite_file column to results
|
|
163
|
+
db.exec("ALTER TABLE results ADD COLUMN suite_file TEXT");
|
|
164
|
+
}
|
|
157
165
|
db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
|
|
158
166
|
})();
|
|
159
167
|
}
|
|
@@ -8,7 +8,7 @@ import { basename } from "node:path";
|
|
|
8
8
|
|
|
9
9
|
export function renderSuitesTab(state: CollectionState): string {
|
|
10
10
|
if (state.suites.length === 0) {
|
|
11
|
-
return `<div class="tab-empty">No test suites found on disk. Generate tests with <code>
|
|
11
|
+
return `<div class="tab-empty">No test suites found on disk. Generate tests with <code>zond guide</code> or use the test-generation skill.</div>`;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
14
|
const rows = state.suites.map((s, i) => renderSuiteRow(s, i)).join("");
|
package/src/cli/commands/mcp.ts
DELETED
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { startMcpServer } from "../../mcp/server.ts";
|
|
2
|
-
import { resolve } from "node:path";
|
|
3
|
-
|
|
4
|
-
export interface McpCommandOptions {
|
|
5
|
-
dbPath?: string;
|
|
6
|
-
dir?: string;
|
|
7
|
-
}
|
|
8
|
-
|
|
9
|
-
export async function mcpCommand(options: McpCommandOptions): Promise<number> {
|
|
10
|
-
if (options.dir) {
|
|
11
|
-
process.chdir(resolve(options.dir));
|
|
12
|
-
}
|
|
13
|
-
await startMcpServer({ dbPath: options.dbPath });
|
|
14
|
-
// Server runs until stdin closes — this promise never resolves during normal operation
|
|
15
|
-
return 0;
|
|
16
|
-
}
|
package/src/mcp/descriptions.ts
DELETED
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Single source of truth for all MCP tool descriptions.
|
|
3
|
-
* Update descriptions here — they are imported by each tool file.
|
|
4
|
-
*/
|
|
5
|
-
export const TOOL_DESCRIPTIONS = {
|
|
6
|
-
set_work_dir:
|
|
7
|
-
"Set the working directory for this MCP session. " +
|
|
8
|
-
"Call this FIRST before any other tool when using a shared MCP server (npx). " +
|
|
9
|
-
"Determines where zond.db and relative test paths resolve to. " +
|
|
10
|
-
"Pass the absolute path to your project root (same as workspace root in your editor).",
|
|
11
|
-
|
|
12
|
-
setup_api:
|
|
13
|
-
"Register a new API for testing. Creates directory structure, reads OpenAPI spec, " +
|
|
14
|
-
"sets up environment variables, and creates a collection in the database. " +
|
|
15
|
-
"Use this before generating tests for a new API. " +
|
|
16
|
-
"Warns if spec has relative server URL. Use insecure: true for self-signed HTTPS certs.",
|
|
17
|
-
|
|
18
|
-
describe_endpoint:
|
|
19
|
-
"Full details for one endpoint: params grouped by type, request body schema, " +
|
|
20
|
-
"all response schemas + response headers, security, deprecated flag. " +
|
|
21
|
-
"Use when a test fails and you need complete endpoint spec without reading the whole file.",
|
|
22
|
-
|
|
23
|
-
save_test_suite:
|
|
24
|
-
"Save a YAML test suite file with validation. Parses and validates the YAML content " +
|
|
25
|
-
"before writing. Returns structured errors if validation fails so you can fix and retry. " +
|
|
26
|
-
"Use after generating test content with generate_and_save.",
|
|
27
|
-
|
|
28
|
-
save_test_suites:
|
|
29
|
-
"Save multiple YAML test suite files in a single call. Each file is validated before writing. " +
|
|
30
|
-
"Returns per-file results. Use when you have generated multiple suites at once.",
|
|
31
|
-
|
|
32
|
-
run_tests:
|
|
33
|
-
"Execute API tests from a YAML file or directory and return results summary with failures. " +
|
|
34
|
-
"Use after saving test suites with save_test_suite. Check query_db(action: 'diagnose_failure') for detailed failure analysis.",
|
|
35
|
-
|
|
36
|
-
query_db:
|
|
37
|
-
"Query the zond database. Actions: list_collections (all APIs with run stats), " +
|
|
38
|
-
"list_runs (recent test runs), get_run_results (full detail for a run), " +
|
|
39
|
-
"diagnose_failure (only failed/errored steps for a run — each failure includes failure_type: api_error/assertion_failed/network_error, " +
|
|
40
|
-
"and summary includes api_errors/assertion_failures/network_errors counts; stack traces are truncated by default, use verbose: true for full traces), " +
|
|
41
|
-
"compare_runs (regressions and fixes between two runs).",
|
|
42
|
-
|
|
43
|
-
coverage_analysis:
|
|
44
|
-
"Compare an OpenAPI spec against existing test files to find untested endpoints. " +
|
|
45
|
-
"Use to identify gaps and prioritize which endpoints to generate tests for next. " +
|
|
46
|
-
"Pass runId to get enriched pass/fail/5xx breakdown per endpoint. " +
|
|
47
|
-
"Always includes static spec warnings (deprecated, missing response schemas, required params without examples).",
|
|
48
|
-
|
|
49
|
-
send_request:
|
|
50
|
-
"Send an ad-hoc HTTP request. Supports variable interpolation from environments (e.g. {{base_url}}). " +
|
|
51
|
-
"Use jsonPath to extract a subset of the response (e.g. '[0].code'), maxResponseChars to truncate large responses.",
|
|
52
|
-
|
|
53
|
-
manage_server:
|
|
54
|
-
"Start, stop, restart, or check status of the zond WebUI server. " +
|
|
55
|
-
"Useful for viewing test results in a browser without leaving the MCP session.",
|
|
56
|
-
|
|
57
|
-
generate_and_save:
|
|
58
|
-
"Read an OpenAPI spec, auto-chunk by tags if large (>30 endpoints), " +
|
|
59
|
-
"and return a focused test generation guide. For large APIs returns a chunking plan — " +
|
|
60
|
-
"call again with tag parameter for each chunk. Use testsDir param to only generate for uncovered endpoints. " +
|
|
61
|
-
"After generating YAML, use save_test_suites to save files, then run_tests to verify. " +
|
|
62
|
-
"Includes YAML format cheatsheet by default; pass includeFormat: false for subsequent tag chunks to save tokens. " +
|
|
63
|
-
"Use mode: 'generate' to auto-generate and save deterministic YAML test files (smoke + CRUD) without LLM. " +
|
|
64
|
-
"Default mode is 'generate'; use mode: 'guide' for the text-based generation guide.",
|
|
65
|
-
|
|
66
|
-
ci_init:
|
|
67
|
-
"Generate a CI/CD workflow file for running API tests automatically on push, PR, and schedule. " +
|
|
68
|
-
"Supports GitHub Actions and GitLab CI. Auto-detects platform from project structure " +
|
|
69
|
-
"(.github/ → GitHub, .gitlab-ci.yml → GitLab). " +
|
|
70
|
-
"Use after tests are generated and passing. After generating the workflow, help the user commit and push to activate CI.",
|
|
71
|
-
} as const;
|
package/src/mcp/server.ts
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
2
|
-
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
3
|
-
import { registerRunTestsTool } from "./tools/run-tests.ts";
|
|
4
|
-
import { registerQueryDbTool } from "./tools/query-db.ts";
|
|
5
|
-
import { registerSendRequestTool } from "./tools/send-request.ts";
|
|
6
|
-
import { registerCoverageAnalysisTool } from "./tools/coverage-analysis.ts";
|
|
7
|
-
import { registerSaveTestSuiteTool, registerSaveTestSuitesTool } from "./tools/save-test-suite.ts";
|
|
8
|
-
import { registerSetupApiTool } from "./tools/setup-api.ts";
|
|
9
|
-
import { registerManageServerTool } from "./tools/manage-server.ts";
|
|
10
|
-
import { registerCiInitTool } from "./tools/ci-init.ts";
|
|
11
|
-
import { registerSetWorkDirTool } from "./tools/set-work-dir.ts";
|
|
12
|
-
import { registerDescribeEndpointTool } from "./tools/describe-endpoint.ts";
|
|
13
|
-
import { registerGenerateAndSaveTool } from "./tools/generate-and-save.ts";
|
|
14
|
-
import { version } from "../../package.json";
|
|
15
|
-
|
|
16
|
-
export interface McpServerOptions {
|
|
17
|
-
dbPath?: string;
|
|
18
|
-
}
|
|
19
|
-
|
|
20
|
-
export async function startMcpServer(options: McpServerOptions = {}): Promise<void> {
|
|
21
|
-
const { dbPath } = options;
|
|
22
|
-
|
|
23
|
-
const server = new McpServer({
|
|
24
|
-
name: "zond",
|
|
25
|
-
version,
|
|
26
|
-
});
|
|
27
|
-
|
|
28
|
-
// Register all tools
|
|
29
|
-
registerRunTestsTool(server, dbPath);
|
|
30
|
-
registerQueryDbTool(server, dbPath);
|
|
31
|
-
registerSendRequestTool(server, dbPath);
|
|
32
|
-
registerCoverageAnalysisTool(server, dbPath);
|
|
33
|
-
registerSaveTestSuiteTool(server, dbPath);
|
|
34
|
-
registerSaveTestSuitesTool(server, dbPath);
|
|
35
|
-
registerSetupApiTool(server, dbPath);
|
|
36
|
-
registerManageServerTool(server, dbPath);
|
|
37
|
-
registerCiInitTool(server);
|
|
38
|
-
registerSetWorkDirTool(server);
|
|
39
|
-
registerDescribeEndpointTool(server);
|
|
40
|
-
registerGenerateAndSaveTool(server);
|
|
41
|
-
|
|
42
|
-
// Connect via stdio transport
|
|
43
|
-
const transport = new StdioServerTransport();
|
|
44
|
-
await server.connect(transport);
|
|
45
|
-
}
|