@kirrosh/zond 0.12.7 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/package.json +2 -6
- package/src/cli/index.ts +8 -207
- package/src/core/generator/guide-builder.ts +53 -1
- package/src/core/generator/index.ts +0 -3
- package/src/core/generator/serializer.ts +46 -10
- package/src/core/parser/schema.ts +51 -4
- package/src/core/parser/types.ts +27 -0
- package/src/core/parser/variables.ts +1 -0
- package/src/core/runner/assertions.ts +126 -0
- package/src/core/runner/executor.ts +120 -2
- package/src/core/runner/expr-eval.ts +41 -0
- package/src/core/runner/transforms.ts +65 -0
- package/src/cli/commands/add-api.ts +0 -53
- package/src/cli/commands/ai-generate.ts +0 -106
- package/src/cli/commands/chat.ts +0 -43
- package/src/cli/commands/collections.ts +0 -41
- package/src/cli/commands/compare.ts +0 -129
- package/src/cli/commands/doctor.ts +0 -127
- package/src/cli/commands/init.ts +0 -84
- package/src/cli/commands/runs.ts +0 -108
- package/src/cli/commands/update.ts +0 -142
- package/src/core/agent/agent-loop.ts +0 -116
- package/src/core/agent/context-manager.ts +0 -41
- package/src/core/agent/system-prompt.ts +0 -27
- package/src/core/agent/tools/diagnose-failure.ts +0 -51
- package/src/core/agent/tools/index.ts +0 -42
- package/src/core/agent/tools/query-results.ts +0 -40
- package/src/core/agent/tools/run-tests.ts +0 -38
- package/src/core/agent/tools/send-request.ts +0 -44
- package/src/core/agent/types.ts +0 -22
- package/src/core/generator/ai/ai-generator.ts +0 -61
- package/src/core/generator/ai/llm-client.ts +0 -159
- package/src/core/generator/ai/output-parser.ts +0 -307
- package/src/core/generator/ai/prompt-builder.ts +0 -153
- package/src/core/generator/ai/types.ts +0 -56
- package/src/tui/chat-ui.ts +0 -150
package/CHANGELOG.md
CHANGED
|
@@ -48,8 +48,25 @@ All notable changes to this project will be documented in this file.
|
|
|
48
48
|
|
|
49
49
|
## [Unreleased]
|
|
50
50
|
|
|
51
|
+
### Removed
|
|
52
|
+
|
|
53
|
+
- **AI subsystem** — removed `ai-generate` CLI, `chat` CLI, AI agent loop, LLM client, TUI chat UI, and all AI SDK dependencies (`ai`, `@ai-sdk/openai`, `@ai-sdk/anthropic`)
|
|
54
|
+
- **CLI commands** — removed `add-api`, `init`, `collections`, `runs`, `compare`, `doctor`, `update` (available via MCP tools or unnecessary)
|
|
55
|
+
- **Directories** — removed `generated/`, `examples/`, `self-tests/`, `apis/`, `docs/archive/`
|
|
56
|
+
- **Files** — removed `seed-demo.ts`, `BACKLOG.md`, `docs/agent.md`
|
|
57
|
+
|
|
51
58
|
### Added
|
|
52
59
|
|
|
60
|
+
- **Extended YAML test format** — 12 new assertion operators, flow control, and data transforms:
|
|
61
|
+
- **Assertion operators**: `not_equals`, `not_contains`, `gte`, `lte`, `length`, `length_gt/gte/lt/lte`
|
|
62
|
+
- **Array assertions**: `each` (every element matches), `contains_item` (at least one matches), `set_equals` (order-independent comparison)
|
|
63
|
+
- **Flow control**: `skip_if` (conditional skip with expression evaluator), `retry_until` (retry with condition/max_attempts/delay_ms), `for_each` (iterate over array)
|
|
64
|
+
- **Data transforms**: `set` steps with directives — `concat`, `append`, `length`, `get`, `first`, `map_field`
|
|
65
|
+
- **Generator**: `{{$isoTimestamp}}` — ISO 8601 timestamp string
|
|
66
|
+
- **Expression evaluator**: supports `==`, `!=`, `>`, `<`, `>=`, `<=` for skip_if/retry_until conditions
|
|
67
|
+
- Guide-builder YAML cheatsheet updated with all new features
|
|
68
|
+
- Full backward compatibility — all existing tests continue to work unchanged
|
|
69
|
+
|
|
53
70
|
- **MCP feedback improvements**
|
|
54
71
|
- `diagnose_failure` now includes `response_headers` in failure output (e.g. `X-Ably-ErrorMessage`)
|
|
55
72
|
- `generate_tests_guide`: annotates `any`-typed request bodies with a warning comment
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@kirrosh/zond",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "API testing platform — define tests in YAML, run from CLI or WebUI, generate from OpenAPI specs",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"module": "index.ts",
|
|
@@ -26,9 +26,8 @@
|
|
|
26
26
|
"scripts": {
|
|
27
27
|
"zond": "bun run src/cli/index.ts",
|
|
28
28
|
"test": "bun run test:unit && bun run test:mocked",
|
|
29
|
-
"test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/
|
|
29
|
+
"test:unit": "bun test tests/db/ tests/parser/ tests/runner/ tests/generator/ tests/core/ tests/cli/args.test.ts tests/cli/ci-init.test.ts tests/cli/commands.test.ts tests/cli/safe-run.test.ts tests/integration/ tests/web/ tests/mcp/tools.test.ts tests/mcp/save-test-suite.test.ts tests/reporter/ tests/version-sync.test.ts",
|
|
30
30
|
"test:mocked": "bun run scripts/run-mocked-tests.ts",
|
|
31
|
-
"test:ai": "bun test tests/ai/",
|
|
32
31
|
"check": "tsc --noEmit --project tsconfig.json",
|
|
33
32
|
"build": "bun build --compile src/cli/index.ts --outfile zond",
|
|
34
33
|
"version:sync": "bun run scripts/sync-version.ts",
|
|
@@ -42,12 +41,9 @@
|
|
|
42
41
|
},
|
|
43
42
|
"dependencies": {
|
|
44
43
|
"@humanwhocodes/momoa": "^2.0.3",
|
|
45
|
-
"@ai-sdk/anthropic": "^2",
|
|
46
|
-
"@ai-sdk/openai": "^2",
|
|
47
44
|
"@hono/zod-openapi": "^1.2.2",
|
|
48
45
|
"@modelcontextprotocol/sdk": "^1.27.1",
|
|
49
46
|
"@readme/openapi-parser": "^5.5.0",
|
|
50
|
-
"ai": "^6",
|
|
51
47
|
"hono": "^4.12.2",
|
|
52
48
|
"openapi-types": "^12.1.3",
|
|
53
49
|
"zod": "^4.3.6"
|
package/src/cli/index.ts
CHANGED
|
@@ -3,18 +3,9 @@
|
|
|
3
3
|
import { runCommand } from "./commands/run.ts";
|
|
4
4
|
import { validateCommand } from "./commands/validate.ts";
|
|
5
5
|
import { serveCommand } from "./commands/serve.ts";
|
|
6
|
-
import { collectionsCommand } from "./commands/collections.ts";
|
|
7
|
-
import { aiGenerateCommand } from "./commands/ai-generate.ts";
|
|
8
6
|
import { mcpCommand } from "./commands/mcp.ts";
|
|
9
|
-
import { initCommand } from "./commands/init.ts";
|
|
10
|
-
import { updateCommand } from "./commands/update.ts";
|
|
11
|
-
import { chatCommand } from "./commands/chat.ts";
|
|
12
|
-
import { runsCommand } from "./commands/runs.ts";
|
|
13
7
|
import { coverageCommand } from "./commands/coverage.ts";
|
|
14
|
-
import { doctorCommand } from "./commands/doctor.ts";
|
|
15
|
-
import { addApiCommand } from "./commands/add-api.ts";
|
|
16
8
|
import { ciInitCommand } from "./commands/ci-init.ts";
|
|
17
|
-
import { compareCommand } from "./commands/compare.ts";
|
|
18
9
|
import { printError } from "./output.ts";
|
|
19
10
|
import { getRuntimeInfo } from "./runtime.ts";
|
|
20
11
|
import { getDb } from "../db/schema.ts";
|
|
@@ -75,51 +66,13 @@ function printUsage(): void {
|
|
|
75
66
|
console.log(`zond - API Testing Platform
|
|
76
67
|
|
|
77
68
|
Usage:
|
|
78
|
-
zond add-api <name> Register a new API (collection)
|
|
79
69
|
zond run <path> Run API tests
|
|
80
70
|
zond validate <path> Validate test files without running
|
|
81
|
-
zond
|
|
82
|
-
zond runs [id] View test run history
|
|
83
|
-
zond coverage --spec <path> --tests <dir> Analyze API test coverage
|
|
84
|
-
zond collections List test collections
|
|
71
|
+
zond coverage Analyze API test coverage
|
|
85
72
|
zond serve Start web dashboard
|
|
86
|
-
zond init Initialize a new zond project
|
|
87
|
-
zond ci init Generate CI/CD workflow (GitHub Actions, GitLab CI)
|
|
88
73
|
zond mcp Start MCP server (stdio transport for AI agents)
|
|
89
74
|
--dir <path> Set working directory (relative paths resolve here)
|
|
90
|
-
zond
|
|
91
|
-
zond compare <runA> <runB> Compare two test runs (regressions/fixes)
|
|
92
|
-
zond doctor Run diagnostic checks
|
|
93
|
-
zond update Update to latest version
|
|
94
|
-
|
|
95
|
-
Options for 'add-api':
|
|
96
|
-
--spec <path-or-url> OpenAPI spec (extracts base_url from servers[0])
|
|
97
|
-
--dir <directory> Base directory (default: ./apis/<name>/)
|
|
98
|
-
--env key=value Set environment variables (repeatable)
|
|
99
|
-
--insecure Skip TLS verification (self-signed certs)
|
|
100
|
-
|
|
101
|
-
Options for 'chat':
|
|
102
|
-
--provider <name> LLM provider: ollama, openai, anthropic, custom (default: ollama)
|
|
103
|
-
--model <name> Model name (default: provider-specific)
|
|
104
|
-
--api-key <key> API key (or set ZOND_AI_KEY env var)
|
|
105
|
-
--base-url <url> Provider base URL override
|
|
106
|
-
--safe Only allow running GET tests (read-only mode)
|
|
107
|
-
|
|
108
|
-
Options for 'runs':
|
|
109
|
-
runs List recent test runs
|
|
110
|
-
runs <id> Show run details with step results
|
|
111
|
-
--limit <n> Number of runs to show (default: 20)
|
|
112
|
-
|
|
113
|
-
Options for 'compare':
|
|
114
|
-
compare <runA> <runB> Compare two run IDs
|
|
115
|
-
Exit code 1 if regressions found, 0 otherwise
|
|
116
|
-
|
|
117
|
-
Options for 'coverage':
|
|
118
|
-
--api <name> Use API collection (auto-resolves spec and tests dir)
|
|
119
|
-
--spec <path> Path to OpenAPI spec (required unless --api used)
|
|
120
|
-
--tests <dir> Path to test files directory (required unless --api used)
|
|
121
|
-
--fail-on-coverage N Exit 1 when coverage percentage is below N (0–100)
|
|
122
|
-
--run-id <number> Cross-reference with a test run for pass/fail/5xx breakdown
|
|
75
|
+
zond ci init Generate CI/CD workflow (GitHub Actions, GitLab CI)
|
|
123
76
|
|
|
124
77
|
Options for 'run':
|
|
125
78
|
--dry-run Show requests without sending them (exit code always 0)
|
|
@@ -135,15 +88,12 @@ Options for 'run':
|
|
|
135
88
|
--safe Run only GET tests (read-only, safe mode)
|
|
136
89
|
--tag <tag> Filter suites by tag (repeatable, comma-separated, OR logic)
|
|
137
90
|
|
|
138
|
-
Options for '
|
|
139
|
-
--api <name> Use API collection (auto-resolves spec and
|
|
140
|
-
--
|
|
141
|
-
--
|
|
142
|
-
--
|
|
143
|
-
--
|
|
144
|
-
--api-key <key> API key (or set ZOND_AI_KEY env var)
|
|
145
|
-
--base-url <url> Provider base URL override
|
|
146
|
-
--output <dir> Output directory (default: ./generated/ai/)
|
|
91
|
+
Options for 'coverage':
|
|
92
|
+
--api <name> Use API collection (auto-resolves spec and tests dir)
|
|
93
|
+
--spec <path> Path to OpenAPI spec (required unless --api used)
|
|
94
|
+
--tests <dir> Path to test files directory (required unless --api used)
|
|
95
|
+
--fail-on-coverage N Exit 1 when coverage percentage is below N (0–100)
|
|
96
|
+
--run-id <number> Cross-reference with a test run for pass/fail/5xx breakdown
|
|
147
97
|
|
|
148
98
|
Options for 'serve':
|
|
149
99
|
--port <port> Server port (default: 8080)
|
|
@@ -186,35 +136,6 @@ async function main(): Promise<number> {
|
|
|
186
136
|
}
|
|
187
137
|
|
|
188
138
|
switch (command) {
|
|
189
|
-
case "add-api": {
|
|
190
|
-
const name = positional[0];
|
|
191
|
-
if (!name) {
|
|
192
|
-
printError("Missing name argument. Usage: zond add-api <name> [--spec <path>] [--dir <dir>]");
|
|
193
|
-
return 2;
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
// Collect all --env flags (parseArgs only stores last one, so re-parse)
|
|
197
|
-
const envValues: string[] = [];
|
|
198
|
-
const rawArgs = process.argv.slice(2);
|
|
199
|
-
for (let i = 0; i < rawArgs.length; i++) {
|
|
200
|
-
if (rawArgs[i] === "--env" && rawArgs[i + 1] && rawArgs[i + 1]!.includes("=")) {
|
|
201
|
-
envValues.push(rawArgs[i + 1]!);
|
|
202
|
-
i++;
|
|
203
|
-
} else if (rawArgs[i]?.startsWith("--env=") && rawArgs[i]!.slice(6).includes("=")) {
|
|
204
|
-
envValues.push(rawArgs[i]!.slice(6));
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
return addApiCommand({
|
|
209
|
-
name,
|
|
210
|
-
spec: typeof flags["spec"] === "string" ? flags["spec"] : undefined,
|
|
211
|
-
dir: typeof flags["dir"] === "string" ? flags["dir"] : undefined,
|
|
212
|
-
envPairs: envValues.length > 0 ? envValues : undefined,
|
|
213
|
-
dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
214
|
-
insecure: flags["insecure"] === true,
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
|
|
218
139
|
case "run": {
|
|
219
140
|
let path = positional[0];
|
|
220
141
|
const apiFlag = typeof flags["api"] === "string" ? flags["api"] : undefined;
|
|
@@ -297,51 +218,6 @@ async function main(): Promise<number> {
|
|
|
297
218
|
return validateCommand({ path });
|
|
298
219
|
}
|
|
299
220
|
|
|
300
|
-
case "ai-generate": {
|
|
301
|
-
let from = flags["from"] as string | undefined;
|
|
302
|
-
let output = typeof flags["output"] === "string" ? flags["output"] : undefined;
|
|
303
|
-
const aiGenApiFlag = typeof flags["api"] === "string" ? flags["api"] : undefined;
|
|
304
|
-
|
|
305
|
-
// Resolve --api to spec and output dir from collection
|
|
306
|
-
if (aiGenApiFlag) {
|
|
307
|
-
try {
|
|
308
|
-
getDb(typeof flags["db"] === "string" ? flags["db"] : undefined);
|
|
309
|
-
const col = findCollectionByNameOrId(aiGenApiFlag);
|
|
310
|
-
if (!col) { printError(`API '${aiGenApiFlag}' not found`); return 1; }
|
|
311
|
-
if (!from && col.openapi_spec) from = col.openapi_spec;
|
|
312
|
-
if (!output && col.test_path) output = col.test_path;
|
|
313
|
-
} catch (err) {
|
|
314
|
-
printError(`Failed to resolve --api: ${(err as Error).message}`);
|
|
315
|
-
return 2;
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
if (typeof from !== "string") {
|
|
320
|
-
printError("Missing --from <spec>. Usage: zond ai-generate --from <spec> --prompt \"...\"");
|
|
321
|
-
return 2;
|
|
322
|
-
}
|
|
323
|
-
const prompt = flags["prompt"];
|
|
324
|
-
if (typeof prompt !== "string") {
|
|
325
|
-
printError("Missing --prompt <text>. Usage: zond ai-generate --from <spec> --prompt \"...\"");
|
|
326
|
-
return 2;
|
|
327
|
-
}
|
|
328
|
-
return aiGenerateCommand({
|
|
329
|
-
from,
|
|
330
|
-
prompt,
|
|
331
|
-
provider: typeof flags["provider"] === "string" ? flags["provider"] : "ollama",
|
|
332
|
-
model: typeof flags["model"] === "string" ? flags["model"] : undefined,
|
|
333
|
-
apiKey: typeof flags["api-key"] === "string" ? flags["api-key"] : undefined,
|
|
334
|
-
baseUrl: typeof flags["base-url"] === "string" ? flags["base-url"] : undefined,
|
|
335
|
-
output,
|
|
336
|
-
});
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
case "collections": {
|
|
340
|
-
return collectionsCommand(
|
|
341
|
-
typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
342
|
-
);
|
|
343
|
-
}
|
|
344
|
-
|
|
345
221
|
case "serve": {
|
|
346
222
|
const portRaw = flags["port"];
|
|
347
223
|
let port: number | undefined;
|
|
@@ -361,12 +237,6 @@ async function main(): Promise<number> {
|
|
|
361
237
|
});
|
|
362
238
|
}
|
|
363
239
|
|
|
364
|
-
case "init": {
|
|
365
|
-
return initCommand({
|
|
366
|
-
force: flags["force"] === true,
|
|
367
|
-
});
|
|
368
|
-
}
|
|
369
|
-
|
|
370
240
|
case "mcp": {
|
|
371
241
|
return mcpCommand({
|
|
372
242
|
dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
@@ -374,49 +244,6 @@ async function main(): Promise<number> {
|
|
|
374
244
|
});
|
|
375
245
|
}
|
|
376
246
|
|
|
377
|
-
case "chat": {
|
|
378
|
-
return chatCommand({
|
|
379
|
-
provider: typeof flags["provider"] === "string" ? flags["provider"] : undefined,
|
|
380
|
-
model: typeof flags["model"] === "string" ? flags["model"] : undefined,
|
|
381
|
-
apiKey: typeof flags["api-key"] === "string" ? flags["api-key"] : undefined,
|
|
382
|
-
baseUrl: typeof flags["base-url"] === "string" ? flags["base-url"] : undefined,
|
|
383
|
-
safe: flags["safe"] === true,
|
|
384
|
-
dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
385
|
-
});
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
case "update": {
|
|
389
|
-
return updateCommand({ force: flags["force"] === true });
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
case "runs": {
|
|
393
|
-
const idRaw = positional[0];
|
|
394
|
-
let runId: number | undefined;
|
|
395
|
-
if (idRaw) {
|
|
396
|
-
runId = parseInt(idRaw, 10);
|
|
397
|
-
if (isNaN(runId)) {
|
|
398
|
-
printError(`Invalid run ID: ${idRaw}`);
|
|
399
|
-
return 2;
|
|
400
|
-
}
|
|
401
|
-
}
|
|
402
|
-
|
|
403
|
-
const limitRaw = flags["limit"];
|
|
404
|
-
let limit: number | undefined;
|
|
405
|
-
if (typeof limitRaw === "string") {
|
|
406
|
-
limit = parseInt(limitRaw, 10);
|
|
407
|
-
if (isNaN(limit) || limit <= 0) {
|
|
408
|
-
printError(`Invalid limit value: ${limitRaw}`);
|
|
409
|
-
return 2;
|
|
410
|
-
}
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
return runsCommand({
|
|
414
|
-
runId,
|
|
415
|
-
limit,
|
|
416
|
-
dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
417
|
-
});
|
|
418
|
-
}
|
|
419
|
-
|
|
420
247
|
case "ci": {
|
|
421
248
|
const ciSub = positional[0];
|
|
422
249
|
if (ciSub !== "init") {
|
|
@@ -433,32 +260,6 @@ async function main(): Promise<number> {
|
|
|
433
260
|
});
|
|
434
261
|
}
|
|
435
262
|
|
|
436
|
-
case "compare": {
|
|
437
|
-
const rawA = positional[0];
|
|
438
|
-
const rawB = positional[1];
|
|
439
|
-
if (!rawA || !rawB) {
|
|
440
|
-
printError("Usage: zond compare <runA> <runB>");
|
|
441
|
-
return 2;
|
|
442
|
-
}
|
|
443
|
-
const runA = parseInt(rawA, 10);
|
|
444
|
-
const runB = parseInt(rawB, 10);
|
|
445
|
-
if (isNaN(runA) || isNaN(runB)) {
|
|
446
|
-
printError("Run IDs must be integers");
|
|
447
|
-
return 2;
|
|
448
|
-
}
|
|
449
|
-
return compareCommand({
|
|
450
|
-
runA,
|
|
451
|
-
runB,
|
|
452
|
-
dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
453
|
-
});
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
case "doctor": {
|
|
457
|
-
return doctorCommand({
|
|
458
|
-
dbPath: typeof flags["db"] === "string" ? flags["db"] : undefined,
|
|
459
|
-
});
|
|
460
|
-
}
|
|
461
|
-
|
|
462
263
|
case "coverage": {
|
|
463
264
|
let spec = flags["spec"] as string | undefined;
|
|
464
265
|
let tests = flags["tests"] as string | undefined;
|
|
@@ -120,7 +120,7 @@ Use \`json:\` for JSON request bodies. Do NOT use \`body:\` — it is not a vali
|
|
|
120
120
|
For form-encoded: use \`form:\` instead of \`json:\`.
|
|
121
121
|
|
|
122
122
|
### Built-in generators
|
|
123
|
-
\`{{$uuid}}\`, \`{{$randomInt}}\`, \`{{$timestamp}}\`, \`{{$randomName}}\`, \`{{$randomEmail}}\`, \`{{$randomString}}\`
|
|
123
|
+
\`{{$uuid}}\`, \`{{$randomInt}}\`, \`{{$timestamp}}\`, \`{{$isoTimestamp}}\`, \`{{$randomName}}\`, \`{{$randomEmail}}\`, \`{{$randomString}}\`
|
|
124
124
|
|
|
125
125
|
### Variable capture & interpolation
|
|
126
126
|
\`\`\`yaml
|
|
@@ -139,6 +139,58 @@ For form-encoded: use \`form:\` instead of \`json:\`.
|
|
|
139
139
|
id: { equals: "{{created_id}}" }
|
|
140
140
|
\`\`\`
|
|
141
141
|
|
|
142
|
+
### Array assertions
|
|
143
|
+
\`\`\`yaml
|
|
144
|
+
items:
|
|
145
|
+
each: # every element must match
|
|
146
|
+
status: { not_equals: "deleted" }
|
|
147
|
+
id: { type: integer }
|
|
148
|
+
|
|
149
|
+
items:
|
|
150
|
+
contains_item: # at least one element matches
|
|
151
|
+
name: { contains: "test" }
|
|
152
|
+
|
|
153
|
+
ids:
|
|
154
|
+
set_equals: [1, 2, 3] # same elements, order-independent
|
|
155
|
+
\`\`\`
|
|
156
|
+
|
|
157
|
+
### Flow control
|
|
158
|
+
\`\`\`yaml
|
|
159
|
+
# skip_if — skip step when condition is true (after variable substitution)
|
|
160
|
+
- name: Delete only if exists
|
|
161
|
+
DELETE: /items/{{item_id}}
|
|
162
|
+
skip_if: "{{item_id}} == 0"
|
|
163
|
+
expect:
|
|
164
|
+
status: 204
|
|
165
|
+
|
|
166
|
+
# retry_until — repeat request until condition met
|
|
167
|
+
- name: Wait for processing
|
|
168
|
+
GET: /jobs/{{job_id}}
|
|
169
|
+
retry_until:
|
|
170
|
+
condition: "{{status}} == completed"
|
|
171
|
+
max_attempts: 5
|
|
172
|
+
delay_ms: 1000
|
|
173
|
+
expect:
|
|
174
|
+
status: 200
|
|
175
|
+
|
|
176
|
+
# for_each — repeat step for each item in array
|
|
177
|
+
- name: Delete item
|
|
178
|
+
DELETE: /items/{{id}}
|
|
179
|
+
for_each:
|
|
180
|
+
var: id
|
|
181
|
+
in: "{{item_ids}}"
|
|
182
|
+
expect:
|
|
183
|
+
status: [200, 204]
|
|
184
|
+
|
|
185
|
+
# set — transform variables without HTTP request
|
|
186
|
+
- name: Extract IDs
|
|
187
|
+
set:
|
|
188
|
+
all_ids: { map_field: ["{{items}}", "id"] }
|
|
189
|
+
count: { length: "{{items}}" }
|
|
190
|
+
first_item: { first: "{{items}}" }
|
|
191
|
+
merged: { concat: ["{{list_a}}", "{{list_b}}"] }
|
|
192
|
+
\`\`\`
|
|
193
|
+
|
|
142
194
|
### Coverage matching
|
|
143
195
|
Use spec paths with \`{param}\` placeholders in the path for coverage to match:
|
|
144
196
|
- Spec says \`GET /products/{id}\` → write \`GET: /products/1\` (hardcode the value)
|
|
@@ -2,9 +2,6 @@ export { readOpenApiSpec, extractEndpoints, extractSecuritySchemes } from "./ope
|
|
|
2
2
|
export { serializeSuite, isRelativeUrl, sanitizeEnvName, resolveSpecPath } from "./serializer.ts";
|
|
3
3
|
export type { RawSuite, RawStep } from "./serializer.ts";
|
|
4
4
|
export { generateFromSchema } from "./data-factory.ts";
|
|
5
|
-
export { generateWithAI } from "./ai/ai-generator.ts";
|
|
6
|
-
export { resolveProviderConfig, PROVIDER_DEFAULTS } from "./ai/types.ts";
|
|
7
|
-
export type { AIProviderConfig, AIGenerateOptions, AIGenerateResult } from "./ai/types.ts";
|
|
8
5
|
export { scanCoveredEndpoints, filterUncoveredEndpoints, normalizePath, specPathToRegex } from "./coverage-scanner.ts";
|
|
9
6
|
export type { CoveredEndpoint } from "./coverage-scanner.ts";
|
|
10
7
|
export { analyzeEndpoints } from "./endpoint-warnings.ts";
|
|
@@ -93,17 +93,53 @@ export function serializeSuite(suite: RawSuite): string {
|
|
|
93
93
|
serializeValue(test.query, 3, lines);
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
//
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
96
|
+
// skip_if
|
|
97
|
+
if (test.skip_if) {
|
|
98
|
+
lines.push(` skip_if: ${yamlScalar(String(test.skip_if))}`);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// retry_until
|
|
102
|
+
if (test.retry_until && typeof test.retry_until === "object") {
|
|
103
|
+
const rt = test.retry_until as Record<string, unknown>;
|
|
104
|
+
lines.push(" retry_until:");
|
|
105
|
+
if (rt.condition !== undefined) lines.push(` condition: ${yamlScalar(String(rt.condition))}`);
|
|
106
|
+
if (rt.max_attempts !== undefined) lines.push(` max_attempts: ${rt.max_attempts}`);
|
|
107
|
+
if (rt.delay_ms !== undefined) lines.push(` delay_ms: ${rt.delay_ms}`);
|
|
100
108
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
109
|
+
|
|
110
|
+
// for_each
|
|
111
|
+
if (test.for_each && typeof test.for_each === "object") {
|
|
112
|
+
const fe = test.for_each as Record<string, unknown>;
|
|
113
|
+
lines.push(" for_each:");
|
|
114
|
+
if (fe.var !== undefined) lines.push(` var: ${yamlScalar(String(fe.var))}`);
|
|
115
|
+
if (fe.in !== undefined) lines.push(` in: ${yamlScalar(String(fe.in))}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// set
|
|
119
|
+
if (test.set && typeof test.set === "object") {
|
|
120
|
+
lines.push(" set:");
|
|
121
|
+
serializeValue(test.set, 3, lines);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// expect
|
|
125
|
+
const hasExpect = test.expect && (test.expect.status !== undefined || test.expect.body);
|
|
126
|
+
if (hasExpect) {
|
|
127
|
+
lines.push(" expect:");
|
|
128
|
+
if (test.expect.status !== undefined) {
|
|
129
|
+
lines.push(` status: ${test.expect.status}`);
|
|
130
|
+
}
|
|
131
|
+
if (test.expect.body) {
|
|
132
|
+
lines.push(" body:");
|
|
133
|
+
for (const [key, rule] of Object.entries(test.expect.body)) {
|
|
134
|
+
lines.push(` ${key}:`);
|
|
135
|
+
for (const [rk, rv] of Object.entries(rule)) {
|
|
136
|
+
if (typeof rv === "object" && rv !== null) {
|
|
137
|
+
lines.push(` ${rk}:`);
|
|
138
|
+
serializeValue(rv, 6, lines);
|
|
139
|
+
} else {
|
|
140
|
+
lines.push(` ${rk}: ${yamlScalar(String(rv))}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
107
143
|
}
|
|
108
144
|
}
|
|
109
145
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
import type { TestSuite, TestStep, AssertionRule, TestStepExpect, SuiteConfig } from "./types.ts";
|
|
2
|
+
import type { TestSuite, TestStep, AssertionRule, TestStepExpect, SuiteConfig, RetryUntil, ForEach } from "./types.ts";
|
|
3
3
|
|
|
4
4
|
const HTTP_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"] as const;
|
|
5
5
|
|
|
@@ -26,11 +26,19 @@ function extractMethodAndPath(raw: unknown): unknown {
|
|
|
26
26
|
return { ...rest, method: foundMethod, path };
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
// set-only step: no HTTP method required
|
|
30
|
+
if (obj.set && !obj.method) {
|
|
31
|
+
return { ...obj, method: "GET", path: "" };
|
|
32
|
+
}
|
|
33
|
+
|
|
29
34
|
return raw;
|
|
30
35
|
}
|
|
31
36
|
|
|
32
37
|
const ASSERTION_KEYS = new Set([
|
|
33
|
-
"capture", "type", "equals", "
|
|
38
|
+
"capture", "type", "equals", "not_equals", "contains", "not_contains",
|
|
39
|
+
"matches", "gt", "lt", "gte", "lte", "exists",
|
|
40
|
+
"length", "length_gt", "length_gte", "length_lt", "length_lte",
|
|
41
|
+
"each", "contains_item", "set_equals",
|
|
34
42
|
]);
|
|
35
43
|
|
|
36
44
|
/**
|
|
@@ -68,7 +76,7 @@ export function flattenBodyAssertions(body: Record<string, unknown>): Record<str
|
|
|
68
76
|
return result;
|
|
69
77
|
}
|
|
70
78
|
|
|
71
|
-
const
|
|
79
|
+
const AssertionRuleSchemaInner: z.ZodType<AssertionRule> = z.preprocess(
|
|
72
80
|
(val) => {
|
|
73
81
|
if (typeof val === "string") return { type: val };
|
|
74
82
|
if (val === null || val === undefined) return { exists: true };
|
|
@@ -86,14 +94,28 @@ const AssertionRuleSchema: z.ZodType<AssertionRule> = z.preprocess(
|
|
|
86
94
|
capture: z.string().optional(),
|
|
87
95
|
type: z.enum(["string", "integer", "number", "boolean", "array", "object"]).optional(),
|
|
88
96
|
equals: z.unknown().optional(),
|
|
97
|
+
not_equals: z.unknown().optional(),
|
|
89
98
|
contains: z.string().optional(),
|
|
99
|
+
not_contains: z.string().optional(),
|
|
90
100
|
matches: z.string().optional(),
|
|
91
101
|
gt: z.number().optional(),
|
|
92
102
|
lt: z.number().optional(),
|
|
103
|
+
gte: z.number().optional(),
|
|
104
|
+
lte: z.number().optional(),
|
|
93
105
|
exists: z.boolean().optional(),
|
|
106
|
+
length: z.number().int().optional(),
|
|
107
|
+
length_gt: z.number().int().optional(),
|
|
108
|
+
length_gte: z.number().int().optional(),
|
|
109
|
+
length_lt: z.number().int().optional(),
|
|
110
|
+
length_lte: z.number().int().optional(),
|
|
111
|
+
each: z.record(z.string(), z.lazy(() => AssertionRuleSchemaInner)).optional(),
|
|
112
|
+
contains_item: z.record(z.string(), z.lazy(() => AssertionRuleSchemaInner)).optional(),
|
|
113
|
+
set_equals: z.unknown().optional(),
|
|
94
114
|
}),
|
|
95
115
|
) as z.ZodType<AssertionRule>;
|
|
96
116
|
|
|
117
|
+
const AssertionRuleSchema = AssertionRuleSchemaInner;
|
|
118
|
+
|
|
97
119
|
const TestStepExpectSchema: z.ZodType<TestStepExpect> = z.preprocess(
|
|
98
120
|
(val) => {
|
|
99
121
|
if (typeof val !== "object" || val === null) return val;
|
|
@@ -117,8 +139,29 @@ const TestStepExpectSchema: z.ZodType<TestStepExpect> = z.preprocess(
|
|
|
117
139
|
}),
|
|
118
140
|
) as z.ZodType<TestStepExpect>;
|
|
119
141
|
|
|
142
|
+
const RetryUntilSchema: z.ZodType<RetryUntil> = z.object({
|
|
143
|
+
condition: z.string(),
|
|
144
|
+
max_attempts: z.number().int().positive(),
|
|
145
|
+
delay_ms: z.number().int().nonnegative(),
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
const ForEachSchema: z.ZodType<ForEach> = z.object({
|
|
149
|
+
var: z.string(),
|
|
150
|
+
in: z.unknown(),
|
|
151
|
+
});
|
|
152
|
+
|
|
120
153
|
const TestStepSchema: z.ZodType<TestStep> = z.preprocess(
|
|
121
|
-
|
|
154
|
+
(raw) => {
|
|
155
|
+
const obj = extractMethodAndPath(raw);
|
|
156
|
+
// Make expect optional for set-only steps
|
|
157
|
+
if (typeof obj === "object" && obj !== null) {
|
|
158
|
+
const o = obj as Record<string, unknown>;
|
|
159
|
+
if (o.set && !o.expect) {
|
|
160
|
+
o.expect = {};
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return obj;
|
|
164
|
+
},
|
|
122
165
|
z.object({
|
|
123
166
|
name: z.string(),
|
|
124
167
|
method: z.enum(HTTP_METHODS),
|
|
@@ -128,6 +171,10 @@ const TestStepSchema: z.ZodType<TestStep> = z.preprocess(
|
|
|
128
171
|
form: z.record(z.string(), z.string()).optional(),
|
|
129
172
|
query: z.record(z.string(), z.string()).optional(),
|
|
130
173
|
expect: TestStepExpectSchema,
|
|
174
|
+
skip_if: z.string().optional(),
|
|
175
|
+
retry_until: RetryUntilSchema.optional(),
|
|
176
|
+
for_each: ForEachSchema.optional(),
|
|
177
|
+
set: z.record(z.string(), z.unknown()).optional(),
|
|
131
178
|
}),
|
|
132
179
|
) as z.ZodType<TestStep>;
|
|
133
180
|
|
package/src/core/parser/types.ts
CHANGED
|
@@ -4,11 +4,23 @@ export interface AssertionRule {
|
|
|
4
4
|
capture?: string;
|
|
5
5
|
type?: "string" | "integer" | "number" | "boolean" | "array" | "object";
|
|
6
6
|
equals?: unknown;
|
|
7
|
+
not_equals?: unknown;
|
|
7
8
|
contains?: string;
|
|
9
|
+
not_contains?: string;
|
|
8
10
|
matches?: string;
|
|
9
11
|
gt?: number;
|
|
10
12
|
lt?: number;
|
|
13
|
+
gte?: number;
|
|
14
|
+
lte?: number;
|
|
11
15
|
exists?: boolean;
|
|
16
|
+
length?: number;
|
|
17
|
+
length_gt?: number;
|
|
18
|
+
length_gte?: number;
|
|
19
|
+
length_lt?: number;
|
|
20
|
+
length_lte?: number;
|
|
21
|
+
each?: Record<string, AssertionRule>;
|
|
22
|
+
contains_item?: Record<string, AssertionRule>;
|
|
23
|
+
set_equals?: unknown;
|
|
12
24
|
}
|
|
13
25
|
|
|
14
26
|
export interface TestStepExpect {
|
|
@@ -18,6 +30,17 @@ export interface TestStepExpect {
|
|
|
18
30
|
duration?: number;
|
|
19
31
|
}
|
|
20
32
|
|
|
33
|
+
export interface RetryUntil {
|
|
34
|
+
condition: string;
|
|
35
|
+
max_attempts: number;
|
|
36
|
+
delay_ms: number;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface ForEach {
|
|
40
|
+
var: string;
|
|
41
|
+
in: unknown;
|
|
42
|
+
}
|
|
43
|
+
|
|
21
44
|
export interface TestStep {
|
|
22
45
|
name: string;
|
|
23
46
|
method: HttpMethod;
|
|
@@ -27,6 +50,10 @@ export interface TestStep {
|
|
|
27
50
|
form?: Record<string, string>;
|
|
28
51
|
query?: Record<string, string>;
|
|
29
52
|
expect: TestStepExpect;
|
|
53
|
+
skip_if?: string;
|
|
54
|
+
retry_until?: RetryUntil;
|
|
55
|
+
for_each?: ForEach;
|
|
56
|
+
set?: Record<string, unknown>;
|
|
30
57
|
}
|
|
31
58
|
|
|
32
59
|
export interface SuiteConfig {
|
|
@@ -19,6 +19,7 @@ function randomChars(len: number): string {
|
|
|
19
19
|
export const GENERATORS: Record<string, () => string | number> = {
|
|
20
20
|
"$uuid": () => crypto.randomUUID(),
|
|
21
21
|
"$timestamp": () => Math.floor(Date.now() / 1000),
|
|
22
|
+
"$isoTimestamp": () => new Date().toISOString(),
|
|
22
23
|
"$randomName": () => randomFrom(NAMES),
|
|
23
24
|
"$randomEmail": () => `${randomChars(8).toLowerCase()}@test.com`,
|
|
24
25
|
"$randomInt": () => Math.floor(Math.random() * 10000),
|