@kirrosh/zond 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +110 -3
- package/README.md +26 -15
- package/package.json +10 -6
- package/src/cli/commands/catalog.ts +62 -0
- package/src/cli/commands/ci-init.ts +12 -6
- package/src/cli/commands/completions.ts +176 -0
- package/src/cli/commands/db.ts +2 -1
- package/src/cli/commands/generate.ts +18 -2
- package/src/cli/commands/init/agents-md.ts +61 -0
- package/src/cli/commands/init/bootstrap.ts +79 -0
- package/src/cli/commands/init/skills.ts +45 -0
- package/src/cli/commands/init/templates/agents.md +73 -0
- package/src/cli/commands/init/templates/markdown.d.ts +4 -0
- package/src/cli/commands/init/templates/skills/scenarios.md +97 -0
- package/src/cli/commands/init/templates/skills/zond.md +184 -0
- package/src/cli/commands/init/templates/zond-config.yml +15 -0
- package/src/cli/commands/init.ts +124 -31
- package/src/cli/commands/probe-methods.ts +108 -0
- package/src/cli/commands/probe-validation.ts +124 -0
- package/src/cli/commands/run.ts +99 -10
- package/src/cli/commands/serve.ts +52 -19
- package/src/cli/commands/sync.ts +28 -1
- package/src/cli/commands/update.ts +1 -1
- package/src/cli/commands/use.ts +57 -0
- package/src/cli/index.ts +21 -591
- package/src/cli/program.ts +655 -0
- package/src/cli/version.ts +3 -0
- package/src/core/context/current.ts +35 -0
- package/src/core/diagnostics/db-analysis.ts +11 -2
- package/src/core/diagnostics/render-md.ts +112 -0
- package/src/core/generator/catalog-builder.ts +179 -0
- package/src/core/generator/chunker.ts +14 -2
- package/src/core/generator/data-factory.ts +50 -19
- package/src/core/generator/guide-builder.ts +1 -1
- package/src/core/generator/index.ts +2 -0
- package/src/core/generator/openapi-reader.ts +18 -0
- package/src/core/generator/serializer.ts +11 -2
- package/src/core/generator/suite-generator.ts +106 -7
- package/src/core/meta/types.ts +0 -2
- package/src/core/parser/schema.ts +3 -1
- package/src/core/parser/types.ts +10 -1
- package/src/core/parser/variables.ts +90 -2
- package/src/core/parser/yaml-parser.ts +50 -1
- package/src/core/probe/method-probe.ts +197 -0
- package/src/core/probe/negative-probe.ts +657 -0
- package/src/core/reporter/console.ts +29 -3
- package/src/core/reporter/index.ts +2 -2
- package/src/core/reporter/json.ts +5 -2
- package/src/core/runner/assertions.ts +4 -1
- package/src/core/runner/executor.ts +132 -37
- package/src/core/runner/http-client.ts +40 -5
- package/src/core/runner/rate-limiter.ts +131 -0
- package/src/core/setup-api.ts +4 -1
- package/src/core/workspace/root.ts +94 -0
- package/src/db/schema.ts +4 -1
- package/src/web/routes/api.ts +80 -0
- package/src/web/routes/dashboard.ts +15 -0
- package/src/web/static/style.css +290 -0
- package/src/web/views/explorer-tab.ts +402 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join, resolve } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { upsertAgentsBlock, type AgentsBlockResult } from "./agents-md.ts";
|
|
5
|
+
import { upsertSkills, type SkillResult } from "./skills.ts";
|
|
6
|
+
import zondConfigTemplate from "./templates/zond-config.yml" with { type: "text" };
|
|
7
|
+
|
|
8
|
+
export interface BootstrapOptions {
|
|
9
|
+
cwd?: string;
|
|
10
|
+
/** Whether to write/upsert AGENTS.md. Defaults to true. */
|
|
11
|
+
writeAgents?: boolean;
|
|
12
|
+
/** Whether to write Claude Code skills under .claude/skills/. Defaults to true. */
|
|
13
|
+
writeSkills?: boolean;
|
|
14
|
+
/** Override $HOME — used by tests and intentional overrides. */
|
|
15
|
+
home?: string;
|
|
16
|
+
dryRun?: boolean;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface BootstrapResult {
|
|
20
|
+
cwd: string;
|
|
21
|
+
configPath: string;
|
|
22
|
+
configAction: "created" | "noop";
|
|
23
|
+
apisDir: string;
|
|
24
|
+
apisAction: "created" | "noop";
|
|
25
|
+
agents: AgentsBlockResult | null;
|
|
26
|
+
skills: SkillResult[];
|
|
27
|
+
warnings: string[];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Idempotent workspace bootstrap. Creates `zond.config.yml`, `apis/`, and
|
|
32
|
+
* (unless `writeAgents` is false) `AGENTS.md`.
|
|
33
|
+
*/
|
|
34
|
+
export function bootstrapWorkspace(opts: BootstrapOptions = {}): BootstrapResult {
|
|
35
|
+
const cwd = resolve(opts.cwd ?? process.cwd());
|
|
36
|
+
const warnings: string[] = [];
|
|
37
|
+
const writeAgents = opts.writeAgents ?? true;
|
|
38
|
+
const writeSkills = opts.writeSkills ?? true;
|
|
39
|
+
|
|
40
|
+
// 1. zond.config.yml
|
|
41
|
+
const configPath = join(cwd, "zond.config.yml");
|
|
42
|
+
let configAction: "created" | "noop" = "noop";
|
|
43
|
+
if (!existsSync(configPath)) {
|
|
44
|
+
if (!opts.dryRun) writeFileSync(configPath, zondConfigTemplate, "utf-8");
|
|
45
|
+
configAction = "created";
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// 2. apis/
|
|
49
|
+
const apisDir = join(cwd, "apis");
|
|
50
|
+
let apisAction: "created" | "noop" = "noop";
|
|
51
|
+
if (!existsSync(apisDir)) {
|
|
52
|
+
if (!opts.dryRun) mkdirSync(apisDir, { recursive: true });
|
|
53
|
+
apisAction = "created";
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// 3. AGENTS.md
|
|
57
|
+
let agents: AgentsBlockResult | null = null;
|
|
58
|
+
if (writeAgents) {
|
|
59
|
+
if (!opts.dryRun) {
|
|
60
|
+
agents = upsertAgentsBlock(cwd);
|
|
61
|
+
} else {
|
|
62
|
+
agents = { path: join(cwd, "AGENTS.md"), action: existsSync(join(cwd, "AGENTS.md")) ? "updated" : "created" };
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// 4. .claude/skills/zond-*/SKILL.md
|
|
67
|
+
const skills: SkillResult[] = writeSkills ? upsertSkills(cwd, { dryRun: opts.dryRun }) : [];
|
|
68
|
+
|
|
69
|
+
return {
|
|
70
|
+
cwd,
|
|
71
|
+
configPath,
|
|
72
|
+
configAction,
|
|
73
|
+
apisDir,
|
|
74
|
+
apisAction,
|
|
75
|
+
agents,
|
|
76
|
+
skills,
|
|
77
|
+
warnings,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { dirname, join } from "node:path";
|
|
3
|
+
|
|
4
|
+
import zondSkill from "./templates/skills/zond.md" with { type: "text" };
|
|
5
|
+
import scenariosSkill from "./templates/skills/scenarios.md" with { type: "text" };
|
|
6
|
+
|
|
7
|
+
export interface SkillResult {
|
|
8
|
+
name: string;
|
|
9
|
+
path: string;
|
|
10
|
+
action: "created" | "updated" | "noop";
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
interface SkillTemplate {
|
|
14
|
+
name: string;
|
|
15
|
+
body: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const SKILLS: SkillTemplate[] = [
|
|
19
|
+
{ name: "zond", body: zondSkill },
|
|
20
|
+
{ name: "zond-scenarios", body: scenariosSkill },
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Idempotently writes Claude Code skills into `<cwd>/.claude/skills/<name>/SKILL.md`.
|
|
25
|
+
* Body is identical to the in-binary template — overwrites on drift, noop on match.
|
|
26
|
+
*/
|
|
27
|
+
export function upsertSkills(cwd: string, opts: { dryRun?: boolean } = {}): SkillResult[] {
|
|
28
|
+
return SKILLS.map(({ name, body }) => {
|
|
29
|
+
const path = join(cwd, ".claude", "skills", name, "SKILL.md");
|
|
30
|
+
const desired = body.endsWith("\n") ? body : body + "\n";
|
|
31
|
+
|
|
32
|
+
if (!existsSync(path)) {
|
|
33
|
+
if (!opts.dryRun) {
|
|
34
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
35
|
+
writeFileSync(path, desired, "utf-8");
|
|
36
|
+
}
|
|
37
|
+
return { name, path, action: "created" };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const current = readFileSync(path, "utf-8");
|
|
41
|
+
if (current === desired) return { name, path, action: "noop" };
|
|
42
|
+
if (!opts.dryRun) writeFileSync(path, desired, "utf-8");
|
|
43
|
+
return { name, path, action: "updated" };
|
|
44
|
+
});
|
|
45
|
+
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
## API testing with zond
|
|
2
|
+
|
|
3
|
+
This workspace uses [zond](https://github.com/kirrosh/zond) for API testing. The MCP
|
|
4
|
+
server is **not** configured for this workspace, so use the CLI directly.
|
|
5
|
+
|
|
6
|
+
### Detailed playbooks (skills)
|
|
7
|
+
|
|
8
|
+
Detailed task-specific playbooks live in `.claude/skills/` and are auto-discovered by
|
|
9
|
+
Claude Code. Other agents (Codex, Cursor, Aider) can read them as plain markdown:
|
|
10
|
+
|
|
11
|
+
- `.claude/skills/zond/SKILL.md` — end-to-end API testing: generate from OpenAPI,
|
|
12
|
+
run, diagnose failures, hunt bugs via probes, report coverage. Has explicit
|
|
13
|
+
entry points so narrow requests (only diagnose, only probe) skip earlier phases.
|
|
14
|
+
- `.claude/skills/zond-scenarios/SKILL.md` — author multi-step user-journey tests
|
|
15
|
+
and fixture creation via the API (hand-written YAML with captures,
|
|
16
|
+
`setup: true`, `always: true`). NOT for spec coverage or bug hunting.
|
|
17
|
+
|
|
18
|
+
### Mandatory rules (always-on)
|
|
19
|
+
|
|
20
|
+
- **NEVER** read OpenAPI/Swagger/JSON spec files with Read/cat — use `zond describe`,
|
|
21
|
+
`zond catalog`, or the generated `.api-catalog.yaml`.
|
|
22
|
+
- **NEVER** use curl/wget for ad-hoc requests — use `zond request <method> <url>`.
|
|
23
|
+
- **NEVER** write test YAML from scratch — start with `zond generate <spec> --output <dir>`,
|
|
24
|
+
then edit failing cases.
|
|
25
|
+
- **NEVER** hardcode tokens — put them in `apis/<name>/.env.yaml` (already gitignored)
|
|
26
|
+
and reference as `{{auth_token}}` in test YAML.
|
|
27
|
+
- `--safe` enforces GET-only; never run CRUD tests against production without explicit
|
|
28
|
+
user confirmation and a staging environment.
|
|
29
|
+
- When `zond db diagnose` reports `recommended_action: report_backend_bug` — STOP, do
|
|
30
|
+
not change the test to make it pass.
|
|
31
|
+
|
|
32
|
+
### Workflow — covering an API end-to-end
|
|
33
|
+
|
|
34
|
+
```bash
|
|
35
|
+
# 1. Register the API
|
|
36
|
+
zond init --spec <path-or-url> --name <name> [--base-url <url>]
|
|
37
|
+
zond use <name> # remember as current
|
|
38
|
+
|
|
39
|
+
# 2. Inspect endpoints (avoid reading the raw spec)
|
|
40
|
+
zond catalog <spec> --output apis/<name> # writes .api-catalog.yaml
|
|
41
|
+
zond describe <spec> --compact
|
|
42
|
+
|
|
43
|
+
# 3. Generate test stubs and run smoke (GET-only)
|
|
44
|
+
zond generate <spec> --output apis/<name>/tests --tag smoke
|
|
45
|
+
zond run --safe --json
|
|
46
|
+
|
|
47
|
+
# 4. Diagnose failures
|
|
48
|
+
zond db runs --limit 5
|
|
49
|
+
zond db diagnose <run-id> --json
|
|
50
|
+
|
|
51
|
+
# 5. Coverage gate
|
|
52
|
+
zond coverage --fail-on-coverage 50
|
|
53
|
+
|
|
54
|
+
# 6. CRUD only with explicit user confirmation + staging env
|
|
55
|
+
zond run --tag crud --dry-run # show what would be sent
|
|
56
|
+
zond run --tag crud --env staging
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
### Filtering by tag
|
|
60
|
+
|
|
61
|
+
`--tag <name>` filters suites. If a `setup` suite primes auth tokens, always include
|
|
62
|
+
its tag together with the target group: `--tag crud,setup`.
|
|
63
|
+
|
|
64
|
+
### Auth patterns
|
|
65
|
+
|
|
66
|
+
For in-memory or test backends that issue tokens via login, use a `setup.yaml` suite
|
|
67
|
+
with `setup: true`. Captured variables (e.g. `auth_token`) propagate to subsequent
|
|
68
|
+
suites in the same run. Do NOT hardcode bearer tokens for these flows.
|
|
69
|
+
|
|
70
|
+
### Environments
|
|
71
|
+
|
|
72
|
+
`zond run --env <name>` loads `.env.<name>.yaml` (or `.env.yaml` by default) from the
|
|
73
|
+
API directory. Environment files are auto-gitignored by `zond init`.
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: zond-scenarios
|
|
3
|
+
description: |
|
|
4
|
+
Author multi-step API scenario tests (user journeys) with zond. Use when asked to:
|
|
5
|
+
write a scenario, model a user flow, replay a UI flow via API, chain requests with
|
|
6
|
+
captures, set up test data via API, test a workflow end-to-end. Activates on:
|
|
7
|
+
"user scenario", "API workflow", "multi-step test", "login then ...", "create then ...".
|
|
8
|
+
allowed-tools: [Read, Write, Bash(zond *), Bash(bunx zond *)]
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
# zond — API Scenario Tests
|
|
12
|
+
|
|
13
|
+
CLI-only skill. Scenarios are **hand-written** YAML chaining multiple requests
|
|
14
|
+
with captures (no `zond generate` for scenarios — `generate` only emits per-endpoint suites).
|
|
15
|
+
|
|
16
|
+
## Critical rules
|
|
17
|
+
- **NEVER** run `zond generate` to produce scenarios — write them manually from `.api-catalog.yaml`.
|
|
18
|
+
- **NEVER** read OpenAPI/Swagger specs directly — use `zond catalog` or `zond describe`.
|
|
19
|
+
- **NEVER** invent endpoints — only use entries present in `.api-catalog.yaml`.
|
|
20
|
+
- **Captures are file-scoped** — variables defined in one file do not leak into others
|
|
21
|
+
unless the producing suite is marked `setup: true`.
|
|
22
|
+
- Tag every scenario `[scenario, <flow-name>]`. Run by `--tag scenario` or `--tag <flow>,setup`.
|
|
23
|
+
- Keep one user journey per file; chain steps within that file.
|
|
24
|
+
|
|
25
|
+
## Workflow
|
|
26
|
+
```bash
|
|
27
|
+
# 1. Make sure the catalog is current
|
|
28
|
+
zond catalog <spec> --output apis/<name>/tests
|
|
29
|
+
|
|
30
|
+
# 2. Read the catalog to pick endpoints (NOT the raw spec)
|
|
31
|
+
cat apis/<name>/tests/.api-catalog.yaml
|
|
32
|
+
|
|
33
|
+
# 3. Author the scenario YAML (see structure below)
|
|
34
|
+
|
|
35
|
+
# 4. Validate + run
|
|
36
|
+
zond validate apis/<name>/tests/scenarios/<flow>.yaml
|
|
37
|
+
zond run apis/<name>/tests/scenarios/<flow>.yaml --json
|
|
38
|
+
|
|
39
|
+
# 5. Diagnose failures
|
|
40
|
+
zond db diagnose <run-id> --json
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Scenario YAML — minimal structure
|
|
44
|
+
```yaml
|
|
45
|
+
name: user_signup_to_first_purchase
|
|
46
|
+
tags: [scenario, signup_purchase]
|
|
47
|
+
steps:
|
|
48
|
+
- name: register
|
|
49
|
+
request:
|
|
50
|
+
method: POST
|
|
51
|
+
url: "{{base_url}}/auth/register"
|
|
52
|
+
body: { email: "{{generate.email}}", password: "{{generate.password}}" }
|
|
53
|
+
expect: { status: 201 }
|
|
54
|
+
capture:
|
|
55
|
+
user_id: "$.id"
|
|
56
|
+
auth_token: "$.token"
|
|
57
|
+
|
|
58
|
+
- name: create_cart
|
|
59
|
+
request:
|
|
60
|
+
method: POST
|
|
61
|
+
url: "{{base_url}}/carts"
|
|
62
|
+
headers: { Authorization: "Bearer {{auth_token}}" }
|
|
63
|
+
expect: { status: 201 }
|
|
64
|
+
capture: { cart_id: "$.id" }
|
|
65
|
+
|
|
66
|
+
- name: checkout
|
|
67
|
+
request:
|
|
68
|
+
method: POST
|
|
69
|
+
url: "{{base_url}}/carts/{{cart_id}}/checkout"
|
|
70
|
+
headers: { Authorization: "Bearer {{auth_token}}" }
|
|
71
|
+
expect: { status: 200 }
|
|
72
|
+
|
|
73
|
+
- name: cleanup
|
|
74
|
+
always: true # runs even if earlier steps failed
|
|
75
|
+
request:
|
|
76
|
+
method: DELETE
|
|
77
|
+
url: "{{base_url}}/users/{{user_id}}"
|
|
78
|
+
headers: { Authorization: "Bearer {{auth_token}}" }
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
Key building blocks (full reference in `ZOND.md`):
|
|
82
|
+
- `capture: { var: "$.json.path" }` — JSONPath extraction into scenario-local vars.
|
|
83
|
+
- `expect.status` / `expect.json` / `expect.headers` — assertions.
|
|
84
|
+
- `{{generate.email}}`, `{{generate.uuid}}`, `{{generate.int(1,100)}}` — value generators.
|
|
85
|
+
- `always: true` on a step — guaranteed cleanup (runs on prior failure).
|
|
86
|
+
- `setup: true` at the suite level — captures propagate to other suites in the run.
|
|
87
|
+
|
|
88
|
+
## Sharing auth across scenarios
|
|
89
|
+
Put login in `apis/<name>/tests/setup.yaml` with `setup: true`; scenarios reference
|
|
90
|
+
`{{auth_token}}` directly. Run with `--tag <flow>,setup`.
|
|
91
|
+
|
|
92
|
+
## When to hand off
|
|
93
|
+
- Need broad endpoint coverage, bug hunting, or run diagnosis → `zond`.
|
|
94
|
+
- This skill is **only** for hand-written multi-step flows / fixture creation.
|
|
95
|
+
|
|
96
|
+
For full YAML structure (assertions, flow control, generators, conditional steps),
|
|
97
|
+
see the YAML format section of `ZOND.md` at the repo root.
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: zond
|
|
3
|
+
description: |
|
|
4
|
+
End-to-end API testing with zond — generate tests from an OpenAPI spec, run them,
|
|
5
|
+
diagnose failures, and hunt for typical backend bugs via probe suites. Use when
|
|
6
|
+
asked to: test an API, cover endpoints, raise coverage, find bugs, diagnose
|
|
7
|
+
failed runs, fix failing tests, debug 4xx/5xx, run probes, sync tests after a
|
|
8
|
+
spec change, set up API test infrastructure. Activates on: openapi.json,
|
|
9
|
+
openapi.yaml, swagger.json, .api-catalog.yaml, "test this API", "cover this
|
|
10
|
+
spec", "tests are failing", "diagnose run", "find bugs", "probe", "5xx",
|
|
11
|
+
"negative input".
|
|
12
|
+
allowed-tools: [Read, Write, Bash(zond *), Bash(bunx zond *)]
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
# zond — API Coverage, Diagnosis & Bug Hunting
|
|
16
|
+
|
|
17
|
+
CLI-only skill. zond is invoked directly via shell. Run `zond --version` first;
|
|
18
|
+
if missing, install via `curl -fsSL https://raw.githubusercontent.com/kirrosh/zond/master/install.sh | sh`.
|
|
19
|
+
|
|
20
|
+
For multi-step user journeys / fixture creation through the API, hand off to
|
|
21
|
+
`zond-scenarios` instead — that is a different concern.
|
|
22
|
+
|
|
23
|
+
## Critical rules (always-on)
|
|
24
|
+
|
|
25
|
+
- **NEVER** open OpenAPI/Swagger files with Read/cat/grep — use `zond describe`,
|
|
26
|
+
`zond catalog`, or the generated `.api-catalog.yaml`.
|
|
27
|
+
- **NEVER** use curl/wget — use `zond request <method> <url>`.
|
|
28
|
+
- **NEVER** write test YAML from scratch — start with `zond generate`, then edit failures.
|
|
29
|
+
- **NEVER** hardcode tokens — `apis/<name>/.env.yaml` (auto-gitignored), reference as `{{auth_token}}`.
|
|
30
|
+
- **`--safe` enforces GET-only** — required for first-pass smoke against unknown envs.
|
|
31
|
+
- **`recommended_action: report_backend_bug` (5xx) → STOP**. Do NOT modify
|
|
32
|
+
`expect: status` to make the test pass. Surface the bug to the user with the
|
|
33
|
+
request/response excerpt.
|
|
34
|
+
- **5xx in any run is a bug candidate** — never edit assertions to mask it.
|
|
35
|
+
- For multi-suite tag filters always include the setup tag: `--tag crud,setup`.
|
|
36
|
+
- Captures are file-scoped — pass auth across suites via a `setup: true` suite.
|
|
37
|
+
- Re-run after each fix with `--safe --json`; do not batch many edits without verifying.
|
|
38
|
+
|
|
39
|
+
## Entry points (skip phases when the request is narrow)
|
|
40
|
+
|
|
41
|
+
| User asked... | Start at phase | Skip |
|
|
42
|
+
|---|---|---|
|
|
43
|
+
| "cover this API", "raise coverage", "test this spec" | 1 (Discover) | — |
|
|
44
|
+
| "find bugs", "probe this API", "test for 5xx" | 1 then 5 (Probes) | — |
|
|
45
|
+
| "tests are failing", "diagnose run X", "fix failures" | 4 (Diagnose) | 1–3 |
|
|
46
|
+
| "the run after my fix" | 3.x (Run) → 4 (Diagnose) | 1–2 |
|
|
47
|
+
|
|
48
|
+
## Phase 1 — Discover
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
# Workspace + register API (idempotent)
|
|
52
|
+
zond init --with-spec <spec> --name <name> [--base-url <url>]
|
|
53
|
+
zond use <name>
|
|
54
|
+
|
|
55
|
+
# Endpoint discovery (do NOT read the raw spec)
|
|
56
|
+
zond catalog <spec> --output apis/<name>/tests # writes .api-catalog.yaml
|
|
57
|
+
zond describe <spec> --compact # quick overview
|
|
58
|
+
zond guide <spec> --tests-dir apis/<name>/tests # uncovered + suggestions
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Phase 2 — Generate
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
zond generate <spec> --output apis/<name>/tests # all
|
|
65
|
+
zond generate <spec> --output apis/<name>/tests --tag <tag> # by spec tag
|
|
66
|
+
zond generate <spec> --output apis/<name>/tests --uncovered-only
|
|
67
|
+
zond validate apis/<name>/tests # YAML lint
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
`generate` fills bodies with `{{$randomString}}`. Format-strict APIs will reject
|
|
71
|
+
many of these — that is a **test-fix**, not a backend bug. See phase 4 for the fix flow.
|
|
72
|
+
|
|
73
|
+
## Phase 3 — Run (sanity → smoke → full)
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
# 3.1 Sanity gate — must pass before broad runs
|
|
77
|
+
zond run apis/<name>/tests --tag sanity --json
|
|
78
|
+
|
|
79
|
+
# 3.2 Smoke (GET-only)
|
|
80
|
+
zond run apis/<name>/tests --safe --json
|
|
81
|
+
|
|
82
|
+
# 3.3 Full CRUD — only after setup-token capture works
|
|
83
|
+
zond run apis/<name>/tests --tag crud,setup --json
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Phase 4 — Diagnose failures
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
zond db runs --limit 5 --json # find failed runId
|
|
90
|
+
zond db diagnose <run-id> --json # full DiagnoseResult (grouped by root_cause)
|
|
91
|
+
zond db run <id> --status 500 --json # filter results within a run
|
|
92
|
+
zond db run <id> --method POST --json
|
|
93
|
+
zond db compare <idA> <idB> --json # regression diff between runs
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
DiagnoseResult guide:
|
|
97
|
+
- `agent_directive` — literal next step. Do exactly that.
|
|
98
|
+
- `recommended_action`:
|
|
99
|
+
- `fix_test_logic` → edit the YAML (assertion, generator, capture).
|
|
100
|
+
- `report_backend_bug` → STOP, report to user.
|
|
101
|
+
- `update_expectation` → only if the user confirms the new contract is correct.
|
|
102
|
+
- `root_cause` groups identical failures so one fix covers many cases.
|
|
103
|
+
|
|
104
|
+
### 4a. Fixing 4xx caused by stub generators
|
|
105
|
+
|
|
106
|
+
When `recommended_action: fix_test_logic` and the body is rejected on format
|
|
107
|
+
(400/422 with a field name and an "expected ..." message):
|
|
108
|
+
|
|
109
|
+
1. Read the failure body: `zond db run <id> --status 422 --json`.
|
|
110
|
+
2. **First pass** — swap `{{$randomString}}` for the matching typed generator:
|
|
111
|
+
|
|
112
|
+
| API expects | Use |
|
|
113
|
+
|---------------|----------------------|
|
|
114
|
+
| email | `{{$randomEmail}}` |
|
|
115
|
+
| hostname/FQDN | `{{$randomFqdn}}` |
|
|
116
|
+
| URL | `{{$randomUrl}}` |
|
|
117
|
+
| IPv4 | `{{$randomIpv4}}` |
|
|
118
|
+
| UUID | `{{$uuid}}` |
|
|
119
|
+
| integer | `{{$randomInt}}` |
|
|
120
|
+
| ISO date | `{{$randomIsoDate}}` |
|
|
121
|
+
| date | `{{$randomDate}}` |
|
|
122
|
+
| person name | `{{$randomName}}` |
|
|
123
|
+
|
|
124
|
+
3. **Second pass** — if a typed generator still fails (regex too strict, enum,
|
|
125
|
+
business constraint), drop to a hardcoded literal that satisfies the contract
|
|
126
|
+
(e.g. `"https://example.com"`, `"info@example.com"`, `"2026-01-01T00:00:00Z"`).
|
|
127
|
+
4. **Dependent IDs** (`audience_id`, `topic_id`, anything that must reference an
|
|
128
|
+
existing resource) — generators cannot help. Either capture the ID from a
|
|
129
|
+
prior `create_*` step in the same suite, or move that creation into a
|
|
130
|
+
`setup: true` suite and reference the captured variable.
|
|
131
|
+
|
|
132
|
+
## Phase 5 — Proactive bug hunting (probes)
|
|
133
|
+
|
|
134
|
+
Run on a passing API to surface latent bugs.
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
# Negative-input — 5xx on malformed bodies/query/path
|
|
138
|
+
zond probe-validation <spec> --output apis/<name>/probes/validation
|
|
139
|
+
zond run apis/<name>/probes/validation --json
|
|
140
|
+
|
|
141
|
+
# Undeclared HTTP methods — 5xx or unexpected 2xx on methods not in spec
|
|
142
|
+
zond probe-methods <spec> --output apis/<name>/probes/methods
|
|
143
|
+
zond run apis/<name>/probes/methods --json
|
|
144
|
+
|
|
145
|
+
# Triage
|
|
146
|
+
zond db diagnose <run-id> --json
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Typical findings:
|
|
150
|
+
- **5xx on null / empty / oversized body** → missing input validation.
|
|
151
|
+
- **5xx on wrong type** (string for int, etc.) → unguarded coercion.
|
|
152
|
+
- **2xx on undeclared method** → contract drift (spec lies, or method is unprotected).
|
|
153
|
+
- **5xx on missing required field** → uncaught NPE on the server.
|
|
154
|
+
|
|
155
|
+
Filter probe scope when an API is large:
|
|
156
|
+
```bash
|
|
157
|
+
zond probe-validation <spec> --tag <spec-tag> --max-per-endpoint 20
|
|
158
|
+
zond probe-methods <spec> --tag <spec-tag>
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
## Phase 6 — Coverage report & spec drift
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
zond coverage --api <name> --fail-on-coverage 80
|
|
165
|
+
zond coverage --api <name> --run-id <id> # per-run breakdown
|
|
166
|
+
zond sync <spec> --tests apis/<name>/tests # detect new/removed endpoints
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## Auth / environments
|
|
170
|
+
|
|
171
|
+
- Tokens go in `apis/<name>/.env.yaml` (auto-gitignored), referenced as `{{auth_token}}`.
|
|
172
|
+
- Login-flow tokens: a `setup: true` suite captures into vars that propagate to later
|
|
173
|
+
suites in the same run. See `apis/<name>/tests/setup.yaml` examples emitted by `zond generate`.
|
|
174
|
+
- `zond run --env <name>` loads `.env.<name>.yaml` from the API directory.
|
|
175
|
+
|
|
176
|
+
## When to hand off to `zond-scenarios`
|
|
177
|
+
|
|
178
|
+
- The user asks for a multi-step user journey, business flow, or fixture creation
|
|
179
|
+
through the API (login → create cart → checkout → cleanup).
|
|
180
|
+
- A failing run's root cause requires a hand-written multi-step suite that
|
|
181
|
+
`zond generate` cannot express.
|
|
182
|
+
|
|
183
|
+
For YAML format (assertions, generators, captures, `always: true` cleanup,
|
|
184
|
+
`setup: true` propagation), see `ZOND.md` at the repo root or `zond run --help`.
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# zond workspace config
|
|
2
|
+
#
|
|
3
|
+
# The presence of this file marks the workspace root for `zond` walk-up
|
|
4
|
+
# resolution (zond.db, apis/<name>/, .zond-current are anchored here).
|
|
5
|
+
#
|
|
6
|
+
# Schema below is a placeholder — TASK-12 will wire a real config loader.
|
|
7
|
+
# Uncomment and edit as needed.
|
|
8
|
+
|
|
9
|
+
version: 1
|
|
10
|
+
|
|
11
|
+
# default_reporter: console # console | json | junit
|
|
12
|
+
# default_safe: false
|
|
13
|
+
# default_timeout_ms: 30000
|
|
14
|
+
# default_tags: []
|
|
15
|
+
# fail_on_coverage: 0
|
package/src/cli/commands/init.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { setupApi } from "../../core/setup-api.ts";
|
|
1
|
+
import { setupApi, type SetupApiResult } from "../../core/setup-api.ts";
|
|
2
2
|
import { printError, printSuccess } from "../output.ts";
|
|
3
3
|
import { jsonOk, jsonError, printJson } from "../json-envelope.ts";
|
|
4
|
+
import { bootstrapWorkspace, type BootstrapResult } from "./init/bootstrap.ts";
|
|
4
5
|
|
|
5
6
|
export interface InitOptions {
|
|
7
|
+
// register-an-API options (existing)
|
|
6
8
|
name?: string;
|
|
7
9
|
spec?: string;
|
|
8
10
|
baseUrl?: string;
|
|
@@ -11,47 +13,138 @@ export interface InitOptions {
|
|
|
11
13
|
insecure?: boolean;
|
|
12
14
|
dbPath?: string;
|
|
13
15
|
json?: boolean;
|
|
16
|
+
|
|
17
|
+
// workspace bootstrap (new)
|
|
18
|
+
workspace?: boolean;
|
|
19
|
+
withSpec?: string;
|
|
20
|
+
/** Skip writing AGENTS.md. */
|
|
21
|
+
noAgents?: boolean;
|
|
22
|
+
/** Skip writing Claude Code skills under .claude/skills/. */
|
|
23
|
+
noSkills?: boolean;
|
|
24
|
+
/** Override cwd for bootstrap (used by tests; CLI always uses process.cwd()). */
|
|
25
|
+
cwd?: string;
|
|
26
|
+
/** Override $HOME for MCP install (used by tests). */
|
|
27
|
+
home?: string;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
type InitMode = "register" | "workspace" | "bootstrap+register";
|
|
31
|
+
|
|
32
|
+
function resolveMode(options: InitOptions): InitMode {
|
|
33
|
+
if (options.spec) return "register";
|
|
34
|
+
if (options.withSpec) return "bootstrap+register";
|
|
35
|
+
return "workspace";
|
|
14
36
|
}
|
|
15
37
|
|
|
16
38
|
export async function initCommand(options: InitOptions): Promise<number> {
|
|
39
|
+
// Reject conflicting combos
|
|
40
|
+
if (options.spec && options.workspace) {
|
|
41
|
+
const msg = "Cannot use --spec and --workspace together. Use --with-spec to bootstrap and register in one step.";
|
|
42
|
+
if (options.json) printJson(jsonError("init", [msg]));
|
|
43
|
+
else printError(msg);
|
|
44
|
+
return 2;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const mode = resolveMode(options);
|
|
48
|
+
const writeAgents = !options.noAgents;
|
|
49
|
+
const writeSkills = !options.noSkills;
|
|
50
|
+
|
|
17
51
|
try {
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
}
|
|
52
|
+
if (mode === "register") {
|
|
53
|
+
const result = await registerApi(options);
|
|
54
|
+
printRegisterResult(options, result);
|
|
55
|
+
return 0;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const bootstrap = bootstrapWorkspace({ writeAgents, writeSkills, cwd: options.cwd, home: options.home });
|
|
59
|
+
let register: SetupApiResult | null = null;
|
|
60
|
+
|
|
61
|
+
if (mode === "bootstrap+register") {
|
|
62
|
+
register = await registerApi({ ...options, spec: options.withSpec });
|
|
63
|
+
}
|
|
30
64
|
|
|
31
65
|
if (options.json) {
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
66
|
+
const data: Record<string, unknown> = {
|
|
67
|
+
mode,
|
|
68
|
+
configPath: bootstrap.configPath,
|
|
69
|
+
configAction: bootstrap.configAction,
|
|
70
|
+
apisDir: bootstrap.apisDir,
|
|
71
|
+
apisAction: bootstrap.apisAction,
|
|
72
|
+
agentsPath: bootstrap.agents?.path ?? null,
|
|
73
|
+
agentsAction: bootstrap.agents?.action ?? null,
|
|
74
|
+
skills: bootstrap.skills.map((s) => ({ name: s.name, path: s.path, action: s.action })),
|
|
75
|
+
};
|
|
76
|
+
if (register) {
|
|
77
|
+
data.collectionId = register.collectionId;
|
|
78
|
+
data.baseDir = register.baseDir;
|
|
79
|
+
data.testPath = register.testPath;
|
|
80
|
+
data.endpoints = register.specEndpoints;
|
|
45
81
|
}
|
|
82
|
+
printJson(jsonOk("init", data, [...bootstrap.warnings, ...(register?.warnings ?? [])]));
|
|
83
|
+
} else {
|
|
84
|
+
printBootstrapResult(bootstrap, writeAgents);
|
|
85
|
+
if (register) printRegisterResult(options, register);
|
|
46
86
|
}
|
|
47
87
|
return 0;
|
|
48
88
|
} catch (err) {
|
|
49
89
|
const message = err instanceof Error ? err.message : String(err);
|
|
50
|
-
if (options.json)
|
|
51
|
-
|
|
52
|
-
} else {
|
|
53
|
-
printError(message);
|
|
54
|
-
}
|
|
90
|
+
if (options.json) printJson(jsonError("init", [message]));
|
|
91
|
+
else printError(message);
|
|
55
92
|
return 2;
|
|
56
93
|
}
|
|
57
94
|
}
|
|
95
|
+
|
|
96
|
+
async function registerApi(options: InitOptions): Promise<SetupApiResult> {
|
|
97
|
+
const envVars: Record<string, string> = {};
|
|
98
|
+
if (options.baseUrl) envVars.base_url = options.baseUrl;
|
|
99
|
+
|
|
100
|
+
return await setupApi({
|
|
101
|
+
name: options.name,
|
|
102
|
+
spec: options.spec ?? options.withSpec,
|
|
103
|
+
dir: options.dir,
|
|
104
|
+
envVars: Object.keys(envVars).length > 0 ? envVars : undefined,
|
|
105
|
+
dbPath: options.dbPath,
|
|
106
|
+
force: options.force,
|
|
107
|
+
insecure: options.insecure,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function printRegisterResult(options: InitOptions, result: SetupApiResult): void {
|
|
112
|
+
if (options.json) {
|
|
113
|
+
// Only used by the legacy "register"-only path
|
|
114
|
+
printJson(jsonOk("init", {
|
|
115
|
+
mode: "register",
|
|
116
|
+
collectionId: result.collectionId,
|
|
117
|
+
baseDir: result.baseDir,
|
|
118
|
+
testPath: result.testPath,
|
|
119
|
+
endpoints: result.specEndpoints,
|
|
120
|
+
}, result.warnings));
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
printSuccess(`Created API '${options.name ?? "api"}' at ${result.baseDir} (${result.specEndpoints} endpoints)`);
|
|
124
|
+
if (result.warnings) {
|
|
125
|
+
for (const w of result.warnings) process.stderr.write(`Warning: ${w}\n`);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function printBootstrapResult(b: BootstrapResult, writeAgents: boolean): void {
|
|
130
|
+
const lines: string[] = [];
|
|
131
|
+
lines.push(` ${verb(b.configAction)} zond.config.yml`);
|
|
132
|
+
lines.push(` ${verb(b.apisAction)} apis/`);
|
|
133
|
+
if (b.agents) lines.push(` ${verb(b.agents.action)} AGENTS.md`);
|
|
134
|
+
for (const s of b.skills) {
|
|
135
|
+
lines.push(` ${verb(s.action)} .claude/skills/${s.name}/SKILL.md`);
|
|
136
|
+
}
|
|
137
|
+
for (const w of b.warnings) {
|
|
138
|
+
process.stderr.write(`Warning: ${w}\n`);
|
|
139
|
+
}
|
|
140
|
+
process.stdout.write(lines.join("\n") + "\n");
|
|
141
|
+
if (!writeAgents) {
|
|
142
|
+
printSuccess("Workspace ready. Run `zond init --spec <path>` to register your first API.");
|
|
143
|
+
} else {
|
|
144
|
+
printSuccess("Workspace ready. See AGENTS.md for the CLI workflow.");
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function verb(action: "created" | "updated" | "noop"): string {
|
|
149
|
+
return action === "created" ? "Created" : action === "updated" ? "Updated" : "Up-to-date:";
|
|
150
|
+
}
|